grab_rss

#!/usr/bin/python

__copyright__ = 'Copyright 2010, 2012 Jack Lloyd'
__license__   = 'GPLv2'
__author__    = 'Jack Lloyd'
__version__   = '0.4'
__email__     = 'lloyd@randombit.net'
__url__       = 'https://github.com/randombit/grab-rss'

"""
Dependencies:
  feedparser      - http://www.feedparser.org
  stripogram      - http://www.zope.org/Members/chrisw/StripOGram

Optional:
  dateutil.parser - http://labix.org/python-dateutil
  multiprocessing - http://code.google.com/p/python-multiprocessing/
"""

import feedparser
import stripogram
import logging
import optparse
import os
import re
import smtplib
import socket
import sqlite3
import sys
import time
import unicodedata
import ConfigParser

from email.MIMEText import MIMEText

try:
    import dateutil.parser

    def parse_date(date_string):
        dt = dateutil.parser.parse(date_string)
        return dt.strftime('%Y-%m-%d %H:%M')
except ImportError:
    def parse_date(date_string): return date_string # leave it unchanged

def conf_dir():
    if os.getenv('GRAB_RSS_DIR'):
        return os.getenv('GRAB_RSS_DIR')
    return os.path.join(os.getenv('HOME'), '.grab_rss')

def read_config():
    config = ConfigParser.RawConfigParser(
        {
            'from': 'grab-rss@localhost',
            'smtp_host': 'localhost',
            'smtp_port': '25',
            'smtp_auth': 'none',
            'socket_timeout': '30',
            'user_agent': 'grab_rss ' + __version__ + ' ' + __url__,
            'pool_size': 0
        })

    config_file = os.path.join(conf_dir(), 'grab_rss.conf')

    try:
        config.readfp(open(config_file))
    except IOError:
        pass

    return config

class feed_item:
    def __init__(self, filename):
        self.db = sqlite3.connect(filename)
        self.cursor = self.db.cursor()
        self.cursor.execute('create table if not exists feeds (id integer primary key, url text unique, email text)')

    def save(self):
        self.db.commit()

    def emails(self, feed):
        self.cursor.execute("select email from feeds where url = ?", (feed,))
        there = self.cursor.fetchone()
        result = list(there)
        return result

    def get(self):
        self.cursor.execute('select url from feeds')
        result = list(self.cursor.fetchall())
        return result

class seen_items:
    def __init__(self, filename):
        self.db = sqlite3.connect(filename)
        self.cursor = self.db.cursor()
        self.cursor.execute('create table if not exists seen (timestamp integer, url text)')

    def save(self):
        self.db.commit()

    def remove_older_than(self, days):
        long_ago = int(time.time()) - int(days)*24*60

        self.cursor.execute('delete from seen where timestamp <= ?', [long_ago])
        return self.cursor.rowcount

    def seen_this_before(self, item):
        self.cursor.execute('select * from seen where url = ?', [item])
        there = self.cursor.fetchall()
        return len(there) > 0

    def note_as_seen(self, item):
        now = int(time.time())
        self.cursor.execute('insert into seen values(?, ?)', [now, item])

def wrap(text, width):
    # From http://code.activestate.com/recipes/148061/
    return reduce(lambda line, word, width=width: '%s%s%s' %
                  (line,
                   ' \n'[(len(line)-line.rfind('\n')-1
                         + len(word.split('\n',1)[0]
                              ) >= width)],
                   word),
                  text.split(' ')
                 )

def force_to_ascii(s):
    def replace_char(matches):
        u = matches.group(1)
        try:
            return unichr(int(u))
        except:
            return u

    s = re.sub("&#(\d+)(;|(?=\s))", replace_char, s)

    return unicodedata.normalize('NFKD', unicode(s)).encode('ascii', 'ignore')

def body_for(entry):
    timestamp = parse_date(entry.get('date', ''))

    link = force_to_ascii(entry.link)

    descr = wrap(
        stripogram.html2safehtml(
            force_to_ascii(entry.get('description', '')),
            valid_tags=('a')),
        80)

    return '\n\n'.join([timestamp, link, descr])

def parse_feed(url):
    logging.info('Reading %s' % (url))

    def feed_name(feed, feed_url):
        try:
            return feed['feed']['title']
        except:
            s = feed_url[0]
            return s.split('/')[3]

    # Don't return feed directly, PyCObjects are unpicklable and this
    # makes multiprocessing very unhappy
    clean_url = url[0]
    feed = feedparser.parse(clean_url)
    name = feed_name(feed, url)
    return (url, name, feed.entries)

def pool_map(pool_size, func, args):
    if pool_size > 0:
        import multiprocessing
        pool = multiprocessing.Pool(pool_size)
        logging.debug('pool_map')
        return pool.map(func, args)
    else:
        return map(func, args)

fstate = feed_item(os.path.join(conf_dir(), 'seen.db'))

def grab_feeds(state, pool_size):
    feeds = fstate.get()
    all_feeds = pool_map(pool_size, parse_feed, feeds)
    for (feed_url, feed_title, feed) in all_feeds:
        num_entries = len(feed)
        clean_feed_url = feed_url[0]
        if num_entries:
            logging.debug('Found %d entries in %s' % (num_entries, clean_feed_url))
        else:
            logging.info('Found no entries in %s' % (clean_feed_url))

        new_entries = 0

        for entry in feed:
            link = entry['link']

            if state.seen_this_before(link):
                continue

            title = entry.get('title', link)

            msg = MIMEText(body_for(entry))
            msg['X-GrabRSS-Feed'] = clean_feed_url
            msg['Subject'] = force_to_ascii('%s - %s' % (feed_title, title))
            yield msg

            new_entries += 1

            state.note_as_seen(link)

        logging.debug('Saw %d new items from %s' % (new_entries, feed_url))

def main(argv = None):
    if argv is None:
        argv = sys.argv

    parser = optparse.OptionParser(version='%%prog %s' % (__version__))

    parser.add_option('-v', '--verbose', dest='verbose',
                      action='store_true', default=False,
                      help='be loud')

    parser.add_option('-q', '--quiet', dest='quiet',
                      action='store_true', default=False,
                      help='be quiet')

    parser.add_option('--dont-send', action='store_true', default=False,
                      help="don't actually send email")

    parser.add_option('--remove-older-than', dest='remove_older_than',
                      metavar='N',
                      help='remove seen.db entries older than N days')

    parser.add_option('--pool-size', metavar='N',
                      help='use N processes for pulling down feeds in parallel')

    (options, args) = parser.parse_args(argv)

    def log_level():
        if options.quiet: # -q overrides -v
            return logging.WARNING
        if options.verbose:
            return logging.DEBUG
        return logging.INFO

    logging.basicConfig(stream = sys.stdout,
                        format = '%(levelname) 7s: %(message)s',
                        level = log_level())

    config = read_config()

    socket.setdefaulttimeout(config.getint('GrabRSS', 'socket_timeout'))
    feedparser.USER_AGENT = config.get('GrabRSS', 'user_agent')

    pool_size = int(options.pool_size or config.get('GrabRSS', 'pool_size'))

    smtp_host = config.get('GrabRSS', 'smtp_host')
    smtp_port = config.getint('GrabRSS', 'smtp_port')
    smtp_protocol = config.get('GrabRSS', 'smtp_protocol')

    logging.debug('SMTP connection opened to %s' % (smtp_host))

    state = seen_items(os.path.join(conf_dir(), 'seen.db'))
    smtp_auth = config.get('GrabRSS', 'smtp_auth')

    if options.remove_older_than:
        removed = state.remove_older_than(options.remove_older_than)
        logging.info('Removed %d old items' % (removed))
        state.save()
        return

    smtp = smtplib.SMTP(smtp_host,smtp_port)

    if smtp_protocol == "tls":
        smtp.ehlo()
        smtp.starttls()

    if smtp_auth:
        smtp_user = config.get('GrabRSS', 'smtp_user')
        smtp_pass = config.get('GrabRSS', 'smtp_pass')

        smtp.login(smtp_user, smtp_pass)

    for email in grab_feeds(state, pool_size):
        if not options.dont_send:
            email['From'] = config.get('GrabRSS', 'from')
            email['To'] = ", ".join(fstate.emails(email['X-GrabRSS-Feed']))
            smtp.sendmail(email['From'], email['To'].split(','), email.as_string())

    state.save()

    smtp.quit()

if __name__ == '__main__':
    sys.exit(main())