source: rss-mirror.py @ c19ca33

#!/usr/bin/env python
# vim: set fileencoding=utf-8 tabstop=4 shiftwidth=4 expandtab smartindent:
u"""

RSS Mirror
----------

Mirrors on the local disk the pages listed in an RSS feed, using Wget or
HTTrack.

Requires Python >= 2.6


Configuration file
~~~~~~~~~~~~~~~~~~

RSS-mirror uses a configuration file to list the RSS feed that should be
downloaded. This file must be placed in ``~/.config/rss-mirror.conf`` and is in
INI format. Example::

    [DEFAULT]
    output = ~/pda/webpages

    [owni]
    url = http://owni.fr/feed

    [zenhabits]
    url = http://zenhabits.net/feed

    [rue89-ecologie]
    url = http://www.rue89.com/tag/ecologie/feed
    title = Rue89 - Ecologie

The ``DEFAULT`` section has an ``output`` key pointing to the output directory
where the webpages will be downloaded.

Each section (except DEFAULT) is a feed to download. It has a ``url`` key which
is self-explanatory and an optional ``title`` key which will be used as a title
for the feed in the summary page.


Credits
~~~~~~~

.. :Authors:
       Aurélien Bompard <aurelien@bompard.org> <http://aurelien.bompard.org>

.. :License:
       GNU GPL v3 or later

"""

import os
import sys
import urllib
import urllib2
import re
import shutil
import optparse
import datetime
import time
import itertools
import xml.etree.ElementTree as etree
from urlparse import urlparse
from subprocess import call
from zipfile import ZipFile, BadZipfile
from ConfigParser import SafeConfigParser


IUI_VERSION = "0.40-alpha1"
CONFIG_PATH = "~/.config/rss-mirror.conf"


class Downloader(object):
    """
    Abstract downloader class

    :cvar return_codes_ok: list of non-zero return codes that are actually OK
    :type return_codes_ok: ``list``
    """

    return_codes_ok = []

    def get_command(self, destdir, url, options=None):
        """Returns the system command to execute"""
        raise NotImplementedError

    def get_start_path(self, basedir, page):
        """Returns the path to the downloaded page on the disk"""
        raise NotImplementedError


class HttrackDownloader(Downloader):
    """
    Download using httrack. More features than wget, but is has some bugs, like
    downloading CSS stylesheets in ``@import`` constructs.
    """

    name = "httrack"
    opts = [
        "-%l", "fr", # language
        "-Y", # mirror links
        "-C0", # no cache
        "-b0", # no cookies
        "-n", # download "near" files
        "-L0", # DOS-compatible file names
        "-d", # stay on the same domain
        "-x", # replace external links by error page
        "-%u", #url hacks: various hacks to limit duplicate URLs
        "-F", "rss-mirror (allow like Gecko)", # user-agent
    ]

    def __init__(self):
        super(HttrackDownloader, self).__init__()
        recursive = config.getint("DEFAULT", "recursive")
        if recursive:
            self.opts.append("-r%d" % recursive)

    def get_command(self, destdir, url, options=None):
        command = ["httrack"]
        command.extend(self.opts)
        if options:
            command.extend(options)
        command.extend(["-O", destdir, url])
        return command

    def get_start_path(self, basedir, title):
        indexfile = open(os.path.join(basedir, title, "index.html"))
        mo = re.search('<meta HTTP-EQUIV="Refresh" CONTENT="0; URL=(.*)">',
                       indexfile.read())
        indexfile.close()
        return mo.group(1)


class WgetDownloader(Downloader):
    """
    Download using wget. Simple and fast.

    I use the ``-nv`` switch to avoid creating the whole directory structure
    mirroring the website structure, because the FAT32 filesystem does not like
    very very long names.
    """

    name = "wget"
    opts = [
        "-nv", # non verbose
        "-k", # convert links
        "-p", # download needed files for the page
        "-N", # timestamping
        "--restrict-file-names=windows,ascii,lowercase",
        "-E", # adjust extension
        "-H", # allow going on a different domain
        "--timeout=15", # it's 900 by default...
        "--tries=2", # it's 20 by default...
        "-nd", # avoid having 255+ chars paths
        "--no-check-certificate", # SSL
        # User-agent: try to get the mobile version of the page
        "-U", ("Mozilla/5.0 (Linux; U; Android 2.2; en-us; rss-mirror) "
               "AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile "
               "Safari/533.1"),
    ]
    return_codes_ok = [3, 4, 8]
    # 3: I/O error, usually because the filename is too long
    # 4: Network error (broken link on the page)
    # 8: Server issued error response (broken link on the page)

    def __init__(self):
        super(WgetDownloader, self).__init__()
        recursive = config.getint("DEFAULT", "recursive")
        if recursive:
            self.opts.extend(["-r", "-l%d" % recursive])

    def get_command(self, destdir, url, options=None):
        command = ["wget", ]
        command.extend(self.opts)
        if options:
            command.extend(options)
        command.extend(["-P", destdir, url])
        return command

    def get_start_path(self, basedir, title):
        urlfile = open(os.path.join(basedir, title, "url.txt"))
        url = urlfile.read().strip()
        urlfile.close()
        if url.endswith("/"):
            url += "index.html"
        if "-nv" in self.opts:
            local_path = self.get_start_path_nodirs(basedir, url)
        else:
            if os.path.exists(os.path.join(basedir, title, "index.html")):
                # downloaded with httrack
                httrack_dl = HttrackDownloader()
                return httrack_dl.get_start_path(basedir, page)
            local_path = self.get_start_path_dirs(basedir, url)
        if os.path.isfile(os.path.join(basedir, title, local_path)):
            return local_path
        if os.path.isfile(os.path.join(basedir, title, local_path) + ".html"):
            return local_path + ".html"
        else:
            print >>sys.stderr, "Can't find the start page: ", local_path
            print >>sys.stderr, "Tried:", os.path.join(basedir, title, local_path), \
                            os.path.join(basedir, title, local_path) + ".html"
            return "."

    def get_start_path_nodirs(self, basedir, url):
        url_parsed = urlparse(url)
        local_path = os.path.basename(url_parsed[2]).lower()
        if not local_path:
            local_path = "index.html"
        if url_parsed[4]:
            local_path += "@" + url_parsed[4].lower().replace("/", "%2f")
        return local_path

    def get_start_path_dirs(self, basedir, url):
        local_path = url.replace("http://","").lower().replace("?", "@")
        return local_path


def get_engines():
    engines = {}
    for downloader in Downloader.__subclasses__():
        engines[downloader.name] = downloader
    return engines


def extract_options(desc):
    opts = []
    options_matches = re.findall("\{options:\s+[^}]+\}", desc)
    for options_match in options_matches:
        inner_options = options_match[10:-1].strip()
        for inner_opt in inner_options.split():
            inner_opt = inner_opt.strip()
            if not inner_opt:
                continue
            opts.append(inner_opt)
    return opts


class Page(object):
    """
        - ``title`` is a shortned title derived from the page title,
        - ``link`` is the URL,
        - ``title_full`` is the HTML page title,
        - ``timestamp`` is the UNIX timestamp of the page in the RSS feed, which is
          probably the moment you bookmarked it.
    """


    allowed_chars = re.compile("[^a-zA-Z0-9_-]")
    desc_cleanup = re.compile("<[^>]+>")

    @classmethod
    def parse(cls, item):
        titlesize = config.getint("DEFAULT", "title_size",)
        page = cls()
        page.title_full = item.findtext("title").strip()
        page.title = page.title_full[:titlesize].strip().lower()
        page.title = cls.allowed_chars.sub("", page.title.replace(" ","_"))
        page.link = item.findtext("link").strip()
        timestamp = item.findtext("pubDate")
        try:
            timestamp = datetime.datetime.strptime(timestamp,
                            "%a, %d %b %Y %H:%M:%S EDT")
            timestamp = int(time.mktime(timestamp.timetuple()))
        except ValueError:
            timestamp = int(time.time())
        page.timestamp = timestamp
        page.description = item.findtext("description", "")
        page.description = cls.desc_cleanup.sub("", page.description)
        page.options = extract_options(page.description)
        return page

    def download(self, outdir, downloader):
        """Use the downloader to mirror the page"""
        destdir = os.path.join(outdir, self.title)
        if os.path.exists(destdir):
            feedname = os.path.basename(outdir)
            print "Already downloaded:", os.path.join(feedname, self.title)
            return
        print "Downloading", self.title, self.link
        try:
            command = downloader.get_command(destdir, self.link,
                                             options=self.options)
            print " ".join(command)
            if config.getboolean("DEFAULT", "debug"):
                retcode = 0
            else:
                retcode = call(command)
            if retcode < 0:
                print
                print >> sys.stderr, "Child was terminated by signal", -retcode
                return
            if retcode != 0 and retcode not in downloader.return_codes_ok:
                print
                print >> sys.stderr, "Something went wrong while downloading " \
                                    + self.title + "(%s)" % self.link
                print >> sys.stderr, "Return code: %s" % retcode
                return
        except OSError, e:
            print
            print >> sys.stderr, "Execution failed:", e
            return
        except KeyboardInterrupt, e:
            print "Removing downloaded dir in 1 sec..." # to avoid partial downloads
            time.sleep(1)
            shutil.rmtree(destdir)
            return
        # Backup the URL in the url.txt file
        link_file = open(os.path.join(destdir, "url.txt"),"w")
        link_file.write(self.link)
        link_file.close()
        # Backup the HTML title in the title.txt file
        title_file = open(os.path.join(destdir, "title.txt"),"w")
        try:
            title_file.write(unicode(self.title_full).encode("utf-8"))
        except UnicodeEncodeError:
            title_file.write(self.title)
        title_file.close()
        # Backup the timestamp in the timestamp.txt file
        timestamp_file = open(os.path.join(destdir, "timestamp.txt"),"w")
        timestamp_file.write(str(self.timestamp))
        timestamp_file.close()
        print
        time.sleep(1) # Can't remember why this was necessary... FIXME



class Feed(object):

    def __init__(self, name, url):
        self.name = name
        self.url = url
        self.title = None
        self.pages = []

    def parse(self):
        """
        Read an RSS feed and return a list of pages to mirror.
        """
        content = urllib2.urlopen(self.url)
        feed = etree.parse(content)
        self.title = self.get_title(feed)
        pages = []
        for item in feed.findall(".//item"):
            pages.append(Page.parse(item))
        self.pages = pages

    def get_title(self, feed):
        if config.has_option(self.name, "title"):
            return config.get(self.name, "title")
        feed_title = feed.findtext("channel/title")
        if not feed_title:
            return self.name
        return feed_title


class Repository(object):
    """
    A folder containing mirrored pages
    """

    def __init__(self, path, feeds):
        self.path = os.path.expanduser(path)
        self.feeds = feeds

    def make_index(self, downloader):
        """Build the HTML index of the mirrored pages"""
        startfiles = {}
        for feed in self.feeds:
            startfiles[feed] = []
            destdir = os.path.join(self.path, feed.name)
            for page in feed.pages:
                if not os.path.exists(os.path.join(destdir, page.title)):
                    continue
                try:
                    local_path = downloader.get_start_path(destdir, page.title)
                except IOError:
                    print >> sys.stderr, "Can't find the url.txt file for %s" \
                                         % page.title
                    continue # no url.txt file, something went wrong
                startfiles[feed].append(
                        ( unicode(page.title_full).encode("utf-8"),
                          page.description,
                          "/".join([feed.name, page.title, local_path]) ) )
        mainindex = open(os.path.join(self.path, "index.html"), "w")
        mainindex.write("""<!DOCTYPE html>
<html>
<head>
  <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
  <meta name="viewport" content="width=device-width; initial-scale=1.0; maximum-scale=1.0; user-scalable=0;"/>
  <title>Mirrored pages</title>
  <link rel="stylesheet" href="_iui/iui.css" type="text/css" />
  <link rel="stylesheet" href="_iui/t/default/default-theme.css" type="text/css"/>
  <script type="application/x-javascript" src="_iui/iui.js"></script>
</head>
<body>

<div class="toolbar">
  <h1 id="pageTitle"></h1>
    <a id="backButton" class="button" href="#"></a>
</div>
""")
        if len(startfiles) > 1:
            mainindex.write("""<ul id="index" title="Mirrored pages" selected="true">\n""")
            for feed in sorted(startfiles, key=lambda f: f.name):
                mainindex.write("""  <li><a href="#%(name)s">%(title)s</a></li>\n"""
                                % {"name": feed.name, "title": feed.title} )
            mainindex.write("</ul>\n\n")

        for feed in sorted(startfiles, key=lambda f: f.name):
            mainindex.write("""<ul id="%(name)s" title="%(title)s">\n"""
                            % {"name": feed.name, "title": feed.title})
            for title, description, index in startfiles[feed]:
                mainindex.write(
                    """  <li><a href="%s" target="_webapp">%s</a></li>\n""" %
                    (urllib.quote(index.encode("utf-8")), title))
            mainindex.write("</ul>\n\n")
        mainindex.write("""
</body>
</html>
""")


    def cleanup(self):
        """
        Remove mirrored pages which are not in the feed anymore (probably
        because you read them)
        """
        dirs_to_remove = self._get_old_feed_dirs()
        for feed in self.feeds:
            dirs_to_remove.extend(self._get_old_page_dirs(feed))
        for dirname in dirs_to_remove:
            print "Removing", dirname
            if not config.getboolean("DEFAULT", "debug"):
                shutil.rmtree(os.path.join(self.path, dirname))
        self.lowercase_dirs()
        if "_iui" not in os.listdir(self.path):
            self.download_iui()

    def _get_old_feed_dirs(self):
        dirs = []
        for feeddirname in os.listdir(self.path):
            if feeddirname.startswith("."):
                continue
            if feeddirname == "_iui":
                continue
            if not os.path.isdir(os.path.join(self.path, feeddirname)):
                continue # like "index.html" for example
            if feeddirname not in [ f.name for f in self.feeds ]:
                dirs.append(feeddirname)
        return dirs

    def _get_old_page_dirs(self, feed):
        if not os.path.isdir(os.path.join(self.path, feed.name)):
            return []
        dirs = []
        for dirname in os.listdir(os.path.join(self.path, feed.name)):
            if dirname.startswith("."):
                continue
            if dirname not in [ p.title for p in feed.pages ]:
                dirs.append(os.path.join(feed.name, dirname))
        return dirs

    def lowercase_dirs(self):
        """For FAT32 transparency"""
        for feed in self.feeds:
            for root, dirs, files in os.walk(
                        os.path.join(self.path, feed.name), topdown=False):
                for name in dirs:
                    newname = name.lower()
                    if name != newname:
                        source = os.path.join(self.path, feed.name, root, name)
                        dest = os.path.join(self.path, feed.name, root, newname)
                        if os.path.exists(dest):
                            continue
                        os.rename(source, dest)


    def download_iui(self):
        print "Downloading iUI... ",
        sys.stdout.flush()
        iui_url = "http://iui.googlecode.com/files/iui-%s.zip" % IUI_VERSION
        try:
            fn, _headers = urllib.urlretrieve(iui_url)
            with ZipFile(fn, "r") as archive:
                archive.extractall(self.path)
            os.remove(fn)
            os.rename(os.path.join(self.path, "iui-%s/web-app/iui" % IUI_VERSION),
                      os.path.join(self.path, "_iui"))
            shutil.rmtree(os.path.join(self.path, "iui-%s" % IUI_VERSION))
        except (IOError, BadZipfile):
            print "FAILED."
            print >>sys.stderr, "WARNING, could not download or unzip iUI"
            from formatter import DumbWriter
            dw = DumbWriter(sys.stderr) # not really necessary, but fun :)
            dw.send_flowing_data("You must download iUI from %s, unpack it, "
                        "rename the folder to \"_iui\", and put it in the "
                        "destination directory." % iui_url)
            print
            return
        print "done."


    def download_jqm(self):
        print "Downloading jQuery Mobile... ",
        sys.stdout.flush()
        jqm_url = ("http://code.jquery.com/mobile/%(ver)s/jquery.mobile-%(ver)s.zip"
                   % {"ver": JQM_VERSION})
        jq_url = "http://code.jquery.com/jquery-%s.min.js" % JQ_VERSION
        try:
            jq_fn, _headers = urllib.urlretrieve(jqm_url)
            with ZipFile(jq_fn, "r") as jq_zip:
                jq_zip.extractall(self.path)
            os.remove(jq_fn)
            os.rename(os.path.join(self.path, "jquery.mobile-%s" % JQM_VERSION),
                      os.path.join(self.path, "_jqm"))
            urllib.urlretrieve(jq_url, os.path.join(self.path, "_jqm",
                                                    os.path.basename(jq_url)))
        except (IOError, BadZipfile):
            print "FAILED."
            print >>sys.stderr, ("WARNING, could not download or unzip "
                                "jQuery Mobile.")
            from formatter import DumbWriter
            dw = DumbWriter(sys.stderr) # not really necessary, but fun :)
            dw.send_flowing_data("You must download jQuery Mobile from "
                      "%(jqmurl)s, unpack it, rename the folder to \"_jqm\", "
                      "put it in the destination directory, then download "
                      "jQuery from %(jqurl)s, and put it in the same folder."
                      % { "jqmurl": jqm_url, "jqurl": jq_url } )
            print
            return
        print "done."


def parse_opts():
    """Command-line options"""
    usage = "usage: %prog -c <config file>"
    parser = optparse.OptionParser(usage)
    parser.add_option("-c", "--config", help="Configuration file")
    parser.add_option("-o", "--output", dest="output", metavar="DIR",
                      help="Output directory (will be purged !)")
    parser.add_option("--list-engines", dest="lse", action="store_true",
                      help="List available engines and exit")
    parser.add_option("-r", "--recursive", dest="recursive",
                      type="int", metavar="DEPTH", help="Download linked "
                      "pages until this depth. Be careful with that. "
                      "Default: %default)")
    parser.add_option("-d", "--debug", dest="debug", action="store_true",
                      help="Debug mode")
    options, args = parser.parse_args()
    if (options.lse):
        engines = get_engines()
        print "\n".join(engines.keys())
        sys.exit()
    if not options.config:
        if os.path.exists(os.path.expanduser(CONFIG_PATH)):
            options.config = CONFIG_PATH
        else:
            parser.error("You must provide a configuration file (or put it "
                         "in %s)" % CONFIG_PATH)
    if not os.path.exists(os.path.expanduser(options.config)):
        parser.error("Unable to find the configuration file: %s"
                     % options.config)
    if args:
        parser.error("No arguments allowed")
    return options


def get_feeds(config):
    feed_list = []
    for s in config.sections():
        if not config.has_option(s, "url"):
            continue
        feed = Feed(s, config.get(s, "url"))
        feed_list.append(feed)
    return feed_list


def choose_engine(config):
    engines = get_engines()
    name = config.get("DEFAULT", "engine")
    return engines[name]()


def get_config(options):
    # TODO: create config
    config = SafeConfigParser({"title_size": "50", "engine": "wget",
                               "recursive": "0"})
    config.read(os.path.expanduser(options.config))
    if options.output is not None:
        config.set("DEFAULT", "output", options.output)
    if not config.has_option("DEFAULT", "output"):
        print >> sys.stderr, "Config file should have an 'output' variable"
        sys.exit(1)
    if not os.path.isdir(os.path.expanduser(config.get("DEFAULT", "output"))):
        print >> sys.stderr, "The output path must be a directory"
        sys.exit(1)
    config.set("DEFAULT", "debug", str(bool(options.debug)))
    if options.recursive is not None:
        config.set("DEFAULT", "recursive", options.recursive)
    return config


def main():
    """The fun starts here"""
    global config
    options = parse_opts()
    config = get_config(options)
    downloader = choose_engine(config)

    feeds = get_feeds(config)
    for feed in feeds[:]:
        try:
            feed.parse()
        except urllib2.HTTPError, e:
            print >>sys.stderr, "Failed downloading %s: %s" % (feed.url, e)
            feeds.remove(feed)

    repo = Repository(config.get("DEFAULT", "output"), feeds)

    for feed in feeds:
        outdir = os.path.join(repo.path, feed.name)
        for page in feed.pages:
            page.download(outdir, downloader)

    repo.make_index(downloader)
    repo.cleanup()



if __name__ == "__main__":
    main()