| #!/usr/bin/env python |
| # -*- coding: UTF-8 -*- |
| """Planet aggregator library. |
| |
| This package is a library for developing web sites or software that |
| aggregate RSS, CDF and Atom feeds taken from elsewhere into a single, |
| combined feed. |
| """ |
| |
| __version__ = "2.0" |
| __authors__ = [ "Scott James Remnant <scott@netsplit.com>", |
| "Jeff Waugh <jdub@perkypants.org>" ] |
| __license__ = "Python" |
| |
| |
| # Modules available without separate import |
| import cache |
| import feedparser |
| import sanitize |
| import htmltmpl |
| import sgmllib |
| try: |
| import logging |
| except: |
| import compat_logging as logging |
| |
| # Limit the effect of "from planet import *" |
| __all__ = ("cache", "feedparser", "htmltmpl", "logging", |
| "Planet", "Channel", "NewsItem") |
| |
| |
| import os |
| import md5 |
| import time |
| import dbhash |
| import re |
| |
| try: |
| from xml.sax.saxutils import escape |
| except: |
| def escape(data): |
| return data.replace("&","&").replace(">",">").replace("<","<") |
| |
| # Version information (for generator headers) |
| VERSION = ("Planet/%s +http://www.planetplanet.org" % __version__) |
| |
| # Default User-Agent header to send when retreiving feeds |
| USER_AGENT = VERSION + " " + feedparser.USER_AGENT |
| |
| # Default cache directory |
| CACHE_DIRECTORY = "cache" |
| |
| # Default number of items to display from a new feed |
| NEW_FEED_ITEMS = 10 |
| |
| # Useful common date/time formats |
| TIMEFMT_ISO = "%Y-%m-%dT%H:%M:%S+00:00" |
| TIMEFMT_822 = "%a, %d %b %Y %H:%M:%S +0000" |
| |
| |
| # Log instance to use here |
| log = logging.getLogger("planet") |
| try: |
| log.warning |
| except: |
| log.warning = log.warn |
| |
| # Defaults for the template file config sections |
| ENCODING = "utf-8" |
| ITEMS_PER_PAGE = 60 |
| DAYS_PER_PAGE = 0 |
| OUTPUT_DIR = "output" |
| DATE_FORMAT = "%B %d, %Y %I:%M %p" |
| NEW_DATE_FORMAT = "%B %d, %Y" |
| ACTIVITY_THRESHOLD = 0 |
| |
| class stripHtml(sgmllib.SGMLParser): |
| "remove all tags from the data" |
| def __init__(self, data): |
| sgmllib.SGMLParser.__init__(self) |
| self.result='' |
| self.feed(data) |
| self.close() |
| def handle_data(self, data): |
| if data: self.result+=data |
| |
| def template_info(item, date_format): |
| """Produce a dictionary of template information.""" |
| info = {} |
| for key in item.keys(): |
| if item.key_type(key) == item.DATE: |
| date = item.get_as_date(key) |
| info[key] = time.strftime(date_format, date) |
| info[key + "_iso"] = time.strftime(TIMEFMT_ISO, date) |
| info[key + "_822"] = time.strftime(TIMEFMT_822, date) |
| else: |
| info[key] = item[key] |
| if 'title' in item.keys(): |
| info['title_plain'] = stripHtml(info['title']).result |
| |
| return info |
| |
| |
| class Planet: |
| """A set of channels. |
| |
| This class represents a set of channels for which the items will |
| be aggregated together into one combined feed. |
| |
| Properties: |
| user_agent User-Agent header to fetch feeds with. |
| cache_directory Directory to store cached channels in. |
| new_feed_items Number of items to display from a new feed. |
| filter A regular expression that articles must match. |
| exclude A regular expression that articles must not match. |
| """ |
| def __init__(self, config): |
| self.config = config |
| |
| self._channels = [] |
| |
| self.user_agent = USER_AGENT |
| self.cache_directory = CACHE_DIRECTORY |
| self.new_feed_items = NEW_FEED_ITEMS |
| self.filter = None |
| self.exclude = None |
| |
| def tmpl_config_get(self, template, option, default=None, raw=0, vars=None): |
| """Get a template value from the configuration, with a default.""" |
| if self.config.has_option(template, option): |
| return self.config.get(template, option, raw=raw, vars=None) |
| elif self.config.has_option("Planet", option): |
| return self.config.get("Planet", option, raw=raw, vars=None) |
| else: |
| return default |
| |
| def gather_channel_info(self, template_file="Planet"): |
| date_format = self.tmpl_config_get(template_file, |
| "date_format", DATE_FORMAT, raw=1) |
| |
| activity_threshold = int(self.tmpl_config_get(template_file, |
| "activity_threshold", |
| ACTIVITY_THRESHOLD)) |
| |
| if activity_threshold: |
| activity_horizon = \ |
| time.gmtime(time.time()-86400*activity_threshold) |
| else: |
| activity_horizon = 0 |
| |
| channels = {} |
| channels_list = [] |
| for channel in self.channels(hidden=1): |
| channels[channel] = template_info(channel, date_format) |
| channels_list.append(channels[channel]) |
| |
| # identify inactive feeds |
| if activity_horizon: |
| latest = channel.items(sorted=1) |
| if len(latest)==0 or latest[0].date < activity_horizon: |
| channels[channel]["message"] = \ |
| "no activity in %d days" % activity_threshold |
| |
| # report channel level errors |
| if not channel.url_status: continue |
| status = int(channel.url_status) |
| if status == 403: |
| channels[channel]["message"] = "403: forbidden" |
| elif status == 404: |
| channels[channel]["message"] = "404: not found" |
| elif status == 408: |
| channels[channel]["message"] = "408: request timeout" |
| elif status == 410: |
| channels[channel]["message"] = "410: gone" |
| elif status == 500: |
| channels[channel]["message"] = "internal server error" |
| elif status >= 400: |
| channels[channel]["message"] = "http status %s" % status |
| |
| return channels, channels_list |
| |
| def gather_items_info(self, channels, template_file="Planet", channel_list=None): |
| items_list = [] |
| prev_date = [] |
| prev_channel = None |
| |
| date_format = self.tmpl_config_get(template_file, |
| "date_format", DATE_FORMAT, raw=1) |
| items_per_page = int(self.tmpl_config_get(template_file, |
| "items_per_page", ITEMS_PER_PAGE)) |
| days_per_page = int(self.tmpl_config_get(template_file, |
| "days_per_page", DAYS_PER_PAGE)) |
| new_date_format = self.tmpl_config_get(template_file, |
| "new_date_format", NEW_DATE_FORMAT, raw=1) |
| |
| for newsitem in self.items(max_items=items_per_page, |
| max_days=days_per_page, |
| channels=channel_list): |
| item_info = template_info(newsitem, date_format) |
| chan_info = channels[newsitem._channel] |
| for k, v in chan_info.items(): |
| item_info["channel_" + k] = v |
| |
| # Check for the start of a new day |
| if prev_date[:3] != newsitem.date[:3]: |
| prev_date = newsitem.date |
| item_info["new_date"] = time.strftime(new_date_format, |
| newsitem.date) |
| |
| # Check for the start of a new channel |
| if item_info.has_key("new_date") \ |
| or prev_channel != newsitem._channel: |
| prev_channel = newsitem._channel |
| item_info["new_channel"] = newsitem._channel.url |
| |
| items_list.append(item_info) |
| |
| return items_list |
| |
| def run(self, planet_name, planet_link, template_files, offline = False): |
| log = logging.getLogger("planet.runner") |
| |
| # Create a planet |
| log.info("Loading cached data") |
| if self.config.has_option("Planet", "cache_directory"): |
| self.cache_directory = self.config.get("Planet", "cache_directory") |
| if self.config.has_option("Planet", "new_feed_items"): |
| self.new_feed_items = int(self.config.get("Planet", "new_feed_items")) |
| self.user_agent = "%s +%s %s" % (planet_name, planet_link, |
| self.user_agent) |
| if self.config.has_option("Planet", "filter"): |
| self.filter = self.config.get("Planet", "filter") |
| |
| # The other configuration blocks are channels to subscribe to |
| for feed_url in self.config.sections(): |
| if feed_url == "Planet" or feed_url in template_files: |
| continue |
| |
| # Create a channel, configure it and subscribe it |
| channel = Channel(self, feed_url) |
| self.subscribe(channel) |
| |
| # Update it |
| try: |
| if not offline and not channel.url_status == '410': |
| channel.update() |
| except KeyboardInterrupt: |
| raise |
| except: |
| log.exception("Update of <%s> failed", feed_url) |
| |
| def generate_all_files(self, template_files, planet_name, |
| planet_link, planet_feed, owner_name, owner_email): |
| |
| log = logging.getLogger("planet.runner") |
| # Go-go-gadget-template |
| for template_file in template_files: |
| manager = htmltmpl.TemplateManager() |
| log.info("Processing template %s", template_file) |
| try: |
| template = manager.prepare(template_file) |
| except htmltmpl.TemplateError: |
| template = manager.prepare(os.path.basename(template_file)) |
| # Read the configuration |
| output_dir = self.tmpl_config_get(template_file, |
| "output_dir", OUTPUT_DIR) |
| date_format = self.tmpl_config_get(template_file, |
| "date_format", DATE_FORMAT, raw=1) |
| encoding = self.tmpl_config_get(template_file, "encoding", ENCODING) |
| |
| # We treat each template individually |
| base = os.path.splitext(os.path.basename(template_file))[0] |
| url = os.path.join(planet_link, base) |
| output_file = os.path.join(output_dir, base) |
| |
| # Gather information |
| channels, channels_list = self.gather_channel_info(template_file) |
| items_list = self.gather_items_info(channels, template_file) |
| |
| # Gather item information |
| |
| # Process the template |
| tp = htmltmpl.TemplateProcessor(html_escape=0) |
| tp.set("Items", items_list) |
| tp.set("Channels", channels_list) |
| |
| # Generic information |
| tp.set("generator", VERSION) |
| tp.set("name", planet_name) |
| tp.set("link", planet_link) |
| tp.set("owner_name", owner_name) |
| tp.set("owner_email", owner_email) |
| tp.set("url", url) |
| |
| if planet_feed: |
| tp.set("feed", planet_feed) |
| tp.set("feedtype", planet_feed.find('rss')>=0 and 'rss' or 'atom') |
| |
| # Update time |
| date = time.gmtime() |
| tp.set("date", time.strftime(date_format, date)) |
| tp.set("date_iso", time.strftime(TIMEFMT_ISO, date)) |
| tp.set("date_822", time.strftime(TIMEFMT_822, date)) |
| |
| try: |
| log.info("Writing %s", output_file) |
| output_fd = open(output_file, "w") |
| if encoding.lower() in ("utf-8", "utf8"): |
| # UTF-8 output is the default because we use that internally |
| output_fd.write(tp.process(template)) |
| elif encoding.lower() in ("xml", "html", "sgml"): |
| # Magic for Python 2.3 users |
| output = tp.process(template).decode("utf-8") |
| output_fd.write(output.encode("ascii", "xmlcharrefreplace")) |
| else: |
| # Must be a "known" encoding |
| output = tp.process(template).decode("utf-8") |
| output_fd.write(output.encode(encoding, "replace")) |
| output_fd.close() |
| except KeyboardInterrupt: |
| raise |
| except: |
| log.exception("Write of %s failed", output_file) |
| |
| def channels(self, hidden=0, sorted=1): |
| """Return the list of channels.""" |
| channels = [] |
| for channel in self._channels: |
| if hidden or not channel.has_key("hidden"): |
| channels.append((channel.name, channel)) |
| |
| if sorted: |
| channels.sort() |
| |
| return [ c[-1] for c in channels ] |
| |
| def find_by_basename(self, basename): |
| for channel in self._channels: |
| if basename == channel.cache_basename(): return channel |
| |
| def subscribe(self, channel): |
| """Subscribe the planet to the channel.""" |
| self._channels.append(channel) |
| |
| def unsubscribe(self, channel): |
| """Unsubscribe the planet from the channel.""" |
| self._channels.remove(channel) |
| |
| def items(self, hidden=0, sorted=1, max_items=0, max_days=0, channels=None): |
| """Return an optionally filtered list of items in the channel. |
| |
| The filters are applied in the following order: |
| |
| If hidden is true then items in hidden channels and hidden items |
| will be returned. |
| |
| If sorted is true then the item list will be sorted with the newest |
| first. |
| |
| If max_items is non-zero then this number of items, at most, will |
| be returned. |
| |
| If max_days is non-zero then any items older than the newest by |
| this number of days won't be returned. Requires sorted=1 to work. |
| |
| |
| The sharp-eyed will note that this looks a little strange code-wise, |
| it turns out that Python gets *really* slow if we try to sort the |
| actual items themselves. Also we use mktime here, but it's ok |
| because we discard the numbers and just need them to be relatively |
| consistent between each other. |
| """ |
| planet_filter_re = None |
| if self.filter: |
| planet_filter_re = re.compile(self.filter, re.I) |
| planet_exclude_re = None |
| if self.exclude: |
| planet_exclude_re = re.compile(self.exclude, re.I) |
| |
| items = [] |
| seen_guids = {} |
| if not channels: channels=self.channels(hidden=hidden, sorted=0) |
| for channel in channels: |
| for item in channel._items.values(): |
| if hidden or not item.has_key("hidden"): |
| |
| channel_filter_re = None |
| if channel.filter: |
| channel_filter_re = re.compile(channel.filter, |
| re.I) |
| channel_exclude_re = None |
| if channel.exclude: |
| channel_exclude_re = re.compile(channel.exclude, |
| re.I) |
| if (planet_filter_re or planet_exclude_re \ |
| or channel_filter_re or channel_exclude_re): |
| title = "" |
| if item.has_key("title"): |
| title = item.title |
| content = item.get_content("content") |
| |
| if planet_filter_re: |
| if not (planet_filter_re.search(title) \ |
| or planet_filter_re.search(content)): |
| continue |
| |
| if planet_exclude_re: |
| if (planet_exclude_re.search(title) \ |
| or planet_exclude_re.search(content)): |
| continue |
| |
| if channel_filter_re: |
| if not (channel_filter_re.search(title) \ |
| or channel_filter_re.search(content)): |
| continue |
| |
| if channel_exclude_re: |
| if (channel_exclude_re.search(title) \ |
| or channel_exclude_re.search(content)): |
| continue |
| |
| if not seen_guids.has_key(item.id): |
| seen_guids[item.id] = 1; |
| items.append((time.mktime(item.date), item.order, item)) |
| |
| # Sort the list |
| if sorted: |
| items.sort() |
| items.reverse() |
| |
| # Apply max_items filter |
| if len(items) and max_items: |
| items = items[:max_items] |
| |
| # Apply max_days filter |
| if len(items) and max_days: |
| max_count = 0 |
| max_time = items[0][0] - max_days * 84600 |
| for item in items: |
| if item[0] > max_time: |
| max_count += 1 |
| else: |
| items = items[:max_count] |
| break |
| |
| return [ i[-1] for i in items ] |
| |
| class Channel(cache.CachedInfo): |
| """A list of news items. |
| |
| This class represents a list of news items taken from the feed of |
| a website or other source. |
| |
| Properties: |
| url URL of the feed. |
| url_etag E-Tag of the feed URL. |
| url_modified Last modified time of the feed URL. |
| url_status Last HTTP status of the feed URL. |
| hidden Channel should be hidden (True if exists). |
| name Name of the feed owner, or feed title. |
| next_order Next order number to be assigned to NewsItem |
| |
| updated Correct UTC-Normalised update time of the feed. |
| last_updated Correct UTC-Normalised time the feed was last updated. |
| |
| id An identifier the feed claims is unique (*). |
| title One-line title (*). |
| link Link to the original format feed (*). |
| tagline Short description of the feed (*). |
| info Longer description of the feed (*). |
| |
| modified Date the feed claims to have been modified (*). |
| |
| author Name of the author (*). |
| publisher Name of the publisher (*). |
| generator Name of the feed generator (*). |
| category Category name (*). |
| copyright Copyright information for humans to read (*). |
| license Link to the licence for the content (*). |
| docs Link to the specification of the feed format (*). |
| language Primary language (*). |
| errorreportsto E-Mail address to send error reports to (*). |
| |
| image_url URL of an associated image (*). |
| image_link Link to go with the associated image (*). |
| image_title Alternative text of the associated image (*). |
| image_width Width of the associated image (*). |
| image_height Height of the associated image (*). |
| |
| filter A regular expression that articles must match. |
| exclude A regular expression that articles must not match. |
| |
| Properties marked (*) will only be present if the original feed |
| contained them. Note that the optional 'modified' date field is simply |
| a claim made by the item and parsed from the information given, 'updated' |
| (and 'last_updated') are far more reliable sources of information. |
| |
| Some feeds may define additional properties to those above. |
| """ |
| IGNORE_KEYS = ("links", "contributors", "textinput", "cloud", "categories", |
| "url", "href", "url_etag", "url_modified", "tags", "itunes_explicit") |
| |
| def __init__(self, planet, url): |
| if not os.path.isdir(planet.cache_directory): |
| os.makedirs(planet.cache_directory) |
| cache_filename = cache.filename(planet.cache_directory, url) |
| cache_file = dbhash.open(cache_filename, "c", 0666) |
| |
| cache.CachedInfo.__init__(self, cache_file, url, root=1) |
| |
| self._items = {} |
| self._planet = planet |
| self._expired = [] |
| self.url = url |
| # retain the original URL for error reporting |
| self.configured_url = url |
| self.url_etag = None |
| self.url_status = None |
| self.url_modified = None |
| self.name = None |
| self.updated = None |
| self.last_updated = None |
| self.filter = None |
| self.exclude = None |
| self.next_order = "0" |
| self.cache_read() |
| self.cache_read_entries() |
| |
| if planet.config.has_section(url): |
| for option in planet.config.options(url): |
| value = planet.config.get(url, option) |
| self.set_as_string(option, value, cached=0) |
| |
| def has_item(self, id_): |
| """Check whether the item exists in the channel.""" |
| return self._items.has_key(id_) |
| |
| def get_item(self, id_): |
| """Return the item from the channel.""" |
| return self._items[id_] |
| |
| # Special methods |
| __contains__ = has_item |
| |
| def items(self, hidden=0, sorted=0): |
| """Return the item list.""" |
| items = [] |
| for item in self._items.values(): |
| if hidden or not item.has_key("hidden"): |
| items.append((time.mktime(item.date), item.order, item)) |
| |
| if sorted: |
| items.sort() |
| items.reverse() |
| |
| return [ i[-1] for i in items ] |
| |
| def __iter__(self): |
| """Iterate the sorted item list.""" |
| return iter(self.items(sorted=1)) |
| |
| def cache_read_entries(self): |
| """Read entry information from the cache.""" |
| keys = self._cache.keys() |
| for key in keys: |
| if key.find(" ") != -1: continue |
| if self.has_key(key): continue |
| |
| item = NewsItem(self, key) |
| self._items[key] = item |
| |
| def cache_basename(self): |
| return cache.filename('',self._id) |
| |
| def cache_write(self, sync=1): |
| """Write channel and item information to the cache.""" |
| for item in self._items.values(): |
| item.cache_write(sync=0) |
| for item in self._expired: |
| item.cache_clear(sync=0) |
| cache.CachedInfo.cache_write(self, sync) |
| |
| self._expired = [] |
| |
| def feed_information(self): |
| """ |
| Returns a description string for the feed embedded in this channel. |
| |
| This will usually simply be the feed url embedded in <>, but in the |
| case where the current self.url has changed from the original |
| self.configured_url the string will contain both pieces of information. |
| This is so that the URL in question is easier to find in logging |
| output: getting an error about a URL that doesn't appear in your config |
| file is annoying. |
| """ |
| if self.url == self.configured_url: |
| return "<%s>" % self.url |
| else: |
| return "<%s> (formerly <%s>)" % (self.url, self.configured_url) |
| |
| def update(self): |
| """Download the feed to refresh the information. |
| |
| This does the actual work of pulling down the feed and if it changes |
| updates the cached information about the feed and entries within it. |
| """ |
| info = feedparser.parse(self.url, |
| etag=self.url_etag, modified=self.url_modified, |
| agent=self._planet.user_agent) |
| if info.has_key("status"): |
| self.url_status = str(info.status) |
| elif info.has_key("entries") and len(info.entries)>0: |
| self.url_status = str(200) |
| elif info.bozo and info.bozo_exception.__class__.__name__=='Timeout': |
| self.url_status = str(408) |
| else: |
| self.url_status = str(500) |
| |
| if self.url_status == '301' and \ |
| (info.has_key("entries") and len(info.entries)>0): |
| log.warning("Feed has moved from <%s> to <%s>", self.url, info.url) |
| try: |
| os.link(cache.filename(self._planet.cache_directory, self.url), |
| cache.filename(self._planet.cache_directory, info.url)) |
| except: |
| pass |
| self.url = info.url |
| elif self.url_status == '304': |
| log.info("Feed %s unchanged", self.feed_information()) |
| return |
| elif self.url_status == '410': |
| log.info("Feed %s gone", self.feed_information()) |
| self.cache_write() |
| return |
| elif self.url_status == '408': |
| log.warning("Feed %s timed out", self.feed_information()) |
| return |
| elif int(self.url_status) >= 400: |
| log.error("Error %s while updating feed %s", |
| self.url_status, self.feed_information()) |
| return |
| else: |
| log.info("Updating feed %s", self.feed_information()) |
| |
| self.url_etag = info.has_key("etag") and info.etag or None |
| self.url_modified = info.has_key("modified") and info.modified or None |
| if self.url_etag is not None: |
| log.debug("E-Tag: %s", self.url_etag) |
| if self.url_modified is not None: |
| log.debug("Last Modified: %s", |
| time.strftime(TIMEFMT_ISO, self.url_modified)) |
| |
| self.update_info(info.feed) |
| self.update_entries(info.entries) |
| self.cache_write() |
| |
| def update_info(self, feed): |
| """Update information from the feed. |
| |
| This reads the feed information supplied by feedparser and updates |
| the cached information about the feed. These are the various |
| potentially interesting properties that you might care about. |
| """ |
| for key in feed.keys(): |
| if key in self.IGNORE_KEYS or key + "_parsed" in self.IGNORE_KEYS: |
| # Ignored fields |
| pass |
| elif feed.has_key(key + "_parsed"): |
| # Ignore unparsed date fields |
| pass |
| elif key.endswith("_detail"): |
| # retain name and email sub-fields |
| if feed[key].has_key('name') and feed[key].name: |
| self.set_as_string(key.replace("_detail","_name"), \ |
| feed[key].name) |
| if feed[key].has_key('email') and feed[key].email: |
| self.set_as_string(key.replace("_detail","_email"), \ |
| feed[key].email) |
| elif key == "items": |
| # Ignore items field |
| pass |
| elif key.endswith("_parsed"): |
| # Date fields |
| if feed[key] is not None: |
| self.set_as_date(key[:-len("_parsed")], feed[key]) |
| elif key == "image": |
| # Image field: save all the information |
| if feed[key].has_key("url"): |
| self.set_as_string(key + "_url", feed[key].url) |
| if feed[key].has_key("link"): |
| self.set_as_string(key + "_link", feed[key].link) |
| if feed[key].has_key("title"): |
| self.set_as_string(key + "_title", feed[key].title) |
| if feed[key].has_key("width"): |
| self.set_as_string(key + "_width", str(feed[key].width)) |
| if feed[key].has_key("height"): |
| self.set_as_string(key + "_height", str(feed[key].height)) |
| elif isinstance(feed[key], (str, unicode)): |
| # String fields |
| try: |
| detail = key + '_detail' |
| if feed.has_key(detail) and feed[detail].has_key('type'): |
| if feed[detail].type == 'text/html': |
| feed[key] = sanitize.HTML(feed[key]) |
| elif feed[detail].type == 'text/plain': |
| feed[key] = escape(feed[key]) |
| self.set_as_string(key, feed[key]) |
| except KeyboardInterrupt: |
| raise |
| except: |
| log.exception("Ignored '%s' of <%s>, unknown format", |
| key, self.url) |
| |
| def update_entries(self, entries): |
| """Update entries from the feed. |
| |
| This reads the entries supplied by feedparser and updates the |
| cached information about them. It's at this point we update |
| the 'updated' timestamp and keep the old one in 'last_updated', |
| these provide boundaries for acceptable entry times. |
| |
| If this is the first time a feed has been updated then most of the |
| items will be marked as hidden, according to Planet.new_feed_items. |
| |
| If the feed does not contain items which, according to the sort order, |
| should be there; those items are assumed to have been expired from |
| the feed or replaced and are removed from the cache. |
| """ |
| if not len(entries): |
| return |
| |
| self.last_updated = self.updated |
| self.updated = time.gmtime() |
| |
| new_items = [] |
| feed_items = [] |
| for entry in entries: |
| # Try really hard to find some kind of unique identifier |
| if entry.has_key("id"): |
| entry_id = cache.utf8(entry.id) |
| elif entry.has_key("link"): |
| entry_id = cache.utf8(entry.link) |
| elif entry.has_key("title"): |
| entry_id = (self.url + "/" |
| + md5.new(cache.utf8(entry.title)).hexdigest()) |
| elif entry.has_key("summary"): |
| entry_id = (self.url + "/" |
| + md5.new(cache.utf8(entry.summary)).hexdigest()) |
| else: |
| log.error("Unable to find or generate id, entry ignored") |
| continue |
| |
| # Create the item if necessary and update |
| if self.has_item(entry_id): |
| item = self._items[entry_id] |
| else: |
| item = NewsItem(self, entry_id) |
| self._items[entry_id] = item |
| new_items.append(item) |
| item.update(entry) |
| feed_items.append(entry_id) |
| |
| # Hide excess items the first time through |
| if self.last_updated is None and self._planet.new_feed_items \ |
| and len(feed_items) > self._planet.new_feed_items: |
| item.hidden = "yes" |
| log.debug("Marked <%s> as hidden (new feed)", entry_id) |
| |
| # Assign order numbers in reverse |
| new_items.reverse() |
| for item in new_items: |
| item.order = self.next_order = str(int(self.next_order) + 1) |
| |
| # Check for expired or replaced items |
| feed_count = len(feed_items) |
| log.debug("Items in Feed: %d", feed_count) |
| for item in self.items(sorted=1): |
| if feed_count < 1: |
| break |
| elif item.id in feed_items: |
| feed_count -= 1 |
| elif item._channel.url_status != '226': |
| del(self._items[item.id]) |
| self._expired.append(item) |
| log.debug("Removed expired or replaced item <%s>", item.id) |
| |
| def get_name(self, key): |
| """Return the key containing the name.""" |
| for key in ("name", "title"): |
| if self.has_key(key) and self.key_type(key) != self.NULL: |
| return self.get_as_string(key) |
| |
| return "" |
| |
| class NewsItem(cache.CachedInfo): |
| """An item of news. |
| |
| This class represents a single item of news on a channel. They're |
| created by members of the Channel class and accessible through it. |
| |
| Properties: |
| id Channel-unique identifier for this item. |
| id_hash Relatively short, printable cryptographic hash of id |
| date Corrected UTC-Normalised update time, for sorting. |
| order Order in which items on the same date can be sorted. |
| hidden Item should be hidden (True if exists). |
| |
| title One-line title (*). |
| link Link to the original format text (*). |
| summary Short first-page summary (*). |
| content Full HTML content. |
| |
| modified Date the item claims to have been modified (*). |
| issued Date the item claims to have been issued (*). |
| created Date the item claims to have been created (*). |
| expired Date the item claims to expire (*). |
| |
| author Name of the author (*). |
| publisher Name of the publisher (*). |
| category Category name (*). |
| comments Link to a page to enter comments (*). |
| license Link to the licence for the content (*). |
| source_name Name of the original source of this item (*). |
| source_link Link to the original source of this item (*). |
| |
| Properties marked (*) will only be present if the original feed |
| contained them. Note that the various optional date fields are |
| simply claims made by the item and parsed from the information |
| given, 'date' is a far more reliable source of information. |
| |
| Some feeds may define additional properties to those above. |
| """ |
| IGNORE_KEYS = ("categories", "contributors", "enclosures", "links", |
| "guidislink", "date", "tags") |
| |
| def __init__(self, channel, id_): |
| cache.CachedInfo.__init__(self, channel._cache, id_) |
| |
| self._channel = channel |
| self.id = id_ |
| self.id_hash = md5.new(id_).hexdigest() |
| self.date = None |
| self.order = None |
| self.content = None |
| self.cache_read() |
| |
| def update(self, entry): |
| """Update the item from the feedparser entry given.""" |
| for key in entry.keys(): |
| if key in self.IGNORE_KEYS or key + "_parsed" in self.IGNORE_KEYS: |
| # Ignored fields |
| pass |
| elif entry.has_key(key + "_parsed"): |
| # Ignore unparsed date fields |
| pass |
| elif key.endswith("_detail"): |
| # retain name, email, and language sub-fields |
| if entry[key].has_key('name') and entry[key].name: |
| self.set_as_string(key.replace("_detail","_name"), \ |
| entry[key].name) |
| if entry[key].has_key('email') and entry[key].email: |
| self.set_as_string(key.replace("_detail","_email"), \ |
| entry[key].email) |
| if entry[key].has_key('language') and entry[key].language and \ |
| (not self._channel.has_key('language') or \ |
| entry[key].language != self._channel.language): |
| self.set_as_string(key.replace("_detail","_language"), \ |
| entry[key].language) |
| elif key.endswith("_parsed"): |
| # Date fields |
| if entry[key] is not None: |
| self.set_as_date(key[:-len("_parsed")], entry[key]) |
| elif key == "source": |
| # Source field: save both url and value |
| if entry[key].has_key("value"): |
| self.set_as_string(key + "_name", entry[key].value) |
| if entry[key].has_key("url"): |
| self.set_as_string(key + "_link", entry[key].url) |
| elif key == "content": |
| # Content field: concatenate the values |
| value = "" |
| for item in entry[key]: |
| if item.type == 'text/html': |
| item.value = sanitize.HTML(item.value) |
| elif item.type == 'text/plain': |
| item.value = escape(item.value) |
| if item.has_key('language') and item.language and \ |
| (not self._channel.has_key('language') or |
| item.language != self._channel.language) : |
| self.set_as_string(key + "_language", item.language) |
| value += cache.utf8(item.value) |
| self.set_as_string(key, value) |
| elif isinstance(entry[key], (str, unicode)): |
| # String fields |
| try: |
| detail = key + '_detail' |
| if entry.has_key(detail): |
| if entry[detail].has_key('type'): |
| if entry[detail].type == 'text/html': |
| entry[key] = sanitize.HTML(entry[key]) |
| elif entry[detail].type == 'text/plain': |
| entry[key] = escape(entry[key]) |
| self.set_as_string(key, entry[key]) |
| except KeyboardInterrupt: |
| raise |
| except: |
| log.exception("Ignored '%s' of <%s>, unknown format", |
| key, self.id) |
| |
| # Generate the date field if we need to |
| self.get_date("date") |
| |
| def get_date(self, key): |
| """Get (or update) the date key. |
| |
| We check whether the date the entry claims to have been changed is |
| since we last updated this feed and when we pulled the feed off the |
| site. |
| |
| If it is then it's probably not bogus, and we'll sort accordingly. |
| |
| If it isn't then we bound it appropriately, this ensures that |
| entries appear in posting sequence but don't overlap entries |
| added in previous updates and don't creep into the next one. |
| """ |
| |
| for other_key in ("updated", "modified", "published", "issued", "created"): |
| if self.has_key(other_key): |
| date = self.get_as_date(other_key) |
| break |
| else: |
| date = None |
| |
| if date is not None: |
| if date > self._channel.updated: |
| date = self._channel.updated |
| # elif date < self._channel.last_updated: |
| # date = self._channel.updated |
| elif self.has_key(key) and self.key_type(key) != self.NULL: |
| return self.get_as_date(key) |
| else: |
| date = self._channel.updated |
| |
| self.set_as_date(key, date) |
| return date |
| |
| def get_content(self, key): |
| """Return the key containing the content.""" |
| for key in ("content", "tagline", "summary"): |
| if self.has_key(key) and self.key_type(key) != self.NULL: |
| return self.get_as_string(key) |
| |
| return "" |
