Commit c20acf99 authored by Sam Ruby's avatar Sam Ruby

Hash content to determine if it was modified

parent 70f97175
......@@ -100,7 +100,16 @@ use for logging output. Note: this configuration value is processed
<dd>Number of items to take from new feeds</dd>
<dd>The number of threads to use when spidering. When set to 0, the default,
no threads are used and spidering follows the traditional algorithm.</dd>
no threads are used and spidering follows the traditional algorithm.</dd>
<dd>The number of threads to use when spidering. When set to 0, the default,
no threads are used and spidering follows the traditional algorithm.</dd>
<dd>If <code>spider_threads</code> is specified, you can also specify a
directory to be used for an additional HTTP cache to front end the Venus
cache. If specified as a relative path, it is evaluated relative to the
......@@ -285,13 +285,13 @@ def downloadReadingList(list, orig_config, callback, use_cache=True, re_read=Tru
def http_cache_directory():
if parser.has_option('Planet', 'http_cache_directory'):
parser.get('Planet', 'http_cache_directory')
return os.path.join(cache_directory(), 'sources/http')
parser.get('Planet', 'http_cache_directory'))
def cache_sources_directory():
if parser.has_option('Planet', 'cache_sources_directory'):
parser.get('Planet', 'cache_sources_directory')
return os.path.join(cache_directory(),
parser.get('Planet', 'cache_sources_directory'))
return os.path.join(cache_directory(), 'sources')
......@@ -105,7 +105,7 @@ def links(xentry, entry):
if entry.has_key('link'):
entry['links'].append({'rel':'alternate', 'href'})
xdoc = xentry.ownerDocument
for link in entry.links:
for link in entry['links']:
if not 'href' in link.keys(): continue
xlink = xdoc.createElement('link')
xlink.setAttribute('href', link.get('href'))
......@@ -121,36 +121,6 @@ def _is_http_uri(uri):
parsed = urlparse.urlparse(uri)
return parsed[0] in ['http', 'https']
def spiderFeed(feed_uri, only_if_new=0):
""" Spider (fetch) a single feed """
log = planet.logger
# read cached feed info
sources = config.cache_sources_directory()
if not os.path.exists(sources):
os.makedirs(sources, 0700)
feed_source = filename(sources, feed_uri)
feed_info = feedparser.parse(feed_source)
if feed_info.feed and only_if_new:"Feed %s already in cache", feed_uri)
if feed_info.feed.get('planet_http_status',None) == '410':"Feed %s gone", feed_uri)
# read feed itself
modified = None
feed_info.feed.get('planet_http_last_modified', None))
data = feedparser.parse(feed_info.feed.get('planet_http_location',feed_uri),
etag=feed_info.feed.get('planet_http_etag',None), modified=modified)
writeCache(feed_uri, feed_info, data)
def writeCache(feed_uri, feed_info, data):
log = planet.logger
sources = config.cache_sources_directory()
......@@ -159,7 +129,8 @@ def writeCache(feed_uri, feed_info, data):
if not data.has_key("status"):
if data.has_key("entries") and len(data.entries)>0:
data.status = 200
elif data.bozo and data.bozo_exception.__class__.__name__.lower()=='timeout':
elif data.bozo and \
data.status = 408
data.status = 500
......@@ -210,11 +181,16 @@ def writeCache(feed_uri, feed_info, data):
if data.has_key('headers'):
if data.has_key('etag') and data.etag:
data.feed['planet_http_etag'] = data.etag
log.debug("E-Tag: %s", data.etag)
if data.has_key('modified') and data.modified:
elif data.headers.has_key('etag') and data.headers['etag']:
data.feed['planet_http_etag'] = data.headers['etag']
if data.headers.has_key('last-modified'):
elif data.has_key('modified') and data.modified:
data.feed['planet_http_last_modified'] = time.asctime(data.modified)
log.debug("Last Modified: %s",
if data.headers.has_key('-content-hash'):
data.feed['planet_content_hash'] = data.headers['-content-hash']
# capture feed and data from the planet configuration file
if data.version:
......@@ -337,13 +313,11 @@ def writeCache(feed_uri, feed_info, data):
def httpThread(thread_index, input_queue, output_queue, log):
from Queue import Empty
import httplib2
import httplib2, md5
from socket import gaierror, error
from httplib import BadStatusLine
http_cache = config.http_cache_directory()
h = httplib2.Http(http_cache)
h = httplib2.Http(config.http_cache_directory())
uri, feed_info = input_queue.get(block=True)
while uri:"Fetching %s via %d", uri, thread_index)
......@@ -363,10 +337,26 @@ def httpThread(thread_index, input_queue, output_queue, log):"unable to map %s to a URI", uri)
idna = uri
# cache control headers
headers = {}
if feed_info.feed.has_key('planet_http_etag'):
headers['If-None-Match'] = feed_info.feed['planet_http_etag']
if feed_info.feed.has_key('planet_http_last_modified'):
headers['If-Modified-Since'] = \
# issue request
(resp, content) = h.request(idna)
if resp.status == 200 and resp.fromcache:
resp.status = 304
(resp, content) = h.request(idna, 'GET', headers=headers)
# unchanged detection
resp['-content-hash'] = or '').hexdigest()
if resp.status == 200:
if resp.fromcache:
resp.status = 304
elif feed_info.feed.has_key('planet_content_hash') and \
feed_info.feed['planet_content_hash'] == \
resp.status = 304
# build a file-like object
feed = StringIO(content)
......@@ -385,8 +375,7 @@ def httpThread(thread_index, input_queue, output_queue, log):
feed.headers['status'] = '408'
log.warn("Timeout in thread-%d", thread_index)
log.error("HTTP Error: %s in thread-%d",
str(e), thread_index)
log.error("HTTP Error: %s in thread-%d", str(e), thread_index)
except Exception, e:
import sys, traceback
type, value, tb = sys.exc_info()
......@@ -428,7 +417,7 @@ def spiderPlanet(only_if_new = False):
threads = {}
if int(config.spider_threads()):
http_cache = config.http_cache_directory()
if not os.path.exists(http_cache):
if http_cache and not os.path.exists(http_cache):
os.makedirs(http_cache, 0700)
# Start all the worker threads
......@@ -484,9 +473,9 @@ def spiderPlanet(only_if_new = False):
data = feedparser.parse(feed, **options)
data = feedparser.FeedParserDict({'version':None,
'headers':feed.headers, 'entries': [],
'status': int(feed.headers.status)})
data = feedparser.FeedParserDict({'version': None,
'headers': feed.headers, 'entries': [], 'feed': {},
'bozo': 0, 'status': int(feed.headers.status)})
writeCache(uri, feed_info, data)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment