Commit b2ccc8c1 authored by Joe Gregorio's avatar Joe Gregorio

added 304 checking before calling spiderFeed()

parent 217e850e
......@@ -372,8 +372,9 @@ def spiderPlanet(only_if_new = False):
item = awaiting_parsing.get(False)
try:
(resp_headers, content, uri) = item
log.info("Parsing pre-fetched %s", uri)
spiderFeed(uri, only_if_new=only_if_new, content=content, resp_headers=resp_headers)
if not resp_headers.fromcache:
log.info("Parsing pre-fetched %s", uri)
spiderFeed(uri, only_if_new=only_if_new, content=content, resp_headers=resp_headers)
except Exception, e:
import sys, traceback
type, value, tb = sys.exc_info()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment