Commit 7cbc2e31 authored by Sam Ruby's avatar Sam Ruby

Theme support

parent 9fa9fb61
......@@ -2,6 +2,9 @@ xmlns = 'http://planet.intertwingly.net/'
logger = None
import config
config.__init__()
def getLogger(level):
""" get a logger with the specified log level """
global logger
......
......@@ -11,7 +11,7 @@ Usage:
config.load('config.ini')
# administrative / structural information
print config.templates()
print config.template_files()
print config.feeds()
# planet wide configuration
......@@ -37,6 +37,7 @@ def __init__():
"""define the struture of an ini file"""
import config
# underlying implementation
def get(section, option, default):
if section and parser.has_option(section, option):
return parser.get(section, option)
......@@ -49,6 +50,10 @@ def __init__():
setattr(config, name, lambda default=default: get(None,name,default))
planet_predefined_options.append(name)
def define_planet_list(name):
setattr(config, name, lambda : get(None,name,'').split())
planet_predefined_options.append(name)
def define_tmpl(name, default):
setattr(config, name, lambda section, default=default:
get(section,name,default))
......@@ -63,25 +68,57 @@ def __init__():
define_planet('cache_directory', "cache")
define_planet('log_level', "WARNING")
define_planet('feed_timeout', 20)
define_planet('date_format', "%B %d, %Y %I:%M %p")
define_planet('generator', 'Venus')
define_planet('generator_uri', 'http://intertwingly.net/code/venus/')
define_planet('owner_name', 'Anonymous Coward')
define_planet('owner_email', '')
define_planet('output_theme', '')
define_planet('output_dir', 'output')
define_planet_list('template_files')
define_planet_list('bill_of_materials')
define_planet_list('template_directories')
# template options
define_tmpl_int('days_per_page', 0)
define_tmpl_int('items_per_page', 60)
define_tmpl('encoding', 'utf-8')
# prevent re-initialization
setattr(config, '__init__', lambda: None)
def load(file):
def load(config_file):
""" initialize and load a configuration"""
__init__()
global parser
parser = ConfigParser()
parser.read(file)
def template_files():
""" list the templates defined """
return parser.get('Planet','template_files').split(' ')
parser.read(config_file)
if parser.has_option('Planet', 'output_theme'):
theme = parser.get('Planet', 'output_theme')
for path in ("", os.path.join(sys.path[0],'themes')):
theme_dir = os.path.join(path,theme)
theme_file = os.path.join(theme_dir,'config.ini')
if os.path.exists(theme_file):
# initial search list for theme directories
dirs = [theme_dir]
if parser.has_option('Planet', 'template_directories'):
dirs.insert(0,parser.get('Planet', 'template_directories'))
# read in the theme
parser = ConfigParser()
parser.read(theme_file)
# complete search list for theme directories
if parser.has_option('Planet', 'template_directories'):
dirs += [os.path.join(theme_dir,dir) for dir in
parser.get('Planet', 'template_directories').split()]
# merge configurations, allowing current one to override theme
parser.read(config_file)
parser.set('Planet', 'template_directories', ' '.join(dirs))
break
else:
import config, planet
log = planet.getLogger(config.log_level())
log.error('Unable to find theme %s', theme)
def cache_sources_directory():
if parser.has_option('Planet', 'cache_sources_directory'):
......
......@@ -18,7 +18,7 @@ from xml.sax.saxutils import escape
from xml.dom import minidom
from BeautifulSoup import BeautifulSoup
from xml.parsers.expat import ExpatError
import planet
import planet, config
illegal_xml_chars = re.compile("[\x01-\x08\x0B\x0C\x0E-\x1F]")
......@@ -29,6 +29,7 @@ def createTextElement(parent, name, value):
xelement = xdoc.createElement(name)
xelement.appendChild(xdoc.createTextNode(value))
parent.appendChild(xelement)
return xelement
def invalidate(c):
""" replace invalid characters """
......@@ -98,7 +99,9 @@ def date(xentry, name, parsed):
""" insert a date-formated element into the entry """
if not parsed: return
formatted = time.strftime("%Y-%m-%dT%H:%M:%SZ", parsed)
createTextElement(xentry, name, formatted)
xdate = createTextElement(xentry, name, formatted)
formatted = time.strftime(config.date_format(), parsed)
xdate.setAttribute('planet:format', formatted)
def author(xentry, name, detail):
""" insert an author-like element into the entry """
......
""" Splice together a planet from a cache of feed entries """
import glob, os
import glob, os, time, shutil
from xml.dom import minidom
import planet, config, feedparser, reconstitute
from reconstitute import createTextElement
from reconstitute import createTextElement, date
from spider import filename
def splice(configFile):
......@@ -11,6 +11,7 @@ def splice(configFile):
config.load(configFile)
log = planet.getLogger(config.log_level())
log.info("Loading cached data")
cache = config.cache_directory()
dir=[(os.stat(file).st_mtime,file) for file in glob.glob(cache+"/*")
if not os.path.isdir(file)]
......@@ -18,17 +19,20 @@ def splice(configFile):
dir.reverse()
items=max([config.items_per_page(templ)
for templ in config.template_files()])
for templ in config.template_files() or ['Planet']])
doc = minidom.parseString('<feed xmlns="http://www.w3.org/2005/Atom"/>')
feed = doc.documentElement
# insert Google/LiveJournal's noindex
feed.setAttribute('indexing:index','no')
feed.setAttribute('xmlns:indexing','urn:atom-extension:indexing')
# insert feed information
createTextElement(feed, 'title', config.name())
date(feed, 'updated', time.gmtime())
gen = createTextElement(feed, 'generator', config.generator())
gen.setAttribute('uri', config.generator_uri())
author = doc.createElement('author')
createTextElement(author, 'name', config.owner_name())
createTextElement(author, 'email', config.owner_email())
feed.appendChild(author)
# insert entry information
for mtime,file in dir[:items]:
......@@ -47,3 +51,75 @@ def splice(configFile):
feed.appendChild(xdoc.documentElement)
return doc
def apply(doc):
output_dir = config.output_dir()
if not os.path.exists(output_dir): os.makedirs(output_dir)
log = planet.getLogger(config.log_level())
try:
# if available, use the python interface to libxslt
import libxml2
import libxslt
dom = libxml2.parseDoc(doc)
docfile = None
except:
# otherwise, use the command line interface
dom = None
import warnings
warnings.simplefilter('ignore', RuntimeWarning)
docfile = os.tmpnam()
file = open(docfile,'w')
file.write(doc)
file.close()
# Go-go-gadget-template
for template_file in config.template_files():
for template_dir in config.template_directories():
template_resolved = os.path.join(template_dir, template_file)
if os.path.exists(template_resolved): break
else:
log.error("Unable to locate template %s", template_file)
continue
base,ext = os.path.splitext(os.path.basename(template_resolved))
if ext != '.xslt':
log.warning("Skipping template %s", template_resolved)
continue
log.info("Processing template %s", template_resolved)
output_file = os.path.join(output_dir, base)
if dom:
styledoc = libxml2.parseFile(template_resolved)
style = libxslt.parseStylesheetDoc(styledoc)
result = style.applyStylesheet(dom, None)
log.info("Writing %s", output_file)
style.saveResultToFilename(output_file, result, 0)
style.freeStylesheet()
result.freeDoc()
else:
log.info("Writing %s", output_file)
os.system('xsltproc %s %s > %s' %
(template_resolved, docfile, output_file))
if dom: dom.freeDoc()
if docfile: os.unlink(docfile)
# Process bill of materials
for copy_file in config.bill_of_materials():
dest = os.path.join(output_dir, copy_file)
for template_dir in config.template_directories():
source = os.path.join(template_dir, copy_file)
if os.path.exists(source): break
else:
log.error('Unable to locate %s', copy_file)
continue
mtime = os.stat(source).st_mtime
if not os.path.exists(dest) or os.stat(dest).st_mtime < mtime:
dest_dir = os.path.split(dest)[0]
if not os.path.exists(dest_dir): os.makedirs(dest_dir)
log.info("Copying %s to %s", source, dest)
shutil.copyfile(source, dest)
shutil.copystat(source, dest)
#!/usr/bin/env python
import glob, trace, unittest
import glob, trace, unittest, os, sys
# start in a consistent, predictable location
os.chdir(sys.path[0])
# find all of the planet test modules
modules = map(trace.fullmodname, glob.glob('tests/test_*.py'))
modules = map(trace.fullmodname, glob.glob(os.path.join('tests', 'test_*.py')))
# load all of the tests into a suite
suite = unittest.TestLoader().loadTestsFromNames(modules)
......
......@@ -13,10 +13,7 @@ if __name__ == '__main__':
# at the moment, we don't have template support, so we cheat and
# simply insert a XSLT processing instruction
doc = splice.splice(sys.argv[1])
pi = doc.createProcessingInstruction(
'xml-stylesheet','type="text/xsl" href="planet.xslt"')
doc.insertBefore(pi, doc.firstChild)
print doc.toxml('utf-8')
splice.apply(doc.toxml('utf-8'))
else:
print "Usage:"
print " python %s config.ini" % sys.argv[0]
#!/usr/bin/env python
"""
While unit tests are intended to be independently executable, it often
is helpful to ensure that some downstream tasks can be run with the
exact output produced by upstream tasks.
This script captures such output. It should be run whenever there is
a major change in the contract between stages
"""
import shutil, os, sys
# move up a directory
sys.path.insert(1, os.path.split(sys.path[0])[0])
os.chdir(sys.path[1])
# copy spider output to splice input
from planet import spider
spider.spiderPlanet('tests/data/spider/config.ini')
if os.path.exists('tests/data/splice/cache'):
shutil.rmtree('tests/data/splice/cache')
shutil.move('tests/work/spider/cache', 'tests/data/splice/cache')
source=open('tests/data/spider/config.ini')
dest1=open('tests/data/splice/config.ini', 'w')
dest1.write(source.read().replace('/work/spider/', '/data/splice/'))
dest1.close()
source.seek(0)
dest2=open('tests/data/apply/config.ini', 'w')
dest2.write(source.read().replace('[Planet]', '''[Planet]
output_theme = asf
output_dir = tests/work/apply'''))
dest2.close()
source.close()
# copy splice output to apply input
from planet import splice
file=open('tests/data/apply/feed.xml', 'w')
file.write(splice.splice('tests/data/splice/config.ini').toxml('utf-8'))
file.close()
[Planet]
output_theme = asf
output_dir = tests/work/apply
name = test planet
cache_directory = tests/work/spider/cache
[tests/data/spider/testfeed0.atom]
name = not found
[tests/data/spider/testfeed1b.atom]
name = one
[tests/data/spider/testfeed2.atom]
name = two
[tests/data/spider/testfeed3.rss]
name = three
This diff is collapsed.
[Planet]
name = Test Configuration
output_theme = asf
items_per_page = 50
template_directories = /foo /bar
[index.html.xslt]
days_per_page = 7
[feed1]
name = one
[feed2]
name = two
[Planet]
name = test planet
cache_directory = tests/work/spider/cache
template_files =
[tests/data/spider/testfeed0.atom]
name = not found
......
......@@ -4,7 +4,7 @@
<link href='http://example.com/3' type='text/html' rel='alternate'/>
<title>Earth</title>
<summary>the Blue Planet</summary>
<updated>2006-01-03T00:00:00Z</updated>
<updated planet:format='January 03, 2006 12:00 AM'>2006-01-03T00:00:00Z</updated>
<source>
<link href='http://intertwingly.net/code/venus/tests/data/spider/testfeed3.rss' type='text/html' rel='alternate'/>
<link href='tests/data/spider/testfeed3.rss' type='application/atom+xml' rel='self'/>
......
......@@ -4,7 +4,7 @@
<link href='http://example.com/4' type='text/html' rel='alternate'/>
<title>Mars</title>
<summary>the Red Planet</summary>
<updated>2006-08-18T18:30:50Z</updated>
<updated planet:format='August 21, 2006 12:54 PM'>2006-08-21T12:54:31Z</updated>
<source>
<link href='http://intertwingly.net/code/venus/tests/data/spider/testfeed3.rss' type='text/html' rel='alternate'/>
<link href='tests/data/spider/testfeed3.rss' type='application/atom+xml' rel='self'/>
......
......@@ -4,7 +4,7 @@
<link href='http://example.com/1' type='text/html' rel='alternate'/>
<title>Mercury</title>
<content>Messenger of the Roman Gods</content>
<updated>2006-01-01T00:00:00Z</updated>
<updated planet:format='January 01, 2006 12:00 AM'>2006-01-01T00:00:00Z</updated>
<source>
<id>tag:planet.intertwingly.net,2006:testfeed1</id>
<author>
......@@ -16,7 +16,7 @@
<link href='http://www.intertwingly.net/blog/' type='text/html' rel='alternate'/>
<subtitle>It’s just data</subtitle>
<title>Sam Ruby</title>
<updated>2006-06-17T00:15:18Z</updated>
<updated planet:format='June 17, 2006 12:15 AM'>2006-06-17T00:15:18Z</updated>
<planet:name>one</planet:name>
</source>
</entry>
......@@ -4,8 +4,8 @@
<link href='http://example.com/2' type='text/html' rel='alternate'/>
<title>Venus</title>
<content>the Jewel of the Sky</content>
<updated>2006-02-02T00:00:00Z</updated>
<published>2006-01-02T00:00:00Z</published>
<updated planet:format='February 02, 2006 12:00 AM'>2006-02-02T00:00:00Z</updated>
<published planet:format='January 02, 2006 12:00 AM'>2006-01-02T00:00:00Z</published>
<source>
<id>tag:planet.intertwingly.net,2006:testfeed1</id>
<author>
......@@ -17,7 +17,7 @@
<link href='http://www.intertwingly.net/blog/' type='text/html' rel='alternate'/>
<subtitle>It’s just data</subtitle>
<title>Sam Ruby</title>
<updated>2006-06-17T00:15:18Z</updated>
<updated planet:format='June 17, 2006 12:15 AM'>2006-06-17T00:15:18Z</updated>
<planet:name>one</planet:name>
</source>
</entry>
......@@ -4,7 +4,7 @@
<link href='http://example.com/3' type='text/html' rel='alternate'/>
<title>Earth</title>
<content>the Blue Planet</content>
<updated>2006-01-03T00:00:00Z</updated>
<updated planet:format='January 03, 2006 12:00 AM'>2006-01-03T00:00:00Z</updated>
<source>
<id>tag:planet.intertwingly.net,2006:testfeed1</id>
<author>
......@@ -16,7 +16,7 @@
<link href='http://www.intertwingly.net/blog/' type='text/html' rel='alternate'/>
<subtitle>It’s just data</subtitle>
<title>Sam Ruby</title>
<updated>2006-06-17T00:15:18Z</updated>
<updated planet:format='June 17, 2006 12:15 AM'>2006-06-17T00:15:18Z</updated>
<planet:name>one</planet:name>
</source>
</entry>
......@@ -4,7 +4,7 @@
<link href='http://example.com/4' type='text/html' rel='alternate'/>
<title>Mars</title>
<content>the Red Planet</content>
<updated>2006-01-04T00:00:00Z</updated>
<updated planet:format='January 04, 2006 12:00 AM'>2006-01-04T00:00:00Z</updated>
<source>
<id>tag:planet.intertwingly.net,2006:testfeed1</id>
<author>
......@@ -16,7 +16,7 @@
<link href='http://www.intertwingly.net/blog/' type='text/html' rel='alternate'/>
<subtitle>It’s just data</subtitle>
<title>Sam Ruby</title>
<updated>2006-06-17T00:15:18Z</updated>
<updated planet:format='June 17, 2006 12:15 AM'>2006-06-17T00:15:18Z</updated>
<planet:name>one</planet:name>
</source>
</entry>
......@@ -4,7 +4,7 @@
<link href='http://example.com/1' type='text/html' rel='alternate'/>
<title>Mercury</title>
<content>Messenger of the Roman Gods</content>
<updated>2006-01-01T00:00:00Z</updated>
<updated planet:format='January 01, 2006 12:00 AM'>2006-01-01T00:00:00Z</updated>
<source>
<id>tag:planet.intertwingly.net,2006:testfeed2</id>
<author>
......@@ -16,7 +16,7 @@
<link href='http://www.intertwingly.net/blog/' type='text/html' rel='alternate'/>
<subtitle>It’s just data</subtitle>
<title>Sam Ruby</title>
<updated>2006-06-17T00:15:18Z</updated>
<updated planet:format='June 17, 2006 12:15 AM'>2006-06-17T00:15:18Z</updated>
<planet:name>two</planet:name>
</source>
</entry>
......@@ -4,7 +4,7 @@
<link href='http://example.com/2' type='text/html' rel='alternate'/>
<title>Venus</title>
<content>the Morning Star</content>
<updated>2006-01-02T00:00:00Z</updated>
<updated planet:format='January 02, 2006 12:00 AM'>2006-01-02T00:00:00Z</updated>
<source>
<id>tag:planet.intertwingly.net,2006:testfeed2</id>
<author>
......@@ -16,7 +16,7 @@
<link href='http://www.intertwingly.net/blog/' type='text/html' rel='alternate'/>
<subtitle>It’s just data</subtitle>
<title>Sam Ruby</title>
<updated>2006-06-17T00:15:18Z</updated>
<updated planet:format='June 17, 2006 12:15 AM'>2006-06-17T00:15:18Z</updated>
<planet:name>two</planet:name>
</source>
</entry>
......@@ -4,7 +4,7 @@
<link href='http://example.com/3' type='text/html' rel='alternate'/>
<title>Earth</title>
<content>the Blue Planet</content>
<updated>2006-01-03T00:00:00Z</updated>
<updated planet:format='January 03, 2006 12:00 AM'>2006-01-03T00:00:00Z</updated>
<source>
<id>tag:planet.intertwingly.net,2006:testfeed2</id>
<author>
......@@ -16,7 +16,7 @@
<link href='http://www.intertwingly.net/blog/' type='text/html' rel='alternate'/>
<subtitle>It’s just data</subtitle>
<title>Sam Ruby</title>
<updated>2006-06-17T00:15:18Z</updated>
<updated planet:format='June 17, 2006 12:15 AM'>2006-06-17T00:15:18Z</updated>
<planet:name>two</planet:name>
</source>
</entry>
......@@ -4,7 +4,7 @@
<link href='http://example.com/4' type='text/html' rel='alternate'/>
<title>Mars</title>
<content>the Red Planet</content>
<updated>2006-01-04T00:00:00Z</updated>
<updated planet:format='January 04, 2006 12:00 AM'>2006-01-04T00:00:00Z</updated>
<source>
<id>tag:planet.intertwingly.net,2006:testfeed2</id>
<author>
......@@ -16,7 +16,7 @@
<link href='http://www.intertwingly.net/blog/' type='text/html' rel='alternate'/>
<subtitle>It’s just data</subtitle>
<title>Sam Ruby</title>
<updated>2006-06-17T00:15:18Z</updated>
<updated planet:format='June 17, 2006 12:15 AM'>2006-06-17T00:15:18Z</updated>
<planet:name>two</planet:name>
</source>
</entry>
......@@ -4,7 +4,7 @@
<link href='http://example.com/1' type='text/html' rel='alternate'/>
<title>Mercury</title>
<summary>Messenger of the Roman Gods</summary>
<updated>2006-01-01T00:00:00Z</updated>
<updated planet:format='January 01, 2006 12:00 AM'>2006-01-01T00:00:00Z</updated>
<source>
<link href='http://intertwingly.net/code/venus/tests/data/spider/testfeed3.rss' type='text/html' rel='alternate'/>
<link href='tests/data/spider/testfeed3.rss' type='application/atom+xml' rel='self'/>
......
......@@ -4,7 +4,7 @@
<link href='http://example.com/2' type='text/html' rel='alternate'/>
<title>Venus</title>
<summary>the Morning Star</summary>
<updated>2006-08-18T18:30:50Z</updated>
<updated planet:format='August 21, 2006 12:54 PM'>2006-08-21T12:54:31Z</updated>
<source>
<link href='http://intertwingly.net/code/venus/tests/data/spider/testfeed3.rss' type='text/html' rel='alternate'/>
<link href='tests/data/spider/testfeed3.rss' type='application/atom+xml' rel='self'/>
......
......@@ -10,6 +10,6 @@
<link href='http://www.intertwingly.net/blog/' type='text/html' rel='alternate'/>
<subtitle>It’s just data</subtitle>
<title>Sam Ruby</title>
<updated>2006-06-17T00:15:18Z</updated>
<updated planet:format='June 17, 2006 12:15 AM'>2006-06-17T00:15:18Z</updated>
<planet:name>one</planet:name>
</feed>
......@@ -10,6 +10,6 @@
<link href='http://www.intertwingly.net/blog/' type='text/html' rel='alternate'/>
<subtitle>It’s just data</subtitle>
<title>Sam Ruby</title>
<updated>2006-06-17T00:15:18Z</updated>
<updated planet:format='June 17, 2006 12:15 AM'>2006-06-17T00:15:18Z</updated>
<planet:name>two</planet:name>
</feed>
[Planet]
name = test planet
cache_directory = tests/data/splice/cache
template_files =
[tests/data/spider/testfeed0.atom]
name = not found
......
#!/usr/bin/env python
import unittest, os, shutil
from planet import config, splice
workdir = 'tests/work/apply'
configfile = 'tests/data/apply/config.ini'
testfeed = 'tests/data/apply/feed.xml'
class ApplyTest(unittest.TestCase):
def setUp(self):
try:
os.makedirs(workdir)
except:
self.tearDown()
os.makedirs(workdir)
def tearDown(self):
shutil.rmtree(workdir)
os.removedirs(os.path.split(workdir)[0])
def test_apply(self):
testfile = open(testfeed)
feeddata = testfile.read()
testfile.close()
config.load(configfile)
splice.apply(feeddata)
for file in ['index.html', 'default.css', 'images/foaf.png']:
path = os.path.join(workdir, file)
self.assertTrue(os.path.exists(path))
self.assertTrue(os.stat(path).st_size > 0)
......@@ -3,8 +3,6 @@
import unittest
from planet import config
workdir = 'tests/work/spider/cache'
class ConfigTest(unittest.TestCase):
def setUp(self):
config.load('tests/data/config/basic.ini')
......@@ -16,7 +14,9 @@ class ConfigTest(unittest.TestCase):
config.template_files())
def test_feeds(self):
self.assertEqual(['feed1', 'feed2'], config.feeds())
feeds = config.feeds()
feeds.sort()
self.assertEqual(['feed1', 'feed2'], feeds)
# planet wide configuration
......
#!/usr/bin/env python
import unittest, os, glob, calendar
import unittest, os, glob, calendar, shutil
from planet.spider import filename, spiderFeed, spiderPlanet
from planet import feedparser, config
......@@ -17,13 +17,8 @@ class SpiderTest(unittest.TestCase):
os.makedirs(workdir)
def tearDown(self):
for file in glob.glob(workdir+"/sources/*"):
os.unlink(file)
if os.path.exists(workdir+"/sources"):
os.rmdir(workdir+"/sources")
for file in glob.glob(workdir+"/*"):
os.unlink(file)
os.removedirs(workdir)
shutil.rmtree(workdir)
os.removedirs(os.path.split(workdir)[0])
def test_filename(self):
self.assertEqual('./example.com,index.html',
......
#!/usr/bin/env python
import unittest
from planet import config
from os.path import split
class ConfigTest(unittest.TestCase):
def setUp(self):
config.load('tests/data/config/themed.ini')
# template directories
def test_template_directories(self):
self.assertEqual(['foo', 'bar', 'asf', 'common'],
[split(dir)[1] for dir in config.template_directories()])
# administrivia
def test_template(self):
self.assertTrue('index.html.xslt' in config.template_files())
def test_feeds(self):
feeds = config.feeds()
feeds.sort()
self.assertEqual(['feed1', 'feed2'], feeds)
# planet wide configuration
def test_name(self):
self.assertEqual('Test Configuration', config.name())
def test_link(self):
self.assertEqual('Unconfigured Planet', config.link())
# per template configuration
def test_days_per_page(self):
self.assertEqual(7, config.days_per_page('index.html.xslt'))
self.assertEqual(0, config.days_per_page('atom.xml.xslt'))
def test_items_per_page(self):
self.assertEqual(50, config.items_per_page('index.html.xslt'))
self.assertEqual(50, config.items_per_page('atom.xml.xslt'))
def test_encoding(self):
self.assertEqual('utf-8', config.encoding('index.html.xslt'))
self.assertEqual('utf-8', config.encoding('atom.xml.xslt'))
# dictionaries
def test_feed_options(self):
self.assertEqual('one', config.feed_options('feed1')['name'])
self.assertEqual('two', config.feed_options('feed2')['name'])