Commit 2ccd1eb8 authored by Antoine Beaupré's avatar Antoine Beaupré

get rid of dispatcher

make renderer dispatch jobs directly, and guess properly the output path based on srcdir or not

this simplifies the rendering code even further and finalizes the removal of duplicate patterns everywhere
parent bffdd13d
......@@ -49,9 +49,15 @@ def render(obj, srcdir):
this looks for patterns matching a certain regex in the given
SRCDIR directory
.. note:: this assumes files have an extension that should be
stripped. for manpages, this should generally be
``.gz``. if manpages are not compressed, this will break
section support.
.. todo:: document that compressed manpages are mandatory
output = obj['output']
cache = obj['cache']
mirror = obj['mirror']
patterns = obj['patterns']
......@@ -59,6 +65,7 @@ def render(obj, srcdir):
if 'changed_paths' in obj:
if srcdir:
logging.warn('ignoring --srcdir parameter because called with extractor')
srcdir = None'received %d paths from extractor',
filelist = match_jobs(obj['changed_paths'], patterns)
......@@ -81,11 +88,26 @@ def render(obj, srcdir):
progress = fake_progress
i = 0
t = time.time()
with progress(list(filelist),
label='rendering manpages') as bar:
for job in bar:
dispatch(job, destdir=output, dryrun=obj['dryrun'],
cache=cache, prefix=obj['prefix'], mirror=mirror)
for renderer, source, match in bar:
if srcdir:
# from srcdir, replace it with output dir
target = source.replace(srcdir, output, 1)
# from extractor: already in the output dir
target = source
# replace .gz extension with .html
base, ext = os.path.splitext(target)
target = os.path.abspath(base + '.html')
renderer.render(source, target,
except CommandRendererError as e:
logging.warn('%s failed to convert %s: %s',
renderer, source, e)
i += 1'rendered %d manpages in %s', i,
naturaldelta(time.time() - t))
......@@ -360,8 +382,9 @@ def find_files(directory, patterns):
for path in files:
path = os.path.join(root, path)
for regex, module in patterns.iteritems():
if, path):
yield module, path
m =, path)
if m:
yield module, path, m
def match_jobs(files, patterns):
......@@ -377,34 +400,9 @@ def match_jobs(files, patterns):
for path in files:
for regex, module in patterns.iteritems():
if, path):
yield module, path
def dispatch(job, destdir, dryrun=False, cache=True, prefix=None, mirror=None):
'''process a specific job and call the correct renderer
.. todo:: abstract manpages away from here. we should strip srcdir
from the source file, if it exists, otherwise assume it
is in destdir and strip then, then replace the file
extension with :func:`os.path.splitext`::
# remove `.gz` prefix
base, ext = os.path.splitext(file)
htmlfile = os.path.abspath(base + '.html')
Unfortunately, just doing the above right now breaks
--srcdir support. This may require moving the dispatcher
in a class to keep better state.
renderer, source = job
# should match ManpageRenderer.pattern
target = re.sub(r'^.*/([^/]*/man/(?:\w+/)?man[1-9]/.+\.[1-9]\w*)(?:\.gz)?$', r'\1.html', source)
target = os.path.join(destdir, target)
renderer.render(source, target, prefix=prefix, suites=mirror.releases)
except CommandRendererError as e:
logging.warn('%s failed to convert %s: %s', renderer, source, e)
m =, path)
if m:
yield module, path, m
......@@ -40,30 +40,19 @@ def search(pattern):
.. note:: inspired by
# XXX: this is getting a tad ridiculous now
# we shouldn't have to hack at the pattern like this
# there is a similar problem in dispatch - maybe file list
# should include the match pattern?
pattern = re.sub('\(\?P<name>[^)]*\)',
pattern, # XXX: wildcard or not?
logger.debug('searching for pattern: %s', pattern)
patterns = {}
# XXX: this doesn't belong here? should be command-level patterns
patterns[re.compile(pattern)] = ManpageRenderer
# XXX: not sure this is the best structure
manpages = {} # suite -> (name, section)
i = 0
assert app.static_folder
for module, path in find_files(app.static_folder, patterns):
filelist = find_files(app.static_folder, app.config['patterns'])
for module, path, match in filelist:
suite ='suite')
name ='name')
section ='section')
if pattern not in name:
i += 1
# XXX: again
m =, path)
assert m
suite ='suite')
name ='name')
section ='section')
if section not in manpages:
manpages[section] = set()
logger.debug('found manpage %s (%s) in %s', name, section, suite)
......@@ -101,12 +90,14 @@ def serve(obj, port):
this is used for development purposes and is not designed to be a
fully-fledged webserver
main(obj['output'], port, releases=obj['mirror'].releases)
main(obj['output'], port, releases=obj['mirror'].releases,
def main(output, port=8000, releases={}):
def main(output, port=8000, releases={}, patterns={}):
app.static_folder = os.path.abspath(output)
app.config['releases'] = releases
app.config['patterns'] = patterns
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment