qrunner.py 9.51 KB
Newer Older
Barry Warsaw's avatar
Barry Warsaw committed
1
# Copyright (C) 2001-2009 by the Free Software Foundation, Inc.
2
#
Barry Warsaw's avatar
Barry Warsaw committed
3
# This file is part of GNU Mailman.
4
#
Barry Warsaw's avatar
Barry Warsaw committed
5 6 7 8
# GNU Mailman is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
9
#
Barry Warsaw's avatar
Barry Warsaw committed
10 11 12 13 14 15 16
# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
17 18 19 20 21

import sys
import signal
import logging

Barry Warsaw's avatar
Barry Warsaw committed
22 23
from mailman.config import config
from mailman.core.logging import reopen
24
from mailman.i18n import _
25
from mailman.options import Options
26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55


COMMASPACE = ', '
log = None



def r_callback(option, opt, value, parser):
    dest = getattr(parser.values, option.dest)
    parts = value.split(':')
    if len(parts) == 1:
        runner = parts[0]
        rslice = rrange = 1
    elif len(parts) == 3:
        runner = parts[0]
        try:
            rslice = int(parts[1])
            rrange = int(parts[2])
        except ValueError:
            parser.print_help()
            print >> sys.stderr, _('Bad runner specification: $value')
            sys.exit(1)
    else:
        parser.print_help()
        print >> sys.stderr, _('Bad runner specification: $value')
        sys.exit(1)
    dest.append((runner, rslice, rrange))



56
class ScriptOptions(Options):
57 58

    usage=_("""\
59 60 61 62 63 64 65 66 67 68 69 70 71 72 73
Run one or more qrunners, once or repeatedly.

Each named runner class is run in round-robin fashion.  In other words, the
first named runner is run to consume all the files currently in its
directory.  When that qrunner is done, the next one is run to consume all the
files in /its/ directory, and so on.  The number of total iterations can be
given on the command line.

Usage: %prog [options]

-r is required unless -l or -h is given, and its argument must be one of the
names displayed by the -l switch.

Normally, this script should be started from mailmanctl.  Running it
separately or with -o is generally useful only for debugging.
74 75 76 77 78 79 80 81 82
""")

    def add_options(self):
        self.parser.add_option(
            '-r', '--runner',
            metavar='runner[:slice:range]', dest='runners',
            type='string', default=[],
            action='callback', callback=r_callback,
            help=_("""\
83 84 85 86 87 88 89 90 91 92 93 94
Run the named qrunner, which must be one of the strings returned by the -l
option.  Optional slice:range if given, is used to assign multiple qrunner
processes to a queue.  range is the total number of qrunners for this queue
while slice is the number of this qrunner from [0..range).

When using the slice:range form, you must ensure that each qrunner for the
queue is given the same range value.  If slice:runner is not given, then 1:1
is used.

Multiple -r options may be given, in which case each qrunner will run once in
round-robin fashion.  The special runner `All' is shorthand for a qrunner for
each listed by the -l option."""))
95 96 97
        self.parser.add_option(
            '-o', '--once',
            default=False, action='store_true', help=_("""\
98
Run each named qrunner exactly once through its main loop.  Otherwise, each
99
qrunner runs indefinitely, until the process receives signal."""))
100 101 102 103 104 105 106
        self.parser.add_option(
            '-l', '--list',
            default=False, action='store_true',
            help=_('List the available qrunner names and exit.'))
        self.parser.add_option(
            '-v', '--verbose',
            default=0, action='count', help=_("""\
107
Display more debugging information to the logs/qrunner log file."""))
108 109 110
        self.parser.add_option(
            '-s', '--subproc',
            default=False, action='store_true', help=_("""\
111 112 113
This should only be used when running qrunner as a subprocess of the
mailmanctl startup script.  It changes some of the exit-on-error behavior to
work better with that framework."""))
114 115 116 117 118 119

    def sanity_check(self):
        if self.arguments:
            self.parser.error(_('Unexpected arguments'))
        if not self.options.runners and not self.options.list:
            self.parser.error(_('No runner name given.'))
120 121 122 123



def make_qrunner(name, slice, range, once=False):
124
    # Several conventions for specifying the runner name are supported.  It
Barry Warsaw's avatar
Barry Warsaw committed
125
    # could be one of the shortcut names.  If the name is a full module path,
126 127
    # use it explicitly.  If the name starts with a dot, it's a class name
    # relative to the Mailman.queue package.
Barry Warsaw's avatar
Barry Warsaw committed
128 129 130 131
    qrunner_config = getattr(config, 'qrunner.' + name, None)
    if qrunner_config is not None:
        # It was a shortcut name.
        class_path = qrunner_config['class']
132
    elif name.startswith('.'):
Barry Warsaw's avatar
Barry Warsaw committed
133
        class_path = 'mailman.queue' + name
134
    else:
Barry Warsaw's avatar
Barry Warsaw committed
135 136
        class_path = name
    module_name, class_name = class_path.rsplit('.', 1)
137
    try:
Barry Warsaw's avatar
Barry Warsaw committed
138
        __import__(module_name)
139
    except ImportError, e:
140
        if config.options.options.subproc:
141 142
            # Exit with SIGTERM exit code so the master watcher won't try to
            # restart us.
Barry Warsaw's avatar
Barry Warsaw committed
143
            print >> sys.stderr, _('Cannot import runner module: $module_name')
144 145 146
            print >> sys.stderr, e
            sys.exit(signal.SIGTERM)
        else:
147
            raise
Barry Warsaw's avatar
Barry Warsaw committed
148
    qrclass = getattr(sys.modules[module_name], class_name)
149
    if once:
Barry Warsaw's avatar
Barry Warsaw committed
150
        # Subclass to hack in the setting of the stop flag in _do_periodic()
151
        class Once(qrclass):
Barry Warsaw's avatar
Barry Warsaw committed
152
            def _do_periodic(self):
153 154 155 156 157 158 159 160 161
                self.stop()
        qrunner = Once(slice, range)
    else:
        qrunner = qrclass(slice, range)
    return qrunner



def set_signals(loop):
162 163 164 165 166 167 168 169 170 171
    """Set up the signal handlers.

    Signals caught are: SIGTERM, SIGINT, SIGUSR1 and SIGHUP.  The latter is
    used to re-open the log files.  SIGTERM and SIGINT are treated exactly the
    same -- they cause qrunner to exit with no restart from the master.
    SIGUSR1 also causes qrunner to exit, but the master watcher will restart
    it in that case.

    :param loop: A loop queue runner instance.
    """
172 173 174 175 176 177 178 179 180 181 182 183
    def sigterm_handler(signum, frame):
        # Exit the qrunner cleanly
        loop.stop()
        loop.status = signal.SIGTERM
        log.info('%s qrunner caught SIGTERM.  Stopping.', loop.name())
    signal.signal(signal.SIGTERM, sigterm_handler)
    def sigint_handler(signum, frame):
        # Exit the qrunner cleanly
        loop.stop()
        loop.status = signal.SIGINT
        log.info('%s qrunner caught SIGINT.  Stopping.', loop.name())
    signal.signal(signal.SIGINT, sigint_handler)
184 185 186 187 188 189
    def sigusr1_handler(signum, frame):
        # Exit the qrunner cleanly
        loop.stop()
        loop.status = signal.SIGUSR1
        log.info('%s qrunner caught SIGUSR1.  Stopping.', loop.name())
    signal.signal(signal.SIGUSR1, sigusr1_handler)
190 191
    # SIGHUP just tells us to rotate our log files.
    def sighup_handler(signum, frame):
Barry Warsaw's avatar
Barry Warsaw committed
192
        reopen()
193 194 195 196 197 198
        log.info('%s qrunner caught SIGHUP.  Reopening logs.', loop.name())
    signal.signal(signal.SIGHUP, sighup_handler)



def main():
199
    global log
200

201 202
    options = ScriptOptions()
    options.initialize()
203

204
    if options.options.list:
205 206 207 208
        prefixlen = max(len(shortname)
                        for shortname in config.qrunner_shortcuts)
        for shortname in sorted(config.qrunner_shortcuts):
            runnername = config.qrunner_shortcuts[shortname]
209
            shortname = (' ' * (prefixlen - len(shortname))) + shortname
210
            print _('$shortname runs $runnername')
211 212 213
        sys.exit(0)

    # Fast track for one infinite runner
214 215
    if len(options.options.runners) == 1 and not options.options.once:
        qrunner = make_qrunner(*options.options.runners[0])
216 217 218 219 220 221 222 223 224 225 226
        class Loop:
            status = 0
            def __init__(self, qrunner):
                self._qrunner = qrunner
            def name(self):
                return self._qrunner.__class__.__name__
            def stop(self):
                self._qrunner.stop()
        loop = Loop(qrunner)
        set_signals(loop)
        # Now start up the main loop
227
        log = logging.getLogger('mailman.qrunner')
228 229 230 231 232 233
        log.info('%s qrunner started.', loop.name())
        qrunner.run()
        log.info('%s qrunner exiting.', loop.name())
    else:
        # Anything else we have to handle a bit more specially
        qrunners = []
234
        for runner, rslice, rrange in options.options.runners:
235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255
            qrunner = make_qrunner(runner, rslice, rrange, once=True)
            qrunners.append(qrunner)
        # This class is used to manage the main loop
        class Loop:
            status = 0
            def __init__(self):
                self._isdone = False
            def name(self):
                return 'Main loop'
            def stop(self):
                self._isdone = True
            def isdone(self):
                return self._isdone
        loop = Loop()
        set_signals(loop)
        log.info('Main qrunner loop started.')
        while not loop.isdone():
            for qrunner in qrunners:
                # In case the SIGTERM came in the middle of this iteration
                if loop.isdone():
                    break
256
                if options.options.verbose:
257 258 259
                    log.info('Now doing a %s qrunner iteration',
                             qrunner.__class__.__bases__[0].__name__)
                qrunner.run()
260
            if options.options.once:
261 262 263 264 265 266 267 268 269
                break
        log.info('Main qrunner loop exiting.')
    # All done
    sys.exit(loop.status)



if __name__ == '__main__':
    main()