testall.py 9.94 KB
Newer Older
Barry Warsaw's avatar
Barry Warsaw committed
1
# Copyright (C) 2001-2009 by the Free Software Foundation, Inc.
2
#
Barry Warsaw's avatar
Barry Warsaw committed
3
# This file is part of GNU Mailman.
4
#
Barry Warsaw's avatar
Barry Warsaw committed
5 6 7 8
# GNU Mailman is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
9
#
Barry Warsaw's avatar
Barry Warsaw committed
10 11 12 13 14 15 16
# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
17 18 19 20 21

"""Mailman unit test driver."""

import os
import re
22 23
import grp
import pwd
24
import sys
25
import random
26
import shutil
27
import optparse
28
import tempfile
29
import unittest
30
import pkg_resources
31

32
from mailman import Defaults
33 34
from mailman.configuration import config
from mailman.i18n import _
35
from mailman.initialize import initialize_1, initialize_2, initialize_3
36
from mailman.testing.helpers import SMTPServer
37 38
from mailman.version import MAILMAN_VERSION

39

40
basedir = None
41 42 43 44 45 46 47 48 49



def v_callback(option, opt, value, parser):
    if opt in ('-q', '--quiet'):
        delta = -1
    elif opt in ('-v', '--verbose'):
        delta = 1
    else:
50
        raise AssertionError('Unexpected option: %s' % opt)
51 52 53 54 55
    dest = getattr(parser.values, option.dest)
    setattr(parser.values, option.dest, max(0, dest + delta))


def parseargs():
56
    parser = optparse.OptionParser(version=MAILMAN_VERSION,
57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75
                                   usage=_("""\
%prog [options] [tests]

Run the Mailman unit test suite.  'tests' is one or more Python regular
expressions matching only the tests you want to run.  Prefix the regular
expression with '!' to specify a negative test."""))
    parser.set_defaults(verbosity=2)
    parser.add_option('-v', '--verbose',
                      action='callback', callback=v_callback,
                      dest='verbosity', help=_("""\
Increase verbosity by 1, which defaults to %default.  Use -q to reduce
verbosity.  -v and -q options accumulate."""))
    parser.add_option('-q', '--quiet',
                      action='callback', callback=v_callback,
                      dest='verbosity', help=_("""\
Reduce verbosity by 1 (but not below 0)."""))
    parser.add_option('-e', '--stderr',
                      default=False, action='store_true',
                      help=_('Propagate log errors to stderr.'))
76 77 78
    parser.add_option('-c', '--coverage',
                      default=False, action='store_true',
                      help=_('Enable code coverage.'))
79 80 81 82 83 84
    parser.add_option('-r', '--randomize',
                      default=False, action='store_true',
                      help=_("""\
Randomize the tests; good for finding subtle dependency errors.  Note that
this isn't completely random though because the doctests are not mixed with
the Python tests.  Each type of test is randomized within its group."""))
85 86 87 88 89 90
    options, arguments = parser.parse_args()
    if len(arguments) == 0:
        arguments = ['.']
    parser.options = options
    parser.arguments = arguments
    return parser
91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111



def search():
    testnames = []
    # Walk the entire tree from the current base directory.  Look for modules
    # that start with 'test_'.  Calculate the full module path name to this
    # module, append 'test_suite' and add that to testnames.  This way, we run
    # all the suites defined in the test_suite() function inside all test
    # modules.
    for dirpath, dirnames, filenames in os.walk(basedir):
        for fn in filenames:
            if fn.startswith('test_') and fn.endswith('.py'):
                # Start with full path
                path = os.path.join(dirpath, fn)
                # Convert the file path to a module path.  First, we must make
                # the file path relative to the root directory.  Then strip
                # off the trailing .py
                path = path[len(basedir)+1:-3]
                # Convert slashes to dots
                modpath = path.replace(os.sep, '.') + '.test_suite'
112
                testnames.append('mailman.' + modpath)
113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145
    return testnames


def match(pat, name):
    if not pat:
        return True
    if pat.startswith('!'):
        # Negative test
        return re.search(pat[1:], name) is None
    else:
        # Positive test
        return re.search(pat, name) is not None


def filter_tests(suite, patterns):
    if '.' in patterns:
        return suite
    new = unittest.TestSuite()
    for test in suite._tests:
        if isinstance(test, unittest.TestCase):
            # Get the fill test name: package.module.class.method
            name = test.id()
            for pat in patterns:
                if match(pat, name):
                    new.addTest(test)
                    break
        else:
            filtered = filter_tests(test, patterns)
            if filtered:
                new.addTest(filtered)
    return new


146
def suite(patterns, randomize):
147 148 149 150 151 152
    if patterns is None:
        patterns = '.'
    loader = unittest.TestLoader()
    # Search for all tests that match the given patterns
    testnames = search()
    suite = loader.loadTestsFromNames(testnames)
153 154 155 156 157 158
    tests = filter_tests(suite, patterns)
    if randomize:
        random.shuffle(tests._tests)
    else:
        tests._tests.sort()
    return tests
159 160 161 162 163 164



def main():
    global basedir

165
    parser = parseargs()
166

167 168
    # Set verbosity level for test_documentation.py.  XXX There should be a
    # better way to do this.
169 170 171 172
    class Bag: pass
    config.tests = Bag()
    config.tests.verbosity = parser.options.verbosity
    config.tests.randomize = parser.options.randomize
173

174
    # Turn on code coverage if selected.
175
    if parser.options.coverage:
176 177 178
        try:
            import coverage
        except ImportError:
179
            parser.options.coverage = False
180 181 182
        else:
            coverage.start()

183 184 185
    # Set up the testing configuration file both for this process, and for all
    # sub-processes testing will spawn (e.g. the qrunners).
    #
186 187 188 189 190 191 192 193 194 195 196 197 198 199 200
    # Also create a logging.cfg file with values reflecting verbosity and
    # stderr propagation.  Enable it only if necessary.
    fd, logging_cfg = tempfile.mkstemp(suffix='.cfg')
    os.close(fd)
    enable_logging_cfg = False
    with open(logging_cfg, 'w') as fp:
        print >> fp, '[*]'
        if parser.options.stderr:
            print >> fp, 'propagate = True'
            enable_logging_cfg = True
        if parser.options.verbosity > 2:
            print >> fp, 'level = DEBUG'
            enable_logging_cfg = True

    cfg_in = pkg_resources.resource_string(
201
        'mailman.testing', 'testing.cfg.in')
202 203 204 205 206 207 208
    fd, cfg_out = tempfile.mkstemp(suffix='.cfg')
    os.close(fd)
    with open(cfg_out, 'w') as fp:
        fp.write(cfg_in)
        if enable_logging_cfg:
            print >> fp, 'LOG_CONFIG_FILE = "%s"' % logging_cfg

209
    # Calculate a temporary VAR_DIR directory so that run-time artifacts of
210 211 212
    # the tests won't tread on the installation's data.  This also makes it
    # easier to clean up after the tests are done, and insures isolation of
    # test suite runs.
213
    var_dir = tempfile.mkdtemp()
214
    if parser.options.verbosity > 2:
215
        print 'VAR_DIR :', var_dir
216
        print 'config file:', cfg_out
217 218
        if enable_logging_cfg:
            print 'logging config file:', logging_cfg
219

220 221 222 223 224
    user_id = os.getuid()
    user_name = pwd.getpwuid(user_id).pw_name
    group_id = os.getgid()
    group_name = grp.getgrgid(group_id).gr_name

225
    try:
226
        with open(cfg_out, 'a') as fp:
227 228
            print >> fp, 'VAR_DIR = "%s"' % var_dir
            print >> fp, 'MAILMAN_USER = "%s"' % user_name
229
            print >> fp, 'MAILMAN_UID =', user_id
230
            print >> fp, 'MAILMAN_GROUP = "%s"' % group_name
231
            print >> fp, 'MAILMAN_GID =', group_id
232
            print >> fp, "LANGUAGES = 'en'"
233
            print >> fp, 'SMTPPORT =', SMTPServer.port
234 235 236
            # A fake MHonArc command, for testing.
            print >> fp, 'MHONARC_COMMAND = """/bin/echo', \
                  Defaults.MHONARC_COMMAND, '"""'
237

238
        initialize_1(cfg_out, propagate_logs=parser.options.stderr)
239 240 241 242 243 244 245 246 247 248 249 250
        mailman_uid = pwd.getpwnam(config.MAILMAN_USER).pw_uid
        mailman_gid = grp.getgrnam(config.MAILMAN_GROUP).gr_gid
        os.chmod(cfg_out, 0660)
        os.chown(cfg_out, mailman_uid, mailman_gid)

        # Create an empty SQLite database file with the proper permissions and
        # calculate the SQLAlchemy engine url to this database file.
        fd, config.dbfile = tempfile.mkstemp(dir=config.DATA_DIR, suffix='.db')
        os.close(fd)
        os.chmod(config.dbfile, 0660)
        os.chown(config.dbfile, mailman_uid, mailman_gid)

251
        # Patch ups.
252
        test_engine_url = 'sqlite:///' + config.dbfile
253
        config.DEFAULT_DATABASE_URL = test_engine_url
254 255 256 257

        # Write this to the config file so subprocesses share the same testing
        # database file.
        with open(cfg_out, 'a') as fp:
258
            print >> fp, 'DEFAULT_DATABASE_URL = "%s"' % test_engine_url
259

260
        # With -vvv, turn on engine debugging.
261
        initialize_2(parser.options.verbosity > 3)
262
        initialize_3()
263

264 265 266
        # Run the tests.  XXX I'm not sure if basedir can be converted to
        # pkg_resources.
        import mailman
267
        basedir = os.path.dirname(mailman.__file__)
268
        runner = unittest.TextTestRunner(verbosity=parser.options.verbosity)
269
        results = runner.run(suite(parser.arguments, parser.options.randomize))
270 271
    finally:
        os.remove(cfg_out)
272
        os.remove(logging_cfg)
273
        shutil.rmtree(var_dir)
274

275
    # Turn off coverage and print a report
276
    if parser.options.coverage:
277 278 279 280
        coverage.stop()
        modules = [module for name, module in sys.modules.items()
                   if module
                   and name is not None
281
                   and name.split('.')[0] == 'mailman']
282
        coverage.report(modules)
283 284 285 286 287 288
    sys.exit(bool(results.failures or results.errors))



if __name__ == '__main__':
    main()