path: root/mailman/bin/testall.py

# Copyright (C) 2001-2008 by the Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
# USA.

"""Mailman unit test driver."""

from __future__ import with_statement

import os
import re
import grp
import pwd
import sys
import random
import shutil
import optparse
import tempfile
import unittest
import pkg_resources

from mailman.configuration import config
from mailman.i18n import _
from mailman.initialize import initialize_1, initialize_2
from mailman.testing.helpers import SMTPServer
from mailman.version import MAILMAN_VERSION


basedir = None



def v_callback(option, opt, value, parser):
    if opt in ('-q', '--quiet'):
        delta = -1
    elif opt in ('-v', '--verbose'):
        delta = 1
    else:
        raise AssertionError('Unexpected option: %s' % opt)
    dest = getattr(parser.values, option.dest)
    setattr(parser.values, option.dest, max(0, dest + delta))


def parseargs():
    parser = optparse.OptionParser(version=MAILMAN_VERSION,
                                   usage=_("""\
%prog [options] [tests]

Run the Mailman unit test suite.  'tests' is one or more Python regular
expressions matching only the tests you want to run.  Prefix the regular
expression with '!' to specify a negative test."""))
    parser.set_defaults(verbosity=2)
    parser.add_option('-v', '--verbose',
                      action='callback', callback=v_callback,
                      dest='verbosity', help=_("""\
Increase verbosity by 1, which defaults to %default.  Use -q to reduce
verbosity.  -v and -q options accumulate."""))
    parser.add_option('-q', '--quiet',
                      action='callback', callback=v_callback,
                      dest='verbosity', help=_("""\
Reduce verbosity by 1 (but not below 0)."""))
    parser.add_option('-e', '--stderr',
                      default=False, action='store_true',
                      help=_('Propagate log errors to stderr.'))
    parser.add_option('-c', '--coverage',
                      default=False, action='store_true',
                      help=_('Enable code coverage.'))
    parser.add_option('-r', '--randomize',
                      default=False, action='store_true',
                      help=_("""\
Randomize the tests; good for finding subtle dependency errors.  Note that
this isn't completely random though because the doctests are not mixed with
the Python tests.  Each type of test is randomized within its group."""))
    options, arguments = parser.parse_args()
    if len(arguments) == 0:
        arguments = ['.']
    parser.options = options
    parser.arguments = arguments
    return parser



def search():
    testnames = []
    # Walk the entire tree from the current base directory.  Look for modules
    # that start with 'test_'.  Calculate the full module path name to this
    # module, append 'test_suite' and add that to testnames.  This way, we run
    # all the suites defined in the test_suite() function inside all test
    # modules.
    for dirpath, dirnames, filenames in os.walk(basedir):
        for fn in filenames:
            if fn.startswith('test_') and fn.endswith('.py'):
                # Start with full path
                path = os.path.join(dirpath, fn)
                # Convert the file path to a module path.  First, we must make
                # the file path relative to the root directory.  Then strip
                # off the trailing .py
                path = path[len(basedir)+1:-3]
                # Convert slashes to dots
                modpath = path.replace(os.sep, '.') + '.test_suite'
                testnames.append('mailman.' + modpath)
    return testnames


def match(pat, name):
    if not pat:
        return True
    if pat.startswith('!'):
        # Negative test
        return re.search(pat[1:], name) is None
    else:
        # Positive test
        return re.search(pat, name) is not None


def filter_tests(suite, patterns):
    if '.' in patterns:
        return suite
    new = unittest.TestSuite()
    for test in suite._tests:
        if isinstance(test, unittest.TestCase):
            # Get the fill test name: package.module.class.method
            name = test.id()
            for pat in patterns:
                if match(pat, name):
                    new.addTest(test)
                    break
        else:
            filtered = filter_tests(test, patterns)
            if filtered:
                new.addTest(filtered)
    return new


def suite(patterns, randomize):
    if patterns is None:
        patterns = '.'
    loader = unittest.TestLoader()
    # Search for all tests that match the given patterns
    testnames = search()
    suite = loader.loadTestsFromNames(testnames)
    tests = filter_tests(suite, patterns)
    if randomize:
        random.shuffle(tests._tests)
    else:
        tests._tests.sort()
    return tests



def main():
    global basedir

    parser = parseargs()

    # Set verbosity level for test_documentation.py.  XXX There should be a
    # better way to do this.
    class Bag: pass
    config.tests = Bag()
    config.tests.verbosity = parser.options.verbosity
    config.tests.randomize = parser.options.randomize

    # Turn on code coverage if selected.
    if parser.options.coverage:
        try:
            import coverage
        except ImportError:
            parser.options.coverage = False
        else:
            coverage.start()

    # Set up the testing configuration file both for this process, and for all
    # sub-processes testing will spawn (e.g. the qrunners).
    #
    # Also create a logging.cfg file with values reflecting verbosity and
    # stderr propagation.  Enable it only if necessary.
    fd, logging_cfg = tempfile.mkstemp(suffix='.cfg')
    os.close(fd)
    enable_logging_cfg = False
    with open(logging_cfg, 'w') as fp:
        print >> fp, '[*]'
        if parser.options.stderr:
            print >> fp, 'propagate = True'
            enable_logging_cfg = True
        if parser.options.verbosity > 2:
            print >> fp, 'level = DEBUG'
            enable_logging_cfg = True

    cfg_in = pkg_resources.resource_string(
        'mailman.testing', 'testing.cfg.in')
    fd, cfg_out = tempfile.mkstemp(suffix='.cfg')
    os.close(fd)
    with open(cfg_out, 'w') as fp:
        fp.write(cfg_in)
        if enable_logging_cfg:
            print >> fp, 'LOG_CONFIG_FILE = "%s"' % logging_cfg

    # Calculate a temporary VAR_DIR directory so that run-time artifacts of
    # the tests won't tread on the installation's data.  This also makes it
    # easier to clean up after the tests are done, and insures isolation of
    # test suite runs.
    var_dir = tempfile.mkdtemp()
    if parser.options.verbosity > 2:
        print 'VAR_DIR :', var_dir
        print 'config file:', cfg_out
        if enable_logging_cfg:
            print 'logging config file:', logging_cfg

    user_id = os.getuid()
    user_name = pwd.getpwuid(user_id).pw_name
    group_id = os.getgid()
    group_name = grp.getgrgid(group_id).gr_name

    try:
        with open(cfg_out, 'a') as fp:
            print >> fp, 'VAR_DIR = "%s"' % var_dir
            print >> fp, 'MAILMAN_USER = "%s"' % user_name
            print >> fp, 'MAILMAN_UID =', user_id
            print >> fp, 'MAILMAN_GROUP = "%s"' % group_name
            print >> fp, 'MAILMAN_GID =', group_id
            print >> fp, "LANGUAGES = 'en'"
            print >> fp, 'SMTPPORT =', SMTPServer.port

        initialize_1(cfg_out, propagate_logs=parser.options.stderr)
        mailman_uid = pwd.getpwnam(config.MAILMAN_USER).pw_uid
        mailman_gid = grp.getgrnam(config.MAILMAN_GROUP).gr_gid
        os.chmod(cfg_out, 0660)
        os.chown(cfg_out, mailman_uid, mailman_gid)

        # Create an empty SQLite database file with the proper permissions and
        # calculate the SQLAlchemy engine url to this database file.
        fd, config.dbfile = tempfile.mkstemp(dir=config.DATA_DIR, suffix='.db')
        os.close(fd)
        os.chmod(config.dbfile, 0660)
        os.chown(config.dbfile, mailman_uid, mailman_gid)

        # Patch ups.
        test_engine_url = 'sqlite:///' + config.dbfile
        config.DEFAULT_DATABASE_URL = test_engine_url

        # Write this to the config file so subprocesses share the same testing
        # database file.
        with open(cfg_out, 'a') as fp:
            print >> fp, 'DEFAULT_DATABASE_URL = "%s"' % test_engine_url

        # With -vvv, turn on engine debugging.
        initialize_2(parser.options.verbosity > 3)

        # Run the tests.  XXX I'm not sure if basedir can be converted to
        # pkg_resources.
        import mailman
        basedir = os.path.dirname(mailman.__file__)
        runner = unittest.TextTestRunner(verbosity=parser.options.verbosity)
        results = runner.run(suite(parser.arguments, parser.options.randomize))
    finally:
        os.remove(cfg_out)
        os.remove(logging_cfg)
        shutil.rmtree(var_dir)

    # Turn off coverage and print a report
    if parser.options.coverage:
        coverage.stop()
        modules = [module for name, module in sys.modules.items()
                   if module
                   and name is not None
                   and name.split('.')[0] == 'mailman']
        coverage.report(modules)
    sys.exit(bool(results.failures or results.errors))



if __name__ == '__main__':
    main()