diff options
Diffstat (limited to 'src')
59 files changed, 605 insertions, 487 deletions
diff --git a/src/mailman/__init__.py b/src/mailman/__init__.py index 8690e77bd..87a2a2100 100644 --- a/src/mailman/__init__.py +++ b/src/mailman/__init__.py @@ -44,7 +44,7 @@ except ImportError: # # Do *not* do this if we're building the documentation. if 'build_sphinx' not in sys.argv: - if sys.argv[0].split(os.sep)[-1] == 'test': + if any('nose2' in arg for arg in sys.argv): from mailman.testing.i18n import initialize else: from mailman.core.i18n import initialize diff --git a/src/mailman/app/docs/hooks.rst b/src/mailman/app/docs/hooks.rst index 7e214f13f..a29c7ee10 100644 --- a/src/mailman/app/docs/hooks.rst +++ b/src/mailman/app/docs/hooks.rst @@ -52,8 +52,9 @@ script that will produce no output to force the hooks to run. >>> import subprocess >>> from mailman.testing.layers import ConfigLayer >>> def call(): + ... exe = os.path.join(os.path.dirname(sys.executable), 'mailman') ... proc = subprocess.Popen( - ... 'bin/mailman lists --domain ignore -q'.split(), + ... [exe, 'lists', '--domain', 'ignore', '-q'], ... cwd=ConfigLayer.root_directory, ... env=dict(MAILMAN_CONFIG_FILE=config_path, ... PYTHONPATH=config_directory), diff --git a/src/mailman/app/moderator.py b/src/mailman/app/moderator.py index d8603366a..59fcb0976 100644 --- a/src/mailman/app/moderator.py +++ b/src/mailman/app/moderator.py @@ -246,7 +246,7 @@ def handle_subscription(mlist, id, action, comment=None): lang=getUtility(ILanguageManager)[data['language']]) elif action is Action.accept: key, data = requestdb.get_request(id) - delivery_mode = DeliveryMode(data['delivery_mode']) + delivery_mode = DeliveryMode[data['delivery_mode']] address = data['address'] display_name = data['display_name'] language = getUtility(ILanguageManager)[data['language']] diff --git a/src/mailman/app/subscriptions.py b/src/mailman/app/subscriptions.py index 23e09ccc7..0995202b6 100644 --- a/src/mailman/app/subscriptions.py +++ b/src/mailman/app/subscriptions.py @@ -54,7 +54,7 @@ def _membership_sort_key(member): The members are sorted first by unique list id, then by subscribed email address, then by role. """ - return (member.list_id, member.address.email, int(member.role)) + return (member.list_id, member.address.email, member.role.value) diff --git a/src/mailman/archiving/docs/__init__.py b/src/mailman/archiving/docs/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/mailman/archiving/docs/__init__.py diff --git a/src/mailman/bin/master.py b/src/mailman/bin/master.py index 9e1bbd6c5..65737ad75 100644 --- a/src/mailman/bin/master.py +++ b/src/mailman/bin/master.py @@ -34,7 +34,7 @@ import socket import logging from datetime import timedelta -from flufl.enum import Enum +from enum import Enum from flufl.lock import Lock, NotLockedError, TimeOutError from lazr.config import as_boolean diff --git a/src/mailman/bin/runner.py b/src/mailman/bin/runner.py index 47133bff7..d6a1dd6db 100644 --- a/src/mailman/bin/runner.py +++ b/src/mailman/bin/runner.py @@ -29,115 +29,37 @@ import os import sys import signal import logging +import argparse import traceback from mailman.config import config from mailman.core.i18n import _ -from mailman.core.logging import reopen -from mailman.options import Options +from mailman.core.initialize import initialize from mailman.utilities.modules import find_name +from mailman.version import MAILMAN_VERSION_FULL log = None -def r_callback(option, opt, value, parser): +class ROptionAction(argparse.Action): """Callback for -r/--runner option.""" - dest = getattr(parser.values, option.dest) - parts = value.split(':') - if len(parts) == 1: - runner = parts[0] - rslice = rrange = 1 - elif len(parts) == 3: - runner = parts[0] - try: - rslice = int(parts[1]) - rrange = int(parts[2]) - except ValueError: - parser.print_help() - print >> sys.stderr, _('Bad runner specification: $value') - sys.exit(1) - else: - parser.print_help() - print >> sys.stderr, _('Bad runner specification: $value') - sys.exit(1) - dest.append((runner, rslice, rrange)) - - - -class ScriptOptions(Options): - """Options for bin/runner.""" - usage = _("""\ -Start one or more runners. - -The runner named on the command line is started, and it can either run through -its main loop once (for those runners that support this) or continuously. The -latter is how the master runner starts all its subprocesses. - -When more than one runner is specified on the command line, they are each run -in round-robin fashion. All runners must support running its main loop once. -In other words, the first named runner is run once. When that runner is done, -the next one is run to consume all the files in *its* directory, and so on. -The number of total iterations can be given on the command line. This mode of -operation is primarily for debugging purposes. - -Usage: %prog [options] - --r is required unless -l or -h is given, and its argument must be one of the -names displayed by the -l switch. - -Normally, this script should be started from 'bin/mailman start'. Running it -separately or with -o is generally useful only for debugging. When run this -way, the environment variable $MAILMAN_UNDER_MASTER_CONTROL will be set which -subtly changes some error handling behavior. -""") - - def add_options(self): - """See `Options`.""" - self.parser.add_option( - '-r', '--runner', - metavar='runner[:slice:range]', dest='runners', - type='string', default=[], - action='callback', callback=r_callback, - help=_("""\ -Start the named runner, which must be one of the strings returned by the -l -option. - -For runners that manage a queue directory, optional slice:range if given, is -used to assign multiple runner processes to that queue. range is the total -number of runners for the queue while slice is the number of this runner from -[0..range). For runners that do not manage a queue, slice and range are -ignored. - -When using the slice:range form, you must ensure that each runner for the -queue is given the same range value. If slice:runner is not given, then 1:1 -is used. - -Multiple -r options may be given, in which case each runner will run once in -round-robin fashion. The special runner 'All' is shorthand for running all -named runners listed by the -l option.""")) - self.parser.add_option( - '-o', '--once', - default=False, action='store_true', help=_("""\ -Run each named runner exactly once through its main loop. Otherwise, each -runner runs indefinitely, until the process receives signal. This is not -compatible with runners that cannot be run once.""")) - self.parser.add_option( - '-l', '--list', - default=False, action='store_true', - help=_('List the available runner names and exit.')) - self.parser.add_option( - '-v', '--verbose', - default=0, action='count', help=_("""\ -Display more debugging information to the log file.""")) - - def sanity_check(self): - """See `Options`.""" - if self.arguments: - self.parser.error(_('Unexpected arguments')) - if not self.options.runners and not self.options.list: - self.parser.error(_('No runner name given.')) + def __call__(self, parser, namespace, values, option_string=None): + parts = values.split(':') + if len(parts) == 1: + runner = parts[0] + rslice = rrange = 1 + elif len(parts) == 3: + runner = parts[0] + try: + rslice = int(parts[1]) + rrange = int(parts[2]) + except ValueError: + parser.error(_('Bad runner specification: $value')) + else: + parser.error(_('Bad runner specification: $value')) + setattr(namespace, self.dest, (runner, rslice, rrange)) @@ -175,53 +97,90 @@ def make_runner(name, slice, range, once=False): -def set_signals(loop): - """Set up the signal handlers. +def main(): + global log - Signals caught are: SIGTERM, SIGINT, SIGUSR1 and SIGHUP. The latter is - used to re-open the log files. SIGTERM and SIGINT are treated exactly the - same -- they cause the runner to exit with no restart from the master. - SIGUSR1 also causes the runner to exit, but the master watcher will - restart it in that case. + parser = argparse.ArgumentParser( + description=_("""\ + Start a runner - :param loop: A runner instance. - :type loop: `IRunner` - """ - def sigterm_handler(signum, frame): - # Exit the runner cleanly - loop.stop() - loop.status = signal.SIGTERM - log.info('%s runner caught SIGTERM. Stopping.', loop.name()) - signal.signal(signal.SIGTERM, sigterm_handler) - def sigint_handler(signum, frame): - # Exit the runner cleanly - loop.stop() - loop.status = signal.SIGINT - log.info('%s runner caught SIGINT. Stopping.', loop.name()) - signal.signal(signal.SIGINT, sigint_handler) - def sigusr1_handler(signum, frame): - # Exit the runner cleanly - loop.stop() - loop.status = signal.SIGUSR1 - log.info('%s runner caught SIGUSR1. Stopping.', loop.name()) - signal.signal(signal.SIGUSR1, sigusr1_handler) - # SIGHUP just tells us to rotate our log files. - def sighup_handler(signum, frame): - reopen() - log.info('%s runner caught SIGHUP. Reopening logs.', loop.name()) - signal.signal(signal.SIGHUP, sighup_handler) + The runner named on the command line is started, and it can + either run through its main loop once (for those runners that + support this) or continuously. The latter is how the master + runner starts all its subprocesses. + -r is required unless -l or -h is given, and its argument must + be one of the names displayed by the -l switch. - -def main(): - global log + Normally, this script should be started from 'bin/mailman + start'. Running it separately or with -o is generally useful + only for debugging. When run this way, the environment variable + $MAILMAN_UNDER_MASTER_CONTROL will be set which subtly changes + some error handling behavior. + """)) + parser.add_argument( + '--version', + action='version', version=MAILMAN_VERSION_FULL, + help=_('Print this version string and exit')) + parser.add_argument( + '-C', '--config', + help=_("""\ + Configuration file to use. If not given, the environment variable + MAILMAN_CONFIG_FILE is consulted and used if set. If neither are + given, a default configuration file is loaded.""")) + parser.add_argument( + '-r', '--runner', + metavar='runner[:slice:range]', dest='runner', + action=ROptionAction, default=None, + help=_("""\ + Start the named runner, which must be one of the strings + returned by the -l option. + + For runners that manage a queue directory, optional + `slice:range` if given is used to assign multiple runner + processes to that queue. range is the total number of runners + for the queue while slice is the number of this runner from + [0..range). For runners that do not manage a queue, slice and + range are ignored. - options = ScriptOptions() - options.initialize() + When using the `slice:range` form, you must ensure that each + runner for the queue is given the same range value. If + `slice:runner` is not given, then 1:1 is used. + """)) + parser.add_argument( + '-o', '--once', + default=False, action='store_true', help=_("""\ + Run the named runner exactly once through its main loop. + Otherwise, the runner runs indefinitely until the process + receives a signal. This is not compatible with runners that + cannot be run once.""")) + parser.add_argument( + '-l', '--list', + default=False, action='store_true', + help=_('List the available runner names and exit.')) + parser.add_argument( + '-v', '--verbose', + default=False, action='store_true', help=_("""\ + Display more debugging information to the log file.""")) + args = parser.parse_args() + if args.runner is None and not args.list: + parser.error(_('No runner name given.')) + + # Initialize the system. Honor the -C flag if given. + config_path = (None if args.config is None + else os.path.abspath(os.path.expanduser(args.config))) + initialize(config_path, args.verbose) log = logging.getLogger('mailman.runner') + if args.verbose: + console = logging.StreamHandler(sys.stderr) + formatter = logging.Formatter(config.logging.root.format, + config.logging.root.datefmt) + console.setFormatter(formatter) + logging.getLogger().addHandler(console) + logging.getLogger().setLevel(logging.DEBUG) - if options.options.list: + if args.list: descriptions = {} for section in config.runner_configs: ignore, dot, shortname = section.name.rpartition('.') @@ -234,56 +193,10 @@ def main(): print _('$name runs $classname') sys.exit(0) - # Fast track for one infinite runner. - if len(options.options.runners) == 1 and not options.options.once: - runner = make_runner(*options.options.runners[0]) - class Loop: - status = 0 - def __init__(self, runner): - self._runner = runner - def name(self): - return self._runner.__class__.__name__ - def stop(self): - self._runner.stop() - loop = Loop(runner) - if runner.intercept_signals: - set_signals(loop) - # Now start up the main loop - log.info('%s runner started.', loop.name()) - runner.run() - log.info('%s runner exiting.', loop.name()) - else: - # Anything else we have to handle a bit more specially. - runners = [] - for runner, rslice, rrange in options.options.runners: - runner = make_runner(runner, rslice, rrange, once=True) - runners.append(runner) - # This class is used to manage the main loop. - class Loop: - status = 0 - def __init__(self): - self._isdone = False - def name(self): - return 'Main loop' - def stop(self): - self._isdone = True - def isdone(self): - return self._isdone - loop = Loop() - if runner.intercept_signals: - set_signals(loop) - log.info('Main runner loop started.') - while not loop.isdone(): - for runner in runners: - # In case the SIGTERM came in the middle of this iteration. - if loop.isdone(): - break - if options.options.verbose: - log.info('Now doing a %s runner iteration', - runner.__class__.__bases__[0].__name__) - runner.run() - if options.options.once: - break - log.info('Main runner loop exiting.') - # All done - sys.exit(loop.status) + runner = make_runner(*args.runner, once=args.once) + runner.set_signals() + # Now start up the main loop + log.info('%s runner started.', runner.name) + runner.run() + log.info('%s runner exiting.', runner.name) + sys.exit(runner.status) diff --git a/src/mailman/commands/cli_conf.py b/src/mailman/commands/cli_conf.py index 3c3143b1a..dfa741e3a 100644 --- a/src/mailman/commands/cli_conf.py +++ b/src/mailman/commands/cli_conf.py @@ -66,16 +66,16 @@ class Conf: displayed. """)) command_parser.add_argument( - '-x', '--sort', + '-t', '--sort', default=False, action='store_true', - help=_("Sort the output by sections and keys.")) + help=_('Sort the output by sections and keys.')) def _get_value(self, section, key): return getattr(getattr(config, section), key) - def _sections(self, to_sort): + def _sections(self, sort_p): sections = config.schema._section_schemas - if to_sort: + if sort_p: sections = sorted(sections) return sections @@ -88,9 +88,9 @@ class Conf: def _show_section_error(self, section): self.parser.error('No such section: {}'.format(section)) - def _print_values_for_section(self, section, output, to_sort): + def _print_values_for_section(self, section, output, sort_p): current_section = getattr(config, section) - if to_sort: + if sort_p: current_section = sorted(current_section) for key in current_section: self._print_full_syntax(section, key, @@ -106,7 +106,7 @@ class Conf: # Process the command, ignoring the closing of the output file. section = args.section key = args.key - to_sort = args.sort + sort_p = args.sort # Case 1: Both section and key are given, so we can directly look up # the value. if section is not None and key is not None: @@ -119,12 +119,12 @@ class Conf: # Case 2: Section is given, key is not given. elif section is not None and key is None: if self._section_exists(section): - self._print_values_for_section(section, output, to_sort) + self._print_values_for_section(section, output, sort_p) else: self._show_section_error(section) # Case 3: Section is not given, key is given. elif section is None and key is not None: - for current_section in self._sections(to_sort): + for current_section in self._sections(sort_p): # We have to ensure that the current section actually exists # and that it contains the given key. if (self._section_exists(current_section) and @@ -137,12 +137,13 @@ class Conf: # Case 4: Neither section nor key are given, just display all the # sections and their corresponding key/value pairs. elif section is None and key is None: - for current_section in self._sections(to_sort): + for current_section in self._sections(sort_p): # However, we have to make sure that the current sections and # key which are being looked up actually exist before trying # to print them. if self._section_exists(current_section): - self._print_values_for_section(current_section, output, to_sort) + self._print_values_for_section( + current_section, output, sort_p) def process(self, args): """See `ICLISubCommand`.""" diff --git a/src/mailman/commands/cli_members.py b/src/mailman/commands/cli_members.py index 7e4b5ec88..37df55cd4 100644 --- a/src/mailman/commands/cli_members.py +++ b/src/mailman/commands/cli_members.py @@ -130,7 +130,7 @@ class Members: DeliveryMode.mime_digests, DeliveryMode.summary_digests] elif args.digest is not None: - digest_types = [DeliveryMode(args.digest + '_digests')] + digest_types = [DeliveryMode[args.digest + '_digests']] else: # Don't filter on digest type. pass @@ -140,7 +140,7 @@ class Members: elif args.nomail == 'byadmin': status_types = [DeliveryStatus.by_moderator] elif args.nomail.startswith('by'): - status_types = [DeliveryStatus('by_' + args.nomail[2:])] + status_types = [DeliveryStatus['by_' + args.nomail[2:]]] elif args.nomail == 'enabled': status_types = [DeliveryStatus.enabled] elif args.nomail == 'unknown': diff --git a/src/mailman/commands/cli_status.py b/src/mailman/commands/cli_status.py index 1f4cc552a..8a5cf9bde 100644 --- a/src/mailman/commands/cli_status.py +++ b/src/mailman/commands/cli_status.py @@ -64,4 +64,4 @@ class Status: message = _('GNU Mailman is in an unexpected state ' '($hostname != $fqdn_name)') print(message) - return int(status) + return status.value diff --git a/src/mailman/commands/docs/__init__.py b/src/mailman/commands/docs/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/mailman/commands/docs/__init__.py diff --git a/src/mailman/commands/docs/conf.rst b/src/mailman/commands/docs/conf.rst index e3a8a488f..7b8529ac3 100644 --- a/src/mailman/commands/docs/conf.rst +++ b/src/mailman/commands/docs/conf.rst @@ -65,12 +65,16 @@ If you specify both a section and a key, you will get the corresponding value. >>> command.process(FakeArgs) noreply@example.com -You can also sort the output. The output will be first sorted by section, then by key. +You can also sort the output. The output is first sorted by section, then by +key. + >>> FakeArgs.key = None >>> FakeArgs.section = 'shell' >>> FakeArgs.sort = True + >>> command.process(FakeArgs) [shell] banner: Welcome to the GNU Mailman shell [shell] prompt: >>> [shell] use_ipython: no + .. _`Postfix command postconf(1)`: http://www.postfix.org/postconf.1.html diff --git a/src/mailman/commands/tests/test_conf.py b/src/mailman/commands/tests/test_conf.py index 758f26d56..307151c74 100644 --- a/src/mailman/commands/tests/test_conf.py +++ b/src/mailman/commands/tests/test_conf.py @@ -31,6 +31,7 @@ import mock import tempfile import unittest +from StringIO import StringIO from mailman.commands.cli_conf import Conf from mailman.testing.layers import ConfigLayer @@ -100,3 +101,13 @@ class TestConf(unittest.TestCase): finally: os.remove(filename) self.assertEqual(contents, 'no\n') + + def test_sort_by_section(self): + self.args.output = '-' + self.args.sort = True + output = StringIO() + with mock.patch('sys.stdout', output): + self.command.process(self.args) + last_line = '' + for line in output.getvalue().splitlines(): + self.assertTrue(line > last_line) diff --git a/src/mailman/core/runner.py b/src/mailman/core/runner.py index a8412195d..347726b85 100644 --- a/src/mailman/core/runner.py +++ b/src/mailman/core/runner.py @@ -26,6 +26,7 @@ __all__ = [ import time +import signal import logging import traceback @@ -37,6 +38,7 @@ from zope.interface import implementer from mailman.config import config from mailman.core.i18n import _ +from mailman.core.logging import reopen from mailman.core.switchboard import Switchboard from mailman.interfaces.languages import ILanguageManager from mailman.interfaces.listmanager import IListManager @@ -46,12 +48,12 @@ from mailman.utilities.string import expand dlog = logging.getLogger('mailman.debug') elog = logging.getLogger('mailman.error') +rlog = logging.getLogger('mailman.runner') @implementer(IRunner) class Runner: - intercept_signals = True is_queue_runner = True def __init__(self, name, slice=None): @@ -85,10 +87,32 @@ class Runner: self.max_restarts = int(section.max_restarts) self.start = as_boolean(section.start) self._stop = False + self.status = 0 def __repr__(self): return '<{0} at {1:#x}>'.format(self.__class__.__name__, id(self)) + def signal_handler(self, signum, frame): + signame = { + signal.SIGTERM: 'SIGTERM', + signal.SIGINT: 'SIGINT', + signal.SIGUSR1: 'SIGUSR1', + }.get(signum, signum) + if signum in (signal.SIGTERM, signal.SIGINT, signal.SIGUSR1): + self.stop() + self.status = signum + rlog.info('%s runner caught %s. Stopping.', self.name, signame) + elif signum == signal.SIGHUP: + reopen() + rlog.info('%s runner caught SIGHUP. Reopening logs.', self.name) + + def set_signals(self): + """See `IRunner`.""" + signal.signal(signal.SIGHUP, self.signal_handler) + signal.signal(signal.SIGINT, self.signal_handler) + signal.signal(signal.SIGTERM, self.signal_handler) + signal.signal(signal.SIGUSR1, self.signal_handler) + def stop(self): """See `IRunner`.""" self._stop = True diff --git a/src/mailman/database/schema/mm_20120407000000.py b/src/mailman/database/schema/mm_20120407000000.py index 8855df5af..0b1e51386 100644 --- a/src/mailman/database/schema/mm_20120407000000.py +++ b/src/mailman/database/schema/mm_20120407000000.py @@ -67,11 +67,11 @@ def upgrade(database, store, version, module_path): def archive_policy(archive, archive_private): """Convert archive and archive_private to archive_policy.""" if archive == 0: - return int(ArchivePolicy.never) + return ArchivePolicy.never.value elif archive_private == 1: - return int(ArchivePolicy.private) + return ArchivePolicy.private.value else: - return int(ArchivePolicy.public) + return ArchivePolicy.public.value diff --git a/src/mailman/database/types.py b/src/mailman/database/types.py index 42f64a640..86d504ca3 100644 --- a/src/mailman/database/types.py +++ b/src/mailman/database/types.py @@ -32,7 +32,7 @@ from storm.variables import Variable class _EnumVariable(Variable): - """Storm variable for supporting flufl.enum.Enum types. + """Storm variable for supporting enum types. To use this, make the database column a INTEGER. """ @@ -46,18 +46,18 @@ class _EnumVariable(Variable): return None if not from_db: return value - return self._enum[value] + return self._enum(value) def parse_get(self, value, to_db): if value is None: return None if not to_db: return value - return int(value) + return value.value class Enum(SimpleProperty): - """Custom Enum type for Storm supporting flufl.enum.Enums.""" + """Custom type for Storm supporting enums.""" variable_class = _EnumVariable diff --git a/src/mailman/docs/8-miles-high.rst b/src/mailman/docs/8-miles-high.rst index d068ebd73..85b186fc5 100644 --- a/src/mailman/docs/8-miles-high.rst +++ b/src/mailman/docs/8-miles-high.rst @@ -37,10 +37,12 @@ processing queues. node [shape=box, color=lightblue, style=filled]; msg [shape=ellipse, color=black, fillcolor=white]; lmtpd [label="LMTP\nSERVER"]; + rts [label="Return\nto Sender"]; msg -> MTA [label="SMTP"]; MTA -> lmtpd [label="LMTP"]; lmtpd -> MTA [label="reject"]; lmtpd -> IN -> PIPELINE [label=".pck"]; + IN -> rts; lmtpd -> BOUNCES [label=".pck"]; lmtpd -> COMMAND [label=".pck"]; } @@ -49,7 +51,9 @@ The `in` queue is processed by *filter chains* (explained below) to determine whether the post (or administrative request) will be processed. If not allowed, the message pickle is discarded, rejected (returned to sender), or held (saved for moderator approval -- not shown). Otherwise the message is -added to the `pipeline` (i.e. posting) queue. +added to the `pipeline` (i.e. posting) queue. (Note that rejecting at this +stage is *not* equivalent to rejecting during LMTP processing. This issue is +currently unresolved.) Each of the `command`, `bounce`, and `pipeline` queues is processed by a *pipeline of handlers* as in Mailman 2's pipeline. (Some functions such as @@ -60,7 +64,8 @@ Handlers may copy messages to other queues (*e.g.*, `archive`), and eventually posted messages for distribution to the list membership end up in the `out` queue for injection into the MTA. -The `virgin` queue is a special queue for messages created by Mailman. +The `virgin` queue (not depicted above) is a special queue for messages created +by Mailman. .. graphviz:: @@ -97,12 +102,13 @@ The default set of rules looks something like this: subgraph rules { rankdir=TB; node [shape=record]; - approved [label="<in> approved | { <no> | <yes> }"]; - emergency [label="<in> emergency | { <no> | <yes> }"]; - loop [label="<in> loop | { <no> | <yes> }"]; - modmember [label="<in> member\nmoderated | { <no> | <yes> }"]; - administrivia [group="0", label="<in> administrivia | <always> "]; - maxsize [label="<in> max\ size | {<in> no | <yes>}"]; + approved [label="<in> approved | { <no> no | <yes> }"]; + emergency [label="<in> emergency | { <no> no | <yes> }"]; + loop [label="<in> loop | { <no> no | <yes> }"]; + modmember [label="<in> member\nmoderated | { <no> no | <yes> }"]; + administrivia [group="0", + label="<in> administrivia | { <no> no | <yes> }"]; + maxsize [label="<in> max\ size | {<no> no | <yes>}"]; any [label="<in> any | {<no> | <yes>}"]; truth [label="<in> truth | <always>"]; @@ -114,6 +120,7 @@ The default set of rules looks something like this: DISCARD [shape=invhouse, color=black, style=solid]; MODERATION [color=wheat]; HOLD [color=wheat]; + action [color=wheat]; } { PIPELINE [shape=box, style=filled, color=cyan]; } @@ -130,7 +137,8 @@ The default set of rules looks something like this: modmember:no -> administrivia:in; modmember:yes -> MODERATION; - administrivia:always -> maxsize:in; + administrivia:no -> maxsize:in; + administrivia:yes -> action; maxsize:no -> any:in; maxsize:yes -> MODERATION; @@ -145,7 +153,7 @@ The default set of rules looks something like this: Configuration ============= -Uses `lazr.config`_, essentially an "ini"-style configuration format. +Mailman 3 uses `lazr.config`_, essentially an "ini"-style configuration format. Each Runner's configuration object knows whether it should be started when the Mailman daemon starts, and what queue the Runner manages. diff --git a/src/mailman/docs/INTRODUCTION.rst b/src/mailman/docs/INTRODUCTION.rst index 1ad71cf16..e7128e756 100644 --- a/src/mailman/docs/INTRODUCTION.rst +++ b/src/mailman/docs/INTRODUCTION.rst @@ -82,7 +82,7 @@ lists and archives, etc., are available at: Requirements ============ -Mailman 3.0 requires `Python 2.7`_ or newer. +Mailman 3.0 requires `Python 2.7`_. .. _`GNU Mailman`: http://www.list.org diff --git a/src/mailman/docs/MTA.rst b/src/mailman/docs/MTA.rst index aa53981f8..7165c30b9 100644 --- a/src/mailman/docs/MTA.rst +++ b/src/mailman/docs/MTA.rst @@ -2,32 +2,73 @@ Hooking up your mail server =========================== -Mailman needs to be hooked up to your mail server (a.k.a. *mail transport -agent* or *MTA*) both to accept incoming mail and to deliver outgoing mail. -Mailman itself never delivers messages to the end user; it lets its immediate -upstream mail server do that. +Mailman needs to communicate with your *MTA* (*mail transport agent* +or *mail server*, the software which handles sending mail across the +Internet), both to accept incoming mail and to deliver outgoing mail. +Mailman itself never delivers messages to the end user. It sends them +to its immediate upstream MTA, which delivers them. In the same way, +Mailman never receives mail directly. Mail from outside always comes +via the MTA. -The preferred way to allow Mailman to accept incoming messages from your mail -server is to use the `Local Mail Transfer Protocol`_ (LMTP_) interface. Most -open source mail server support LMTP for local delivery, and this is much more -efficient than spawning a process just to do the delivery. +Mailman accepts incoming messages from the MTA using the `Local Mail +Transfer Protocol`_ (LMTP_) interface. Mailman can use other incoming +transports, but LMTP is much more efficient than spawning a process +just to do the delivery. Most open source MTAs support LMTP for local +delivery. If yours doesn't, and you need to use a different +interface, please ask on the `mailing list or on IRC`_. -Your mail server should also accept `Simple Mail Transfer Protocol`_ (SMTP_) -connections from Mailman, for all outgoing messages. +Mailman passes all outgoing messages to the MTA using the `Simple Mail +Transfer Protocol`_ (SMTP_). -The specific instructions for hooking your mail server up to Mailman differs -depending on which mail server you're using. The following are instructions -for the popular open source mail servers. +Cooperation between Mailman and the MTA requires some configuration of +both. MTA configuration differs for each of the available MTAs, and +there is a section for each one. Instructions for Postfix are given +below. We would really appreciate contributions of configurations for +Exim and Sendmail, and welcome information about other popular open +source mail servers. -Note that Mailman provides lots of configuration variables that you can use to -tweak performance for your operating environment. See the -``src/mailman/config/schema.cfg`` file for details. +Configuring Mailman to communicate with the MTA is straightforward, +and basically the same for all MTAs. In your ``mailman.cfg`` file, +add (or edit) a section like the following:: + + [mta] + incoming: mailman.mta.postfix.LMTP + outgoing: mailman.mta.deliver.deliver + lmtp_host: 127.0.0.1 + lmtp_port: 8024 + smtp_host: localhost + smtp_port: 25 +This configuration is for a system where Mailman and the MTA are on +the same host. -Exim -==== +The ``incoming`` and ``outgoing`` parameters identify the Python +objects used to communicate with the MTA. The ``deliver`` module used +in ``outgoing`` is pretty standard across all MTAs. The ``postfix`` +module in ``incoming`` is specific to Postfix. See the section for +your MTA below for details on these parameters. -Contributions are welcome! +``lmtp_host`` and ``lmtp_port`` are parameters which are used by +Mailman, but also will be passed to the MTA to identify the Mailman +host. The "same host" case is special; some MTAs (including Postfix) +do not recognize "localhost", and need the numerical IP address. If +they are on different hosts, ``lmtp_host`` should be set to the domain +name or IP address of the Mailman host. ``lmtp_port`` is fairly +arbitrary (there is no standard port for LMTP). Use any port +convenient for your site. "8024" is as good as any, unless another +service is using it. + +``smtp_host`` and ``smtp_port`` are parameters used to identify the +MTA to Mailman. If the MTA and Mailman are on separate hosts, +``smtp_host`` should be set to the domain name or IP address of the +MTA host. ``smtp_port`` will almost always be 25, which is the +standard port for SMTP. (Some special site configurations set it to a +different port. If you need this, you probably already know that, +know why, and what to do, too!) + +Mailman also provides many other configuration variables that you can +use to tweak performance for your operating environment. See the +``src/mailman/config/schema.cfg`` file for details. Postfix @@ -131,12 +172,19 @@ the Postfix documentation at: .. _`mydestination`: http://www.postfix.org/postconf.5.html#mydestination +Exim +==== + +Contributions are welcome! + + Sendmail ======== Contributions are welcome! +.. _`mailing list or on IRC`: START.html#contact-us .. _`Local Mail Transfer Protocol`: http://en.wikipedia.org/wiki/Local_Mail_Transfer_Protocol .. _LMTP: http://www.faqs.org/rfcs/rfc2033.html diff --git a/src/mailman/docs/NEWS.rst b/src/mailman/docs/NEWS.rst index 65b8cb05e..6572c9a7b 100644 --- a/src/mailman/docs/NEWS.rst +++ b/src/mailman/docs/NEWS.rst @@ -12,12 +12,12 @@ Here is a history of user visible changes to Mailman. =============================== (2013-XX-XX) -Bugs ----- - * Non-queue runners should not create ``var/queue`` subdirectories. Fixed by - Sandesh Kumar Agrawal. (LP: #1095422) - * Creation of lists with upper case names should be coerced to lower case. - (LP: #1117176) +Development +----------- + * Mailman 3 no longer uses ``zc.buildout`` and tests are now run by the + ``nose2`` test runner. See ``src/mailman/docs/START.rst`` for details on + how to build Mailman and run the test suite. + * Use the ``enum34`` package instead of ``flufl.enum``. REST ---- @@ -25,6 +25,14 @@ REST attributes of a mailing list's configuration. (LP: #1157881) * Support pagination of some large collections (lists, users, members). Given by Florian Fuchs. (LP: #1156529) + * Expose ``hide_address`` to the ``.../preferences`` REST API. Contributed + by Sneha Priscilla. + +Commands +-------- + * `mailman conf` now has a `-t/--sort` flag which sorts the output by section + and then key. Contributed by Karl-Aksel Puulmann and David Soto + (LP: 1162492) Configuration ------------- @@ -34,6 +42,17 @@ Configuration the configuration file, just after ``./mailman.cfg`` and before ``~/.mailman.cfg``. (LP: #1157861) +Bugs +---- + * Non-queue runners should not create ``var/queue`` subdirectories. Fixed by + Sandesh Kumar Agrawal. (LP: #1095422) + * Creation of lists with upper case names should be coerced to lower case. + (LP: #1117176) + * Fix REST server crash on `mailman reopen` due to no interception of + signals. (LP: #1184376) + * Add `subject_prefix` to the `IMailingList` interface, and clarify the + docstring for `display_name`. (LP: #1181498) + 3.0 beta 3 -- "Here Again" ========================== @@ -126,6 +145,9 @@ Commands the Postfix `relay_domains` support. * `bin/mailman start` was passing the wrong relative path to its runner subprocesses when -C was given. (LP: #982551) + * `bin/runner` command has been simplified and its command line options + reduced. Now, only one `-r/--runner` option may be provided and the + round-robin feature has been removed. Other ----- @@ -140,6 +162,8 @@ Other Bugs ---- * Fixed `send_goodbye_message()`. (LP: #1091321) + * Fixed REST server crash on `reopen` command. Identification and test + provided by Aurélien Bompard. (LP: #1184376) 3.0 beta 2 -- "Freeze" diff --git a/src/mailman/docs/START.rst b/src/mailman/docs/START.rst index 85b869c2d..a50deabf9 100644 --- a/src/mailman/docs/START.rst +++ b/src/mailman/docs/START.rst @@ -25,58 +25,64 @@ search, and retrieval of archived messages to a separate application (a simple implementation is provided). The web interface (known as `Postorius`_) and archiver (known as `Hyperkitty`_) are in separate development. -Contributions are welcome. Please submit bug reports on the Mailman bug -tracker at https://bugs.launchpad.net/mailman though you will currently need -to have a login on Launchpad to do so. You can also send email to the -mailman-developers@python.org mailing list. + +Contact Us +========== + +Contributions of code, problem reports, and feature requests are welcome. +Please submit bug reports on the Mailman bug tracker at +https://bugs.launchpad.net/mailman (you need to have a login on Launchpad to +do so). You can also send email to the mailman-developers@python.org mailing +list, or ask on IRC channel ``#mailman`` on Freenode. Requirements ============ Python 2.7 is required. It can either be the default 'python' on your -``$PATH`` or it can be accessible via the ``python2.7`` binary. -If your operating system does not include Python, see http://www.python.org -downloading and installing it from source. Python 3 is not yet supported. - -In this documentation, a bare ``python`` refers to the Python executable used -to invoke ``bootstrap.py``. +``$PATH`` or it can be accessible via the ``python2.7`` binary. If +your operating system does not include Python, see http://www.python.org +for information about downloading installers (where available) and +installing it from source (when necessary or preferred). Python 3 is +not yet supported. -Mailman 3 is now based on the `zc.buildout`_ infrastructure, which greatly -simplifies testing Mailman. Buildout is not required for installation. - -During the beta program, you may need some additional dependencies, such as a -C compiler and the Python development headers and libraries. You will need an -internet connection. +You may need some additional dependencies, which are either available from +your OS vendor, or can be downloaded automatically from the `Python +Cheeseshop`_. Building Mailman 3 ================== -We provide several recipes for building Mailman. All should generally work, -but some may provide a better experience for developing Mailman versus -deploying Mailman. - +To build Mailman for development purposes, you will create a virtual +environment. You need to have the `virtualenv`_ program installed. Building for development ------------------------ -The best way to build Mailman for development is to use the `zc.buildout`_ -tools. This will download all Mailman dependencies from the `Python -Cheeseshop`_. The dependencies will get installed locally, but isolated from -your system Python. Here are the commands to build Mailman for development:: +First, create a virtual environment. By default ``virtualenv`` uses the +``python`` executable it finds first on your ``$PATH``. Make sure this is +Python 2.7. The directory you install the virtualenv into is up to you, but +for purposes of this document, we'll install it into ``/tmp/py27``:: + + % virtualenv --system-site-packages /tmp/py27 - % python bootstrap.py - % bin/buildout +Now, activate the virtual environment and set it up for development:: + + % source /tmp/py27/bin/activate + % python setup.py develop Sit back and have some Kombucha while you wait for everything to download and install. Now you can run the test suite via:: - % bin/test -vv + % nose2 -v + +You should see no failures. You can also run a subset of the full test suite +by filter tests on the module or test name using the ``-P`` option:: -You should see no failures. + % nose2 -v -P user Build the online docs by running:: @@ -91,21 +97,6 @@ doctests by looking in all the 'doc' directories under the 'mailman' package. Doctests are documentation first, so they should give you a pretty good idea how various components of Mailman 3 work. - -Building for deployment using virtualenv ----------------------------------------- - -`virtualenv`_ is a way to create isolated Python environments. You can use -virtualenv as a way to deploy Mailman without installing it into your system -Python. There are lots of ways to use virtualenv, but as described here, it -will be default use any dependencies which are already installed in your -system, downloading from the Cheeseshop only those which are missing. Here -are the steps to install Mailman using virtualenv:: - - $ virtualenv --system-site-packages /path/to/your/installation - $ source /path/to/your/installation/bin/activate - $ python setup.py install - Once everything is downloaded and installed, you can initialize Mailman and get a display of the basic configuration settings by running:: @@ -149,8 +140,9 @@ Try ``bin/mailman --help`` for more details. You can use the commands ``bin/mailman start`` to start the runner subprocess daemons, and of course ``bin/mailman stop`` to stop them. -Postorius is being developed as a separate, Django-based project. For now, -all configuration happens via the command line and REST API. +Postorius, a web UI for administration and subscriber settings, is being +developed as a separate, Django-based project. For now, the most flexible +means of configuration is via the command line and REST API. Mailman Web UI @@ -167,16 +159,16 @@ Postorius was prototyped at the `Pycon 2012 sprint`_, so it is "very alpha" as of Mailman 3 beta 1, and comes in several components. In particular, it requires a `Django`_ installation, and Bazaar checkouts of the `REST client module`_ and `Postorius`_ itself. Building it is fairly straightforward, -however, given Florian Fuchs' `Five Minute Guide` from his `blog post`_ on the +based on Florian Fuchs' `Five Minute Guide` from his `blog post`_ on the Mailman wiki. (Check the `blog post`_ for the most recent version!) The Archiver ------------ -In Mailman 3, the archivers are decoupled from the core engine. It is useful -to provide a simple, standard interface for third-party archiving tools and -services. For this reason, Mailman 3 defines a formal interface to insert +In Mailman 3, the archivers are decoupled from the core engine. Instead, +Mailman 3 provides a simple, standard interface for third-party archiving tools +and services. For this reason, Mailman 3 defines a formal interface to insert messages into any of a number of configured archivers, using whatever protocol is appropriate for that archiver. Summary, search, and retrieval of archived posts are handled by a separate application. @@ -190,7 +182,6 @@ email application that speaks LMTP or SMTP will be able to use Hyperkitty. A `five minute guide to Hyperkitty`_ is based on Toshio Kuratomi's README. -.. _`zc.buildout`: http://pypi.python.org/pypi/zc.buildout .. _`Postorius`: https://launchpad.net/postorius .. _`Hyperkitty`: https://launchpad.net/hyperkitty .. _`Django`: http://djangoproject.org/ diff --git a/src/mailman/handlers/cook_headers.py b/src/mailman/handlers/cook_headers.py index 07235d6bd..6269f4c45 100644 --- a/src/mailman/handlers/cook_headers.py +++ b/src/mailman/handlers/cook_headers.py @@ -155,8 +155,8 @@ def process(mlist, msg, msgdata): # above code? # Also skip Cc if this is an anonymous list as list posting address # is already in From and Reply-To in this case. - if (mlist.personalize == Personalization.full and - mlist.reply_goes_to_list != ReplyToMunging.point_to_list and + if (mlist.personalize is Personalization.full and + mlist.reply_goes_to_list is not ReplyToMunging.point_to_list and not mlist.anonymous_list): # Watch out for existing Cc headers, merge, and remove dups. Note # that RFC 2822 says only zero or one Cc header is allowed. diff --git a/src/mailman/handlers/docs/__init__.py b/src/mailman/handlers/docs/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/mailman/handlers/docs/__init__.py diff --git a/src/mailman/interfaces/action.py b/src/mailman/interfaces/action.py index 6df380088..22a65d5e1 100644 --- a/src/mailman/interfaces/action.py +++ b/src/mailman/interfaces/action.py @@ -24,7 +24,7 @@ __all__ = [ ] -from flufl.enum import Enum +from enum import Enum @@ -36,6 +36,11 @@ class Action(Enum): defer = 4 -class FilterAction(Action): +class FilterAction(Enum): + hold = 0 + reject = 1 + discard = 2 + accept = 3 + defer = 4 forward = 5 preserve = 6 diff --git a/src/mailman/interfaces/archiver.py b/src/mailman/interfaces/archiver.py index 2182798ad..5f074503e 100644 --- a/src/mailman/interfaces/archiver.py +++ b/src/mailman/interfaces/archiver.py @@ -27,7 +27,7 @@ __all__ = [ ] -from flufl.enum import Enum +from enum import Enum from zope.interface import Interface, Attribute diff --git a/src/mailman/interfaces/autorespond.py b/src/mailman/interfaces/autorespond.py index 7c66d984f..acad717f1 100644 --- a/src/mailman/interfaces/autorespond.py +++ b/src/mailman/interfaces/autorespond.py @@ -30,7 +30,7 @@ __all__ = [ from datetime import timedelta -from flufl.enum import Enum +from enum import Enum from zope.interface import Interface, Attribute ALWAYS_REPLY = timedelta() diff --git a/src/mailman/interfaces/bounce.py b/src/mailman/interfaces/bounce.py index 4719553c4..8e7266687 100644 --- a/src/mailman/interfaces/bounce.py +++ b/src/mailman/interfaces/bounce.py @@ -28,7 +28,7 @@ __all__ = [ ] -from flufl.enum import Enum +from enum import Enum from zope.interface import Attribute, Interface diff --git a/src/mailman/interfaces/chain.py b/src/mailman/interfaces/chain.py index b4fd98bb2..9fffc2760 100644 --- a/src/mailman/interfaces/chain.py +++ b/src/mailman/interfaces/chain.py @@ -35,7 +35,7 @@ __all__ = [ ] -from flufl.enum import Enum +from enum import Enum from zope.interface import Interface, Attribute diff --git a/src/mailman/interfaces/command.py b/src/mailman/interfaces/command.py index 42a859f36..18a8024a8 100644 --- a/src/mailman/interfaces/command.py +++ b/src/mailman/interfaces/command.py @@ -28,7 +28,7 @@ __all__ = [ ] -from flufl.enum import Enum +from enum import Enum from zope.interface import Interface, Attribute diff --git a/src/mailman/interfaces/digests.py b/src/mailman/interfaces/digests.py index c00758f7a..416e18f13 100644 --- a/src/mailman/interfaces/digests.py +++ b/src/mailman/interfaces/digests.py @@ -25,7 +25,7 @@ __all__ = [ ] -from flufl.enum import Enum +from enum import Enum from zope.interface import Interface, Attribute diff --git a/src/mailman/interfaces/mailinglist.py b/src/mailman/interfaces/mailinglist.py index 7b409b5a1..7beaf9c46 100644 --- a/src/mailman/interfaces/mailinglist.py +++ b/src/mailman/interfaces/mailinglist.py @@ -29,7 +29,7 @@ __all__ = [ ] -from flufl.enum import Enum +from enum import Enum from zope.interface import Interface, Attribute from mailman.interfaces.member import MemberRole @@ -98,7 +98,8 @@ class IMailingList(Interface): display_name = Attribute("""\ The short human-readable descriptive name for the mailing list. This - is used in locations such as the message footers and Subject prefix. + is used in locations such as the message footers and as the default + value for the Subject prefix. """) description = Attribute("""\ @@ -108,6 +109,17 @@ class IMailingList(Interface): mailing lists, or in headers, and so forth. It should be as succinct as you can get it, while still identifying what the list is.""") + subject_prefix = Attribute("""\ + The text to insert at the front of the Subject field. + + When messages posted to this mailing list are sent to the list + subscribers, the Subject header may be rewritten to include an + identifying prefix. Typically this prefix will appear in square + brackets and the default value inside the brackets is taken as the + list's display name. However, any value can be used, including the + empty string to prevent Subject header rewriting. + """) + allow_list_posts = Attribute( """Flag specifying posts to the list are generally allowed. diff --git a/src/mailman/interfaces/member.py b/src/mailman/interfaces/member.py index ee609a0d9..b561a7571 100644 --- a/src/mailman/interfaces/member.py +++ b/src/mailman/interfaces/member.py @@ -36,7 +36,7 @@ __all__ = [ ] -from flufl.enum import Enum +from enum import Enum from zope.interface import Interface, Attribute from mailman.core.errors import MailmanError diff --git a/src/mailman/interfaces/mime.py b/src/mailman/interfaces/mime.py index 629fc776b..549e6c3db 100644 --- a/src/mailman/interfaces/mime.py +++ b/src/mailman/interfaces/mime.py @@ -27,7 +27,7 @@ __all__ = [ ] -from flufl.enum import Enum +from enum import Enum from zope.interface import Interface, Attribute diff --git a/src/mailman/interfaces/nntp.py b/src/mailman/interfaces/nntp.py index 2246fff6e..03be8f11d 100644 --- a/src/mailman/interfaces/nntp.py +++ b/src/mailman/interfaces/nntp.py @@ -25,7 +25,7 @@ __all__ = [ ] -from flufl.enum import Enum +from enum import Enum diff --git a/src/mailman/interfaces/requests.py b/src/mailman/interfaces/requests.py index 9f841a6a3..35de05100 100644 --- a/src/mailman/interfaces/requests.py +++ b/src/mailman/interfaces/requests.py @@ -30,7 +30,7 @@ __all__ = [ ] -from flufl.enum import Enum +from enum import Enum from zope.interface import Interface, Attribute diff --git a/src/mailman/interfaces/runner.py b/src/mailman/interfaces/runner.py index 1dfe7e266..a8b76652c 100644 --- a/src/mailman/interfaces/runner.py +++ b/src/mailman/interfaces/runner.py @@ -63,13 +63,16 @@ class IRunner(Interface): through the main loop. """) - intercept_signals = Attribute("""\ - Should the runner mechanism intercept signals? + def set_signals(): + """Set up the signal handlers necessary to control the runner. - In general, the runner catches SIGINT, SIGTERM, SIGUSR1, and SIGHUP to - manage the process. Some runners need to manage their own signals, - and set this attribute to False. - """) + The runner should catch the following signals: + - SIGTERM and SIGINT: treated exactly the same, they cause the runner + to exit with no restart from the master. + - SIGUSR1: Also causes the runner to exit, but the master watcher will + retart it. + - SIGHUP: Re-open the log files. + """ def _one_iteration(): """The work done in one iteration of the main loop. diff --git a/src/mailman/model/docs/autorespond.rst b/src/mailman/model/docs/autorespond.rst index 3a9ad01b2..b2bf03ed9 100644 --- a/src/mailman/model/docs/autorespond.rst +++ b/src/mailman/model/docs/autorespond.rst @@ -90,7 +90,7 @@ You can also use the response set to get the date of the last response sent. >>> response.address <Address: aperson@example.com [not verified] at ...> >>> response.response_type - <EnumValue: Response.hold [int=1]> + <Response.hold: 1> >>> response.date_sent datetime.date(2005, 8, 1) diff --git a/src/mailman/model/docs/membership.rst b/src/mailman/model/docs/membership.rst index f257f25ce..eeba9b332 100644 --- a/src/mailman/model/docs/membership.rst +++ b/src/mailman/model/docs/membership.rst @@ -217,7 +217,7 @@ There is also a roster containing all the subscribers of a mailing list, regardless of their role. >>> def sortkey(member): - ... return (member.address.email, int(member.role)) + ... return (member.address.email, member.role.value) >>> for member in sorted(mlist.subscribers.members, key=sortkey): ... print member.address.email, member.role aperson@example.com MemberRole.member diff --git a/src/mailman/model/docs/users.rst b/src/mailman/model/docs/users.rst index 997f983b2..889fe46d1 100644 --- a/src/mailman/model/docs/users.rst +++ b/src/mailman/model/docs/users.rst @@ -332,8 +332,7 @@ membership role. >>> len(members) 4 >>> def sortkey(member): - ... return (member.address.email, member.mailing_list, - ... int(member.role)) + ... return member.address.email, member.mailing_list, member.role.value >>> for member in sorted(members, key=sortkey): ... print member.address.email, member.mailing_list.list_id, \ ... member.role diff --git a/src/mailman/mta/docs/__init__.py b/src/mailman/mta/docs/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/mailman/mta/docs/__init__.py diff --git a/src/mailman/rest/addresses.py b/src/mailman/rest/addresses.py index 2b0bf5890..2908fad57 100644 --- a/src/mailman/rest/addresses.py +++ b/src/mailman/rest/addresses.py @@ -178,7 +178,7 @@ class UserAddresses(_AddressBase): def membership_key(member): # Sort first by mailing list, then by address, then by role. - return member.list_id, member.address.email, int(member.role) + return member.list_id, member.address.email, member.role.value class AddressMemberships(MemberCollection): diff --git a/src/mailman/rest/docs/preferences.rst b/src/mailman/rest/docs/preferences.rst index e82fd6001..8694364a4 100644 --- a/src/mailman/rest/docs/preferences.rst +++ b/src/mailman/rest/docs/preferences.rst @@ -95,6 +95,7 @@ PUT operation. ... 'acknowledge_posts': True, ... 'delivery_mode': 'plaintext_digests', ... 'delivery_status': 'by_user', + ... 'hide_address': False, ... 'preferred_language': 'ja', ... 'receive_list_copy': True, ... 'receive_own_postings': False, @@ -109,6 +110,7 @@ PUT operation. acknowledge_posts: True delivery_mode: plaintext_digests delivery_status: by_user + hide_address: False http_etag: "..." preferred_language: ja receive_list_copy: True @@ -133,6 +135,7 @@ You can also update just a few of the attributes using PATCH. acknowledge_posts: True delivery_mode: plaintext_digests delivery_status: by_user + hide_address: False http_etag: "..." preferred_language: ja receive_list_copy: False diff --git a/src/mailman/rest/helpers.py b/src/mailman/rest/helpers.py index 3a548cb10..7d911d42b 100644 --- a/src/mailman/rest/helpers.py +++ b/src/mailman/rest/helpers.py @@ -36,7 +36,7 @@ import hashlib from cStringIO import StringIO from datetime import datetime, timedelta -from flufl.enum import Enum +from enum import Enum from lazr.config import as_boolean from restish import http from restish.http import Response @@ -79,7 +79,7 @@ class ExtendedEncoder(json.JSONEncoder): seconds = obj.seconds + obj.microseconds / 1000000.0 return '{0}d{1}s'.format(obj.days, seconds) return '{0}d'.format(obj.days) - elif hasattr(obj, 'enum') and issubclass(obj.enum, Enum): + elif isinstance(obj, Enum): # It's up to the decoding validator to associate this name with # the right Enum class. return obj.name diff --git a/src/mailman/rest/lists.py b/src/mailman/rest/lists.py index 2a3c72bfd..32e22a76b 100644 --- a/src/mailman/rest/lists.py +++ b/src/mailman/rest/lists.py @@ -57,7 +57,7 @@ def member_matcher(request, segments): return None try: role = MemberRole[segments[0]] - except ValueError: + except KeyError: # Not a valid role. return None # No more segments. @@ -76,7 +76,7 @@ def roster_matcher(request, segments): return None try: return (), dict(role=MemberRole[segments[1]]), () - except ValueError: + except KeyError: # Not a valid role. return None diff --git a/src/mailman/rest/moderation.py b/src/mailman/rest/moderation.py index c0dbe93b8..491807f38 100644 --- a/src/mailman/rest/moderation.py +++ b/src/mailman/rest/moderation.py @@ -60,7 +60,7 @@ class _ModerationBase: resource.update(data) # Check for a matching request type, and insert the type name into the # resource. - request_type = RequestType(resource.pop('_request_type')) + request_type = RequestType[resource.pop('_request_type')] if request_type not in expected_request_types: return None resource['type'] = request_type.name @@ -205,7 +205,7 @@ class MembershipChangeRequest(resource.Resource, _ModerationBase): return http.not_found() key, data = results try: - request_type = RequestType(data['_request_type']) + request_type = RequestType[data['_request_type']] except ValueError: return http.bad_request() if request_type is RequestType.subscription: diff --git a/src/mailman/rest/preferences.py b/src/mailman/rest/preferences.py index d392c0299..49db6c632 100644 --- a/src/mailman/rest/preferences.py +++ b/src/mailman/rest/preferences.py @@ -84,6 +84,7 @@ class Preferences(ReadOnlyPreferences): return http.not_found() kws = dict( acknowledge_posts=GetterSetter(as_boolean), + hide_address = GetterSetter(as_boolean), delivery_mode=GetterSetter(enum_validator(DeliveryMode)), delivery_status=GetterSetter(enum_validator(DeliveryStatus)), preferred_language=GetterSetter(language_validator), diff --git a/src/mailman/rest/validator.py b/src/mailman/rest/validator.py index c59bc53d8..2d178226f 100644 --- a/src/mailman/rest/validator.py +++ b/src/mailman/rest/validator.py @@ -48,9 +48,13 @@ class enum_validator: self._enum_class = enum_class def __call__(self, enum_value): - # This will raise a ValueError if the enum value is unknown. Let that - # percolate up. - return self._enum_class[enum_value] + # This will raise a KeyError if the enum value is unknown. The + # Validator API requires turning this into a ValueError. + try: + return self._enum_class[enum_value] + except KeyError as exception: + # Retain the error message. + raise ValueError(exception.message) def subscriber_validator(subscriber): diff --git a/src/mailman/rules/docs/__init__.py b/src/mailman/rules/docs/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/mailman/rules/docs/__init__.py diff --git a/src/mailman/rules/docs/moderation.rst b/src/mailman/rules/docs/moderation.rst index eacc1cff3..3000f23a2 100644 --- a/src/mailman/rules/docs/moderation.rst +++ b/src/mailman/rules/docs/moderation.rst @@ -124,7 +124,7 @@ cperson is neither a member, nor a nonmember of the mailing list. :: >>> def memberkey(member): - ... return member.mailing_list, member.address.email, int(member.role) + ... return member.mailing_list, member.address.email, member.role.value >>> dump_list(mlist.members.members, key=memberkey) <Member: Anne Person <aperson@example.com> diff --git a/src/mailman/runners/docs/__init__.py b/src/mailman/runners/docs/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/mailman/runners/docs/__init__.py diff --git a/src/mailman/runners/lmtp.py b/src/mailman/runners/lmtp.py index 31d7723f5..2bc3af979 100644 --- a/src/mailman/runners/lmtp.py +++ b/src/mailman/runners/lmtp.py @@ -156,12 +156,13 @@ class LMTPRunner(Runner, smtpd.SMTPServer): is_queue_runner = False - def __init__(self, slice=None, numslices=1): + def __init__(self, name, slice=None): localaddr = config.mta.lmtp_host, int(config.mta.lmtp_port) # Do not call Runner's constructor because there's no QDIR to create qlog.debug('LMTP server listening on %s:%s', localaddr[0], localaddr[1]) smtpd.SMTPServer.__init__(self, localaddr, remoteaddr=None) + super(LMTPRunner, self).__init__(name, slice) def handle_accept(self): conn, addr = self.accept() diff --git a/src/mailman/runners/rest.py b/src/mailman/runners/rest.py index 468d9a95c..67987805d 100644 --- a/src/mailman/runners/rest.py +++ b/src/mailman/runners/rest.py @@ -25,11 +25,9 @@ __all__ = [ ] -import sys -import errno -import select import signal import logging +import threading from mailman.core.runner import Runner from mailman.rest.wsgiapp import make_server @@ -40,25 +38,47 @@ log = logging.getLogger('mailman.http') class RESTRunner(Runner): - intercept_signals = False + # Don't install the standard signal handlers because as defined, they + # won't actually stop the TCPServer started by .serve_forever(). is_queue_runner = False + def __init__(self, name, slice=None): + """See `IRunner`.""" + super(RESTRunner, self).__init__(name, slice) + # Both the REST server and the signal handlers must run in the main + # thread; the former because of SQLite requirements (objects created + # in one thread cannot be shared with the other threads), and the + # latter because of Python's signal handling semantics. + # + # Unfortunately, we cannot issue a TCPServer shutdown in the main + # thread, because that will cause a deadlock. Yay. So what we do is + # to use the signal handler to notify a shutdown thread that the + # shutdown should happen. That thread will wake up and stop the main + # server. + self._server = make_server() + self._event = threading.Event() + def stopper(event, server): + event.wait() + server.shutdown() + self._thread = threading.Thread( + target=stopper, args=(self._event, self._server)) + self._thread.start() + def run(self): - log.info('Starting REST server') - # Handle SIGTERM the same way as SIGINT. - def stop_server(signum, frame): - log.info('REST server shutdown') - sys.exit(signal.SIGTERM) - signal.signal(signal.SIGTERM, stop_server) - try: - make_server().serve_forever() - except KeyboardInterrupt: - log.info('REST server interrupted') - sys.exit(signal.SIGINT) - except select.error as (errcode, message): - if errcode == errno.EINTR: - log.info('REST server exiting') - sys.exit(errno.EINTR) - raise - except: - raise + """See `IRunner`.""" + self._server.serve_forever() + + def signal_handler(self, signum, frame): + super(RESTRunner, self).signal_handler(signum, frame) + if signum in (signal.SIGTERM, signal.SIGINT, signal.SIGUSR1): + # Set the flag that will terminate the TCPserver loop. + self._event.set() + + def _one_iteration(self): + # Just keep going + if self._thread.is_alive(): + self._thread.join(timeout=0.1) + return 1 + + def _snooze(self, filecnt): + pass diff --git a/src/mailman/runners/tests/test_rest.py b/src/mailman/runners/tests/test_rest.py new file mode 100644 index 000000000..a7b51f720 --- /dev/null +++ b/src/mailman/runners/tests/test_rest.py @@ -0,0 +1,52 @@ +# Copyright (C) 2013 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test the REST runner.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestRESTRunner', + ] + + +import os +import signal +import unittest + +from mailman.testing.helpers import call_api, wait_for_webservice +from mailman.testing.layers import RESTLayer + + + +class TestRESTRunner(unittest.TestCase): + """Test the REST runner.""" + + layer = RESTLayer + + def test_sighup_restart(self): + # The REST runner must survive a SIGHUP. + wait_for_webservice() + for pid in self.layer.server.runner_pids: + os.kill(pid, signal.SIGHUP) + wait_for_webservice() + # This should not raise an exception. The best way to assert this is + # to ensure that the response is valid. + response, json = call_api('http://localhost:9001/3.0/system') + self.assertEqual(json['content-location'], + 'http://localhost:9001/3.0/system') diff --git a/src/mailman/runners/tests/test_retry.py b/src/mailman/runners/tests/test_retry.py index 5869f42a2..99e87235e 100644 --- a/src/mailman/runners/tests/test_retry.py +++ b/src/mailman/runners/tests/test_retry.py @@ -21,6 +21,7 @@ from __future__ import absolute_import, unicode_literals __metaclass__ = type __all__ = [ + 'TestRetryRunner', ] diff --git a/src/mailman/testing/__init__.py b/src/mailman/testing/__init__.py index e6a5047b6..e69de29bb 100644 --- a/src/mailman/testing/__init__.py +++ b/src/mailman/testing/__init__.py @@ -1,39 +0,0 @@ -# Copyright (C) 2011-2013 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Set up testing. - -This is used as an interface to buildout.cfg's [test] section. -zope.testrunner supports an initialization variable. It is set to import and -run the following test initialization method. -""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'initialize', - ] - - - -def initialize(root_directory): - """Initialize the test infrastructure.""" - from mailman.testing import layers - layers.MockAndMonkeyLayer.testing_mode = True - layers.ConfigLayer.enable_stderr(); - layers.ConfigLayer.set_root_directory(root_directory) diff --git a/src/mailman/tests/test_documentation.py b/src/mailman/testing/documentation.py index 37eb760ab..b1cc36f91 100644 --- a/src/mailman/tests/test_documentation.py +++ b/src/mailman/testing/documentation.py @@ -25,23 +25,16 @@ from __future__ import absolute_import, unicode_literals __metaclass__ = type __all__ = [ - 'test_suite', + 'setup', + 'teardown' ] -import os -import sys -import doctest -import unittest - from inspect import isfunction, ismethod -import mailman - from mailman.app.lifecycle import create_list from mailman.config import config -from mailman.testing.helpers import ( - call_api, chdir, specialized_message_from_string) +from mailman.testing.helpers import call_api, specialized_message_from_string from mailman.testing.layers import SMTPLayer @@ -181,53 +174,3 @@ def teardown(testobj): cleanup() else: cleanup[0](*cleanup[1:]) - - - -def test_suite(): - """Create test suites for all .rst documentation tests. - - .txt files are also tested, but .rst is highly preferred. - """ - suite = unittest.TestSuite() - topdir = os.path.dirname(mailman.__file__) - packages = [] - for dirpath, dirnames, filenames in os.walk(topdir): - if 'docs' in dirnames: - docsdir = os.path.join(dirpath, 'docs')[len(topdir)+1:] - packages.append(docsdir) - # Under higher verbosity settings, report all doctest errors, not just the - # first one. - flags = (doctest.ELLIPSIS | - doctest.NORMALIZE_WHITESPACE | - doctest.REPORT_NDIFF) - # Add all the doctests in all subpackages. - doctest_files = {} - with chdir(topdir): - for docsdir in packages: - # Look to see if the package defines a test layer, otherwise use - # SMTPLayer. - package_path = 'mailman.' + DOT.join(docsdir.split(os.sep)) - try: - __import__(package_path) - except ImportError: - layer = SMTPLayer - else: - layer = getattr(sys.modules[package_path], 'layer', SMTPLayer) - for filename in os.listdir(docsdir): - base, extension = os.path.splitext(filename) - if os.path.splitext(filename)[1] in ('.txt', '.rst'): - module_path = package_path + '.' + base - doctest_files[module_path] = ( - os.path.join(docsdir, filename), layer) - for module_path in sorted(doctest_files): - path, layer = doctest_files[module_path] - test = doctest.DocFileSuite( - path, - package='mailman', - optionflags=flags, - setUp=setup, - tearDown=teardown) - test.layer = layer - suite.addTest(test) - return suite diff --git a/src/mailman/testing/layers.py b/src/mailman/testing/layers.py index bee116b6c..6d150815f 100644 --- a/src/mailman/testing/layers.py +++ b/src/mailman/testing/layers.py @@ -45,11 +45,9 @@ import logging import datetime import tempfile -from base64 import b64encode -from lazr.config import as_boolean, as_timedelta +from lazr.config import as_boolean from pkg_resources import resource_string from textwrap import dedent -from urllib2 import Request, URLError, urlopen from zope.component import getUtility from mailman.config import config @@ -59,7 +57,7 @@ from mailman.core.logging import get_handler from mailman.database.transaction import transaction from mailman.interfaces.domain import IDomainManager from mailman.testing.helpers import ( - TestableMaster, get_lmtp_client, reset_the_world) + TestableMaster, get_lmtp_client, reset_the_world, wait_for_webservice) from mailman.testing.mta import ConnectionCountingController from mailman.utilities.string import expand @@ -156,7 +154,7 @@ class ConfigLayer(MockAndMonkeyLayer): continue logger_name = 'mailman.' + sub_name log = logging.getLogger(logger_name) - log.propagate = True + #log.propagate = True # Reopen the file to a new path that tests can get at. Instead of # using the configuration file path though, use a path that's # specific to the logger so that tests can find expected output @@ -315,29 +313,10 @@ class RESTLayer(SMTPLayer): server = None - @staticmethod - def _wait_for_rest_server(): - until = datetime.datetime.now() + as_timedelta(config.devmode.wait) - while datetime.datetime.now() < until: - try: - request = Request('http://localhost:9001/3.0/system') - basic_auth = '{0}:{1}'.format(config.webservice.admin_user, - config.webservice.admin_pass) - request.add_header('Authorization', - 'Basic ' + b64encode(basic_auth)) - fp = urlopen(request) - except URLError: - pass - else: - fp.close() - break - else: - raise RuntimeError('REST server did not start up') - @classmethod def setUp(cls): assert cls.server is None, 'Layer already set up' - cls.server = TestableMaster(cls._wait_for_rest_server) + cls.server = TestableMaster(wait_for_webservice) cls.server.start('rest') @classmethod diff --git a/src/mailman/testing/nose.py b/src/mailman/testing/nose.py new file mode 100644 index 000000000..86a3e6a01 --- /dev/null +++ b/src/mailman/testing/nose.py @@ -0,0 +1,107 @@ +# Copyright (C) 2013 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""nose2 test infrastructure.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'NosePlugin', + ] + + +import os +import re +import doctest +import mailman +import importlib + +from mailman.testing.documentation import setup, teardown +from mailman.testing.layers import ConfigLayer, MockAndMonkeyLayer, SMTPLayer +from nose2.events import Plugin + +DOT = '.' +FLAGS = doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF +TOPDIR = os.path.dirname(mailman.__file__) + + + +class NosePlugin(Plugin): + configSection = 'mailman' + + def __init__(self): + super(NosePlugin, self).__init__() + self.patterns = [] + self.addArgument(self.patterns, 'P', 'pattern', + 'Add a test matching pattern') + + def startTestRun(self, event): + MockAndMonkeyLayer.testing_mode = True + ConfigLayer.enable_stderr() + + def getTestCaseNames(self, event): + if len(self.patterns) == 0: + # No filter patterns, so everything should be tested. + return + # Does the pattern match the fully qualified class name? + for pattern in self.patterns: + full_name = '{}.{}'.format( + event.testCase.__module__, event.testCase.__name__) + if re.search(pattern, full_name): + # Don't suppress this test class. + return + names = filter(event.isTestMethod, dir(event.testCase)) + for name in names: + for pattern in self.patterns: + if re.search(pattern, name): + break + else: + event.excludedNames.append(name) + + def handleFile(self, event): + path = event.path[len(TOPDIR)+1:] + if len(self.patterns) > 0: + for pattern in self.patterns: + if re.search(pattern, path): + break + else: + # Skip this doctest. + return + base, ext = os.path.splitext(path) + if ext != '.rst': + return + # Look to see if the package defines a test layer, otherwise use the + # default layer. First turn the file system path into a dotted Python + # module path. + parent = os.path.dirname(path) + dotted = 'mailman.' + DOT.join(parent.split(os.path.sep)) + try: + module = importlib.import_module(dotted) + except ImportError: + layer = SMTPLayer + else: + layer = getattr(module, 'layer', SMTPLayer) + test = doctest.DocFileTest( + path, package='mailman', + optionflags=FLAGS, + setUp=setup, + tearDown=teardown) + test.layer = layer + # Suppress the extra "Doctest: ..." line. + test.shortDescription = lambda: None + event.extraTests.append(test) diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py index 20941bfd6..f5aa8d10a 100644 --- a/src/mailman/utilities/importer.py +++ b/src/mailman/utilities/importer.py @@ -28,6 +28,7 @@ __all__ = [ import sys import datetime +from mailman.interfaces.action import FilterAction from mailman.interfaces.autorespond import ResponseAction from mailman.interfaces.digests import DigestFrequency from mailman.interfaces.mailinglist import Personalization, ReplyToMunging @@ -47,6 +48,7 @@ TYPES = dict( bounce_info_stale_after=seconds_to_delta, bounce_you_are_disabled_warnings_interval=seconds_to_delta, digest_volume_frequency=DigestFrequency, + filter_action=FilterAction, newsgroup_moderation=NewsgroupModeration, personalize=Personalization, reply_goes_to_list=ReplyToMunging, |