14 files changed, 95 insertions, 51 deletions

diff --git a/mailman/Message.py b/mailman/Message.py
index fc890fb51..aaac5693f 100644
--- a/mailman/Message.py
+++ b/mailman/Message.py
@@ -29,6 +29,7 @@ import email.utils
 from email.charset import Charset
 from email.header import Header
 
+from mailman import Defaults
 from mailman import Utils
 from mailman.config import config
 
@@ -186,7 +187,7 @@ class Message(email.message.Message):
         names without the trailing colon.
         """
         if headers is None:
-            headers = config.SENDER_HEADERS
+            headers = Defaults.SENDER_HEADERS
         pairs = []
         for h in headers:
             if h is None:
diff --git a/mailman/app/replybot.py b/mailman/app/replybot.py
index 394a93aaa..252b1b15a 100644
--- a/mailman/app/replybot.py
+++ b/mailman/app/replybot.py
@@ -29,9 +29,9 @@ __all__ = [
 import logging
 import datetime
 
+from mailman import Defaults
 from mailman import Utils
 from mailman import i18n
-from mailman.config import config
 
 
 log = logging.getLogger('mailman.vette')
@@ -48,7 +48,7 @@ def autorespond_to_sender(mlist, sender, lang=None):
     """
     if lang is None:
         lang = mlist.preferred_language
-    if config.MAX_AUTORESPONSES_PER_DAY == 0:
+    if Defaults.MAX_AUTORESPONSES_PER_DAY == 0:
         # Unlimited.
         return True
     today = datetime.date.today()
@@ -64,7 +64,7 @@ def autorespond_to_sender(mlist, sender, lang=None):
         # them of this fact, so there's nothing more to do.
         log.info('-request/hold autoresponse discarded for: %s', sender)
         return False
-    if count >= config.MAX_AUTORESPONSES_PER_DAY:
+    if count >= Defaults.MAX_AUTORESPONSES_PER_DAY:
         log.info('-request/hold autoresponse limit hit for: %s', sender)
         mlist.hold_and_cmd_autoresponses[sender] = (today, -1)
         # Send this notification message instead.
diff --git a/mailman/bin/docs/master.txt b/mailman/bin/docs/master.txt
index 7565e93b1..2938c7ffb 100644
--- a/mailman/bin/docs/master.txt
+++ b/mailman/bin/docs/master.txt
@@ -6,7 +6,6 @@ directories.  In normal operation, a command line script called 'mailmanctl'
 is used to start, stop and manage the queue runners.  mailmanctl actually is
 just a wrapper around the real queue runner watcher script called master.py.
 
-    >>> from mailman.configuration import config
     >>> from mailman.testing.helpers import TestableMaster
 
 Start the master in a subthread.
diff --git a/mailman/bin/master.py b/mailman/bin/master.py
index 39f119725..129df0009 100644
--- a/mailman/bin/master.py
+++ b/mailman/bin/master.py
@@ -269,7 +269,9 @@ class Loop:
 
         :param spec: A queue runner spec, in a format acceptable to
             bin/qrunner's --runner argument, e.g. name:slice:count
+        :type spec: string
         :return: The process id of the child queue runner.
+        :rtype: int
         """
         pid = os.fork()
         if pid:
@@ -292,25 +294,43 @@ class Loop:
         # We should never get here.
         raise RuntimeError('os.execl() failed')
 
-    def start_qrunners(self, qrunners=None):
+    def start_qrunners(self, qrunner_names=None):
         """Start all the configured qrunners.
 
         :param qrunners: If given, a sequence of queue runner names to start.
             If not given, this sequence is taken from the configuration file.
+        :type qrunners: a sequence of strings
         """
-        if not qrunners:
-            spec_parts = config.qrunners.items()
-        else:
-            spec_parts = []
-            for qrname in qrunners:
-                if '.' in qrname:
-                    spec_parts.append((qrname, 1))
-                else:
-                    spec_parts.append((config.qrunner_shortcuts[qrname], 1))
-        for qrname, count in spec_parts:
+        if not qrunner_names:
+            qrunner_names = []
+            for qrunner_config in config.qrunner_configs:
+                # Strip off the 'qrunner.' prefix.
+                assert qrunner_config.name.startswith('qrunner.'), (
+                    'Unexpected qrunner configuration section name: %s',
+                    qrunner_config.name)
+                qrunner_names.append(qrunner_config.name[8:])
+        # For each qrunner we want to start, find their config section, which
+        # will tell us the name of the class to instantiate, along with the
+        # number of hash space slices to manage.
+        for name in qrunner_names:
+            section_name = 'qrunner.' + name
+            # Let AttributeError propagate.
+            qrunner_config = getattr(config, section_name)
+            if not qrunner_config.start:
+                continue
+            class_path = qrunner_config['class'].split(DOT)
+            package = DOT.join(class_path[:-1])
+            __import__(package)
+            # Let AttributeError propagate.
+            class_ = getattr(sys.modules[package], class_path[-1])
+            # Find out how many qrunners to instantiate.  This must be a power
+            # of 2.
+            count = int(qrunner_config.instances)
+            assert (count & (count - 1)) == 0, (
+                'Queue runner "%s", not a power of 2: %s', name, count)
             for slice_number in range(count):
                 # qrunner name, slice #, # of slices, restart count
-                info = (qrname, slice_number, count, 0)
+                info = (name, slice_number, count, 0)
                 spec = '%s:%d:%d' % (qrname, slice_number, count)
                 pid = self._start_runner(spec)
                 log = logging.getLogger('mailman.qrunner')
diff --git a/mailman/chains/accept.py b/mailman/chains/accept.py
index 0ced02d44..4aacb6bf8 100644
--- a/mailman/chains/accept.py
+++ b/mailman/chains/accept.py
@@ -25,7 +25,6 @@ import logging
 from mailman.chains.base import TerminalChainBase
 from mailman.config import config
 from mailman.i18n import _
-from mailman.queue import Switchboard
 
 
 log = logging.getLogger('mailman.vette')
@@ -50,6 +49,6 @@ class AcceptChain(TerminalChainBase):
         rule_misses = msgdata.get('rule_misses')
         if rule_misses:
             msg['X-Mailman-Rule-Misses'] = SEMISPACE.join(rule_misses)
-        accept_queue = Switchboard(config.PIPELINEQUEUE_DIR)
+        accept_queue = config.switchboards['pipeline']
         accept_queue.enqueue(msg, msgdata)
         log.info('ACCEPT: %s', msg.get('message-id', 'n/a'))
diff --git a/mailman/config/config.py b/mailman/config/config.py
index f3bbed316..3ac117283 100644
--- a/mailman/config/config.py
+++ b/mailman/config/config.py
@@ -48,8 +48,6 @@ class Configuration(object):
 
     def __init__(self):
         self.domains = {}       # email host -> IDomain
-        self.qrunners = {}
-        self.qrunner_shortcuts = {}
         self.switchboards = {}
         self.languages = LanguageManager()
         self.QFILE_SCHEMA_VERSION = version.QFILE_SCHEMA_VERSION
@@ -64,13 +62,8 @@ class Configuration(object):
 
     def _clear(self):
         """Clear the cached configuration variables."""
-        # First, stop all registered qrunners.
-        for runner in self.qrunners.values():
-            runner.stop()
         self.domains.clear()
         self.switchboards.clear()
-        self.qrunners.clear()
-        self.qrunner_shortcuts.clear()
         self.languages = LanguageManager()
 
     def __getattr__(self, name):
diff --git a/mailman/core/logging.py b/mailman/core/logging.py
index 8c1463be9..8a9db5745 100644
--- a/mailman/core/logging.py
+++ b/mailman/core/logging.py
@@ -37,12 +37,15 @@ from lazr.config import as_boolean, as_log_level
 from mailman.config import config
 
 
-_handlers = []
+_handlers = {}
 
 
 
 class ReopenableFileHandler(logging.Handler):
-    def __init__(self, filename):
+    """A file handler that supports reopening."""
+
+    def __init__(self, name, filename):
+        self.name = name
         self._filename = filename
         self._stream = self._open()
         logging.Handler.__init__(self)
@@ -76,7 +79,15 @@ class ReopenableFileHandler(logging.Handler):
         self._stream = None
         logging.Handler.close(self)
 
-    def reopen(self):
+    def reopen(self, filename=None):
+        """Reopen the output stream.
+
+        :param filename: If given, this reopens the output stream to a new
+            file.  This is used in the test suite.
+        :type filename: string
+        """
+        if filename is not None:
+            self._filename = filename
         self._stream.close()
         self._stream = self._open()
 
@@ -118,8 +129,8 @@ def initialize(propagate=None):
         formatter = logging.Formatter(fmt=log_format, datefmt=log_datefmt)
         path_str  = logger_config.path
         path_abs  = os.path.normpath(os.path.join(config.LOG_DIR, path_str))
-        handler   = ReopenableFileHandler(path_abs)
-        _handlers.append(handler)
+        handler   = ReopenableFileHandler(sub_name, path_abs)
+        _handlers[sub_name] = handler
         handler.setFormatter(formatter)
         log.addHandler(handler)
 
@@ -127,5 +138,17 @@ def initialize(propagate=None):
 
 def reopen():
     """Re-open all log files."""
-    for handler in _handlers:
+    for handler in _handlers.values():
         handler.reopen()
+
+
+
+def get_handler(sub_name):
+    """Return the handler associated with a named logger.
+
+    :param sub_name: The logger name, sans the 'mailman.' prefix.
+    :type sub_name: string
+    :return: The file handler associated with the named logger.
+    :rtype: `ReopenableFileHandler`
+    """
+    return _handlers[sub_name]
diff --git a/mailman/database/mailinglist.py b/mailman/database/mailinglist.py
index e1c6414be..553fa8c48 100644
--- a/mailman/database/mailinglist.py
+++ b/mailman/database/mailinglist.py
@@ -21,6 +21,7 @@ import string
 from storm.locals import *
 from zope.interface import implements
 
+from mailman import Defaults
 from mailman.Utils import fqdn_listname, makedirs, split_listname
 from mailman.config import config
 from mailman.database import roster
@@ -218,7 +219,7 @@ class MailingList(Model):
 
     @property
     def no_reply_address(self):
-        return '%s@%s' % (config.NO_REPLY_ADDRESS, self.host_name)
+        return '%s@%s' % (config.mailman.noreply_address, self.host_name)
 
     @property
     def owner_address(self):
@@ -249,7 +250,7 @@ class MailingList(Model):
         return '%s-unsubscribe@%s' % (self.list_name, self.host_name)
 
     def confirm_address(self, cookie):
-        template = string.Template(config.VERP_CONFIRM_FORMAT)
+        template = string.Template(Defaults.VERP_CONFIRM_FORMAT)
         local_part = template.safe_substitute(
             address = '%s-confirm' % self.list_name,
             cookie  = cookie)
diff --git a/mailman/docs/message.txt b/mailman/docs/message.txt
index b580091cc..dab9ddf0e 100644
--- a/mailman/docs/message.txt
+++ b/mailman/docs/message.txt
@@ -28,8 +28,7 @@ address, an optional subject, optional body text, and optional language.
 
 The message will end up in the virgin queue.
 
-    >>> from mailman.queue import Switchboard
-    >>> switchboard = Switchboard(config.VIRGINQUEUE_DIR)
+    >>> switchboard = config.switchboards['virgin']
     >>> len(switchboard.files)
     1
     >>> filebase = switchboard.files[0]
diff --git a/mailman/docs/mlist-addresses.txt b/mailman/docs/mlist-addresses.txt
index bfffe10eb..fe76fb4d5 100644
--- a/mailman/docs/mlist-addresses.txt
+++ b/mailman/docs/mlist-addresses.txt
@@ -67,8 +67,9 @@ dependent on the VERP_CONFIRM_FORMAT configuration variable.
     >>> mlist.confirm_address('wookie')
     u'_xtest-confirm+wookie@example.com'
 
-    >>> old_format = config.VERP_CONFIRM_FORMAT
-    >>> config.VERP_CONFIRM_FORMAT = '$address---$cookie'
+    >>> from mailman import Defaults
+    >>> old_format = Defaults.VERP_CONFIRM_FORMAT
+    >>> Defaults.VERP_CONFIRM_FORMAT = '$address---$cookie'
     >>> mlist.confirm_address('cookie')
     u'_xtest-confirm---cookie@example.com'
     >>> config.VERP_CONFIRM_FORMAT = old_format
diff --git a/mailman/pipeline/to_usenet.py b/mailman/pipeline/to_usenet.py
index 14e7811bb..83980e071 100644
--- a/mailman/pipeline/to_usenet.py
+++ b/mailman/pipeline/to_usenet.py
@@ -28,7 +28,6 @@ from zope.interface import implements
 from mailman.config import config
 from mailman.i18n import _
 from mailman.interfaces import IHandler
-from mailman.queue import Switchboard
 
 COMMASPACE = ', '
 
@@ -62,5 +61,5 @@ class ToUsenet:
                       COMMASPACE.join(error))
             return
         # Put the message in the news runner's queue
-        newsq = Switchboard(config.NEWSQUEUE_DIR)
-        newsq.enqueue(msg, msgdata, listname=mlist.fqdn_listname)
+        config.switchboards['news'].enqueue(
+            msg, msgdata, listname=mlist.fqdn_listname)
diff --git a/mailman/queue/docs/incoming.txt b/mailman/queue/docs/incoming.txt
index 6bb5ead15..22b32d828 100644
--- a/mailman/queue/docs/incoming.txt
+++ b/mailman/queue/docs/incoming.txt
@@ -43,20 +43,19 @@ The incoming queue runner runs until it is empty.
 
     >>> from mailman.queue.incoming import IncomingRunner
     >>> from mailman.testing.helpers import make_testable_runner
-    >>> incoming = make_testable_runner(IncomingRunner)
+    >>> incoming = make_testable_runner(IncomingRunner, 'in')
     >>> incoming.run()
 
 And now the message is in the pipeline queue.
 
-    >>> from mailman.queue import Switchboard
-    >>> pipeline_queue = Switchboard(config.PIPELINEQUEUE_DIR)
+    >>> pipeline_queue = config.switchboards['pipeline']
     >>> len(pipeline_queue.files)
     1
-    >>> incoming_queue = Switchboard(config.INQUEUE_DIR)
+    >>> incoming_queue = config.switchboards['in']
     >>> len(incoming_queue.files)
     0
     >>> from mailman.testing.helpers import get_queue_messages
-    >>> item = get_queue_messages(pipeline_queue)[0]
+    >>> item = get_queue_messages('pipeline')[0]
     >>> print item.msg.as_string()
     From: aperson@example.com
     To: _xtest@example.com
@@ -152,8 +151,8 @@ just create a new chain that does.
 The virgin queue needs to be cleared out due to artifacts from the previous
 tests above.
 
-    >>> virgin_queue = Switchboard(config.VIRGINQUEUE_DIR)
-    >>> ignore = get_queue_messages(virgin_queue)
+    >>> virgin_queue = config.switchboards['virgin']
+    >>> ignore = get_queue_messages('virgin')
 
     >>> inject_message(mlist, msg)
     >>> file_pos = fp.tell()
@@ -165,7 +164,7 @@ tests above.
 
     >>> len(virgin_queue.files)
     1
-    >>> item = get_queue_messages(virgin_queue)[0]
+    >>> item = get_queue_messages('virgin')[0]
     >>> print item.msg.as_string()
     Subject: My first post
     From: _xtest-owner@example.com
diff --git a/mailman/queue/incoming.py b/mailman/queue/incoming.py
index aaab56b21..6e80024e7 100644
--- a/mailman/queue/incoming.py
+++ b/mailman/queue/incoming.py
@@ -33,7 +33,7 @@ from mailman.queue import Runner
 
 
 class IncomingRunner(Runner):
-    QDIR = config.INQUEUE_DIR
+    """The incoming queue runner."""
 
     def _dispose(self, mlist, msg, msgdata):
         if msgdata.get('envsender') is None:
diff --git a/mailman/testing/layers.py b/mailman/testing/layers.py
index 883a5c784..ce9f15736 100644
--- a/mailman/testing/layers.py
+++ b/mailman/testing/layers.py
@@ -36,6 +36,7 @@ from textwrap import dedent
 from mailman.config import config
 from mailman.core.initialize import initialize
 from mailman.i18n import _
+from mailman.core.logging import get_handler
 from mailman.testing.helpers import SMTPServer
 
 
@@ -65,7 +66,10 @@ class ConfigLayer:
         # Read the testing config and push it.
         test_config += resource_string('mailman.testing', 'testing.cfg')
         config.push('test config', test_config)
-        # Enable log message propagation.
+        # Enable log message propagation and reset the log paths so that the
+        # doctests can check the output.  XXX Switch to using the log support
+        # in zope.testing.
+        os.makedirs(config.LOG_DIR)
         for logger_config in config.logger_configs:
             sub_name = logger_config.name.split('.')[-1]
             if sub_name == 'root':
@@ -73,6 +77,12 @@ class ConfigLayer:
             logger_name = 'mailman.' + sub_name
             log = logging.getLogger(logger_name)
             log.propagate = True
+            # Reopen the file to a new path that tests can get at.  Instead of
+            # using the configuration file path though, use a path that's
+            # specific to the logger so that tests can find expected output
+            # more easily.
+            path = os.path.join(config.LOG_DIR, sub_name)
+            get_handler(sub_name).reopen(path)
             log.setLevel(logging.DEBUG)
         # zope.testing sets up logging before we get to our own initialization
         # function.  This messes with the root logger, so explicitly set it to