14 files changed, 102 insertions, 60 deletions

diff --git a/mailman/Defaults.py b/mailman/Defaults.py
index 4f4f7981e..6efc47f90 100644
--- a/mailman/Defaults.py
+++ b/mailman/Defaults.py
@@ -693,11 +693,6 @@ USE_MAILDIR = No
 #     alias_maps = hash:/etc/aliases, hash:<prefix>/data/aliases
 USE_LMTP = No
 
-# Change LMTP_HOST and LMTP_PORT for your convenience.  You should be careful
-# enough to use a firewall if you open your port on global address interface.
-LMTP_HOST = 'localhost'
-LMTP_PORT = 8025
-
 # Name of the domains which operate on LMTP Mailman only.  Currently valid
 # only for Postfix alias generation.
 LMTP_ONLY_DOMAINS = []
diff --git a/mailman/bin/master.py b/mailman/bin/master.py
index 129df0009..e47e0d839 100644
--- a/mailman/bin/master.py
+++ b/mailman/bin/master.py
@@ -318,11 +318,10 @@ class Loop:
             qrunner_config = getattr(config, section_name)
             if not qrunner_config.start:
                 continue
-            class_path = qrunner_config['class'].split(DOT)
-            package = DOT.join(class_path[:-1])
+            package, class_name = qrunner_config['class'].rsplit(DOT, 1)
             __import__(package)
             # Let AttributeError propagate.
-            class_ = getattr(sys.modules[package], class_path[-1])
+            class_ = getattr(sys.modules[package], class_name)
             # Find out how many qrunners to instantiate.  This must be a power
             # of 2.
             count = int(qrunner_config.instances)
@@ -331,7 +330,7 @@ class Loop:
             for slice_number in range(count):
                 # qrunner name, slice #, # of slices, restart count
                 info = (name, slice_number, count, 0)
-                spec = '%s:%d:%d' % (qrname, slice_number, count)
+                spec = '%s:%d:%d' % (name, slice_number, count)
                 pid = self._start_runner(spec)
                 log = logging.getLogger('mailman.qrunner')
                 log.debug('[%d] %s', pid, spec)
@@ -369,12 +368,14 @@ class Loop:
             # command line switch was not given.  This lets us better handle
             # runaway restarts (e.g.  if the subprocess had a syntax error!)
             qrname, slice_number, count, restarts = self._kids.pop(pid)
+            config_name = 'qrunner.' + qrname
             restart = False
             if why == signal.SIGUSR1 and self._restartable:
                 restart = True
             # Have we hit the maximum number of restarts?
             restarts += 1
-            if restarts > config.MAX_RESTARTS:
+            max_restarts = int(getattr(config, config_name).max_restarts)
+            if restarts > max_restarts:
                 restart = False
             # Are we permanently non-restartable?
             log.debug("""\
@@ -383,10 +384,10 @@ Master detected subprocess exit
                      pid, why, qrname, slice_number + 1, count,
                      ('[restarting]' if restart else ''))
             # See if we've reached the maximum number of allowable restarts
-            if restarts > config.MAX_RESTARTS:
+            if restarts > max_restarts:
                 log.info("""\
 qrunner %s reached maximum restart limit of %d, not restarting.""",
-                         qrname, config.MAX_RESTARTS)
+                         qrname, max_restarts)
             # Now perhaps restart the process unless it exited with a
             # SIGTERM or we aren't restarting.
             if restart:
diff --git a/mailman/bin/qrunner.py b/mailman/bin/qrunner.py
index 422809fa3..dea8f4053 100644
--- a/mailman/bin/qrunner.py
+++ b/mailman/bin/qrunner.py
@@ -19,8 +19,8 @@ import sys
 import signal
 import logging
 
-from mailman import loginit
-from mailman.configuration import config
+from mailman.config import config
+from mailman.core.logging import reopen
 from mailman.i18n import _
 from mailman.options import Options
 
@@ -112,10 +112,6 @@ This should only be used when running qrunner as a subprocess of the
 mailmanctl startup script.  It changes some of the exit-on-error behavior to
 work better with that framework."""))
 
-    def initialize(self):
-        """Override initialization to propagate logs correctly."""
-        super(ScriptOptions, self).initialize(not self.options.subproc)
-
     def sanity_check(self):
         if self.arguments:
             self.parser.error(_('Unexpected arguments'))
@@ -126,28 +122,30 @@ work better with that framework."""))
 
 def make_qrunner(name, slice, range, once=False):
     # Several conventions for specifying the runner name are supported.  It
-    # could be one of the shortcut names.  Or the name is a full module path,
+    # could be one of the shortcut names.  If the name is a full module path,
     # use it explicitly.  If the name starts with a dot, it's a class name
     # relative to the Mailman.queue package.
-    if name in config.qrunner_shortcuts:
-        classpath = config.qrunner_shortcuts[name]
+    qrunner_config = getattr(config, 'qrunner.' + name, None)
+    if qrunner_config is not None:
+        # It was a shortcut name.
+        class_path = qrunner_config['class']
     elif name.startswith('.'):
-        classpath = 'mailman.queue' + name
+        class_path = 'mailman.queue' + name
     else:
-        classpath = name
-    modulename, classname = classpath.rsplit('.', 1)
+        class_path = name
+    module_name, class_name = class_path.rsplit('.', 1)
     try:
-        __import__(modulename)
+        __import__(module_name)
     except ImportError, e:
         if config.options.options.subproc:
             # Exit with SIGTERM exit code so the master watcher won't try to
             # restart us.
-            print >> sys.stderr, _('Cannot import runner module: $modulename')
+            print >> sys.stderr, _('Cannot import runner module: $module_name')
             print >> sys.stderr, e
             sys.exit(signal.SIGTERM)
         else:
             raise
-    qrclass = getattr(sys.modules[modulename], classname)
+    qrclass = getattr(sys.modules[module_name], class_name)
     if once:
         # Subclass to hack in the setting of the stop flag in _do_periodic()
         class Once(qrclass):
@@ -191,7 +189,7 @@ def set_signals(loop):
     signal.signal(signal.SIGUSR1, sigusr1_handler)
     # SIGHUP just tells us to rotate our log files.
     def sighup_handler(signum, frame):
-        loginit.reopen()
+        reopen()
         log.info('%s qrunner caught SIGHUP.  Reopening logs.', loop.name())
     signal.signal(signal.SIGHUP, sighup_handler)
 
diff --git a/mailman/config/config.py b/mailman/config/config.py
index 3ac117283..ce1b4b732 100644
--- a/mailman/config/config.py
+++ b/mailman/config/config.py
@@ -52,6 +52,7 @@ class Configuration(object):
         self.languages = LanguageManager()
         self.QFILE_SCHEMA_VERSION = version.QFILE_SCHEMA_VERSION
         self._config = None
+        self.filename = None
         # Create various registries.
         self.archivers = {}
         self.chains = {}
@@ -80,8 +81,9 @@ class Configuration(object):
         config_string = resource_string('mailman.config', 'mailman.cfg')
         self._config = schema.loadFile(StringIO(config_string), 'mailman.cfg')
         if filename is not None:
+            self.filename = filename
             with open(filename) as user_config:
-                self._config.push(user_config.read())
+                self._config.push(filename, user_config.read())
         self._post_process()
 
     def push(self, config_name, config_string):
@@ -147,6 +149,7 @@ class Configuration(object):
                                         language.charset, language.enabled)
         # Always enable the server default language, which must be defined.
         self.languages.enable_language(self._config.mailman.default_language)
+        self.ensure_directories_exist()
 
     @property
     def logger_configs(self):
diff --git a/mailman/config/schema.cfg b/mailman/config/schema.cfg
index 59ae160f4..c66ba50ae 100644
--- a/mailman/config/schema.cfg
+++ b/mailman/config/schema.cfg
@@ -244,6 +244,10 @@ outgoing: mailman.mta.direct.SMTPDirect
 smtp_host: localhost
 smtp_port: 25
 
+# Where the LMTP server listens for connections.
+lmtp_host: localhost
+lmtp_port: 8025
+
 
 [archiver.master]
 base_url: http://archive.example.com/
diff --git a/mailman/core/initialize.py b/mailman/core/initialize.py
index 39e924545..f4b8de627 100644
--- a/mailman/core/initialize.py
+++ b/mailman/core/initialize.py
@@ -42,7 +42,7 @@ from mailman.interfaces import IDatabase
 # initialization, but before database initialization.  Generally all other
 # code will just call initialize().
 
-def initialize_1(config_path, propagate_logs):
+def initialize_1(config_path=None, propagate_logs=None):
     """First initialization step.
 
     * The configuration system
@@ -52,7 +52,7 @@ def initialize_1(config_path, propagate_logs):
     :param config_path: The path to the configuration file.
     :type config_path: string
     :param propagate_logs: Should the log output propagate to stderr?
-    :type propagate_logs: boolean
+    :type propagate_logs: boolean or None
     """
     # By default, set the umask so that only owner and group can read and
     # write our files.  Specifically we must have g+rw and we probably want
@@ -111,7 +111,7 @@ def initialize_3():
 
 
 
-def initialize(config_path=None, propagate_logs=False):
+def initialize(config_path=None, propagate_logs=None):
     initialize_1(config_path, propagate_logs)
     initialize_2()
     initialize_3()
diff --git a/mailman/database/__init__.py b/mailman/database/__init__.py
index 65022ffe4..e3b23a4ba 100644
--- a/mailman/database/__init__.py
+++ b/mailman/database/__init__.py
@@ -21,6 +21,7 @@ __all__ = [
     ]
 
 import os
+import logging
 
 from locknix.lockfile import Lock
 from lazr.config import as_boolean
@@ -41,6 +42,8 @@ from mailman.database.usermanager import UserManager
 from mailman.database.version import Version
 from mailman.interfaces.database import IDatabase, SchemaVersionMismatchError
 
+log = logging.getLogger('mailman.config')
+
 
 
 class StockDatabase:
@@ -84,6 +87,7 @@ class StockDatabase:
     def _create(self, debug):
         # Calculate the engine url.
         url = Template(config.database.url).safe_substitute(config.paths)
+        log.debug('Database url: %s', url)
         # XXX By design of SQLite, database file creation does not honor
         # umask.  See their ticket #1193:
         # http://www.sqlite.org/cvstrac/tktview?tn=1193,31
diff --git a/mailman/database/transaction.py b/mailman/database/transaction.py
index 14c494f89..e2eb98eff 100644
--- a/mailman/database/transaction.py
+++ b/mailman/database/transaction.py
@@ -23,7 +23,7 @@ __all__ = [
     ]
 
 
-from mailman.configuration import config
+from mailman.config import config
 
 
 
diff --git a/mailman/options.py b/mailman/options.py
index adfe26cd9..81870ee3d 100644
--- a/mailman/options.py
+++ b/mailman/options.py
@@ -95,16 +95,18 @@ class Options:
             '-C', '--config',
             help=_('Alternative configuration file to use'))
 
-    def initialize(self, propagate_logs=False):
+    def initialize(self, propagate_logs=None):
         """Initialize the configuration system.
 
         After initialization of the configuration system, perform sanity
         checks.  We do it in this order because some sanity checks require the
         configuration to be initialized.
 
-        :param propagate_logs: Flag specifying whether log messages in
-            sub-loggers should be propagated to the master logger (and hence
-            to the root logger).
+        :param propagate_logs: Optional flag specifying whether log messages
+            in sub-loggers should be propagated to the master logger (and
+            hence to the root logger).  If not given, propagation is taken
+            from the configuration files.
+        :type propagate_logs: bool or None.
         """
         initialize(self.options.config, propagate_logs=propagate_logs)
         self.sanity_check()
diff --git a/mailman/queue/docs/lmtp.txt b/mailman/queue/docs/lmtp.txt
index bb77203b4..75e91fd4e 100644
--- a/mailman/queue/docs/lmtp.txt
+++ b/mailman/queue/docs/lmtp.txt
@@ -25,7 +25,7 @@ It also helps to have a nice LMTP client.
 Posting address
 ---------------
 
-If the mail server tries to send a message to a non-existant mailing list, it
+If the mail server tries to send a message to a nonexistent mailing list, it
 will get a 550 error.
 
     >>> lmtp.sendmail(
diff --git a/mailman/queue/lmtp.py b/mailman/queue/lmtp.py
index 18f430e55..e64d777ac 100644
--- a/mailman/queue/lmtp.py
+++ b/mailman/queue/lmtp.py
@@ -42,10 +42,11 @@ import asyncore
 
 from email.utils import parseaddr
 
+from mailman import Defaults
 from mailman.Message import Message
 from mailman.config import config
 from mailman.database.transaction import txn
-from mailman.queue import Runner, Switchboard
+from mailman.queue import Runner
 
 elog = logging.getLogger('mailman.error')
 qlog = logging.getLogger('mailman.qrunner')
@@ -63,7 +64,7 @@ CRLF    = '\r\n'
 ERR_451 = '451 Requested action aborted: error in processing'
 ERR_501 = '501 Message has defects'
 ERR_502 = '502 Error: command HELO not implemented'
-ERR_550 = config.LMTP_ERR_550
+ERR_550 = Defaults.LMTP_ERR_550
 
 # XXX Blech
 smtpd.__version__ = 'Python LMTP queue runner 1.0'
@@ -86,7 +87,7 @@ def split_recipient(address):
         subaddress may be None if this is the list's posting address.
     """
     localpart, domain = address.split('@', 1)
-    localpart = localpart.split(config.VERP_DELIMITER, 1)[0]
+    localpart = localpart.split(Defaults.VERP_DELIMITER, 1)[0]
     parts = localpart.split(DASH)
     if parts[-1] in SUBADDRESS_NAMES:
         listname = DASH.join(parts[:-1])
@@ -121,11 +122,11 @@ class LMTPRunner(Runner, smtpd.SMTPServer):
     # connections from the MTA.  slice and numslices are ignored and are
     # necessary only to satisfy the API.
     def __init__(self, slice=None, numslices=1):
-        localaddr = config.LMTP_HOST, config.LMTP_PORT
+        localaddr = config.mta.lmtp_host, int(config.mta.lmtp_port)
         # Do not call Runner's constructor because there's no QDIR to create
         smtpd.SMTPServer.__init__(self, localaddr, remoteaddr=None)
         qlog.debug('LMTP server listening on %s:%s',
-                   config.LMTP_HOST, config.LMTP_PORT)
+                   localaddr[0], localaddr[1])
 
     def handle_accept(self):
         conn, addr = self.accept()
@@ -138,6 +139,7 @@ class LMTPRunner(Runner, smtpd.SMTPServer):
             # Refresh the list of list names every time we process a message
             # since the set of mailing lists could have changed.
             listnames = set(config.db.list_manager.names)
+            qlog.debug('listnames: %s', listnames)
             # Parse the message data.  If there are any defects in the
             # message, reject it right away; it's probably spam. 
             msg = email.message_from_string(data, Message)
@@ -169,29 +171,29 @@ class LMTPRunner(Runner, smtpd.SMTPServer):
                 queue = None
                 msgdata = dict(listname=listname)
                 if subaddress in ('bounces', 'admin'):
-                    queue = Switchboard(config.BOUNCEQUEUE_DIR)
+                    queue = 'bounce'
                 elif subaddress == 'confirm':
                     msgdata['toconfirm'] = True
-                    queue = Switchboard(config.CMDQUEUE_DIR)
+                    queue = 'command'
                 elif subaddress in ('join', 'subscribe'):
                     msgdata['tojoin'] = True
-                    queue = Switchboard(config.CMDQUEUE_DIR)
+                    queue = 'command'
                 elif subaddress in ('leave', 'unsubscribe'):
                     msgdata['toleave'] = True
-                    queue = Switchboard(config.CMDQUEUE_DIR)
+                    queue = 'command'
                 elif subaddress == 'owner':
                     msgdata.update(dict(
                         toowner=True,
-                        envsender=config.SITE_OWNER_ADDRESS,
-                        pipeline=config.OWNER_PIPELINE,
+                        envsender=config.mailman.site_owner,
+                        pipeline=Defaults.OWNER_PIPELINE,
                         ))
-                    queue = Switchboard(config.INQUEUE_DIR)
+                    queue = 'in'
                 elif subaddress is None:
                     msgdata['tolist'] = True
-                    queue = Switchboard(config.INQUEUE_DIR)
+                    queue = 'in'
                 elif subaddress == 'request':
                      msgdata['torequest'] = True
-                     queue = Switchboard(config.CMDQUEUE_DIR)
+                     queue = 'command'
                 else:
                     elog.error('Unknown sub-address: %s', subaddress)
                     status.append(ERR_550)
@@ -199,7 +201,7 @@ class LMTPRunner(Runner, smtpd.SMTPServer):
                 # If we found a valid subaddress, enqueue the message and add
                 # a success status for this recipient.
                 if queue is not None:
-                    queue.enqueue(msg, msgdata)
+                    config.switchboards[queue].enqueue(msg, msgdata)
                     status.append('250 Ok')
             except Exception, e:
                 elog.exception('Queue detection: %s', msg['message-id'])
diff --git a/mailman/rules/docs/emergency.txt b/mailman/rules/docs/emergency.txt
index 0f1005410..e71566853 100644
--- a/mailman/rules/docs/emergency.txt
+++ b/mailman/rules/docs/emergency.txt
@@ -27,8 +27,7 @@ There are two messages in the virgin queue.  The one addressed to the original
 sender will contain a token we can use to grab the held message out of the
 pending requests.
 
-    >>> from mailman.queue import Switchboard
-    >>> virginq = Switchboard(config.VIRGINQUEUE_DIR)
+    >>> virginq = config.switchboards['virgin']
 
     >>> def get_held_message():
     ...     import re
diff --git a/mailman/testing/helpers.py b/mailman/testing/helpers.py
index 1bcadedaf..c22d92aff 100644
--- a/mailman/testing/helpers.py
+++ b/mailman/testing/helpers.py
@@ -236,7 +236,8 @@ def get_lmtp_client():
     lmtp = LMTP()
     for attempts in range(3):
         try:
-            response = lmtp.connect(config.LMTP_HOST, config.LMTP_PORT)
+            response = lmtp.connect(
+                config.mta.lmtp_host, int(config.mta.lmtp_port))
             print response
             return lmtp
         except socket.error, error:
diff --git a/mailman/testing/layers.py b/mailman/testing/layers.py
index ce9f15736..5976d9e4e 100644
--- a/mailman/testing/layers.py
+++ b/mailman/testing/layers.py
@@ -31,10 +31,11 @@ import logging
 import tempfile
 
 from pkg_resources import resource_string
+from string import Template
 from textwrap import dedent
 
 from mailman.config import config
-from mailman.core.initialize import initialize
+from mailman.core import initialize
 from mailman.i18n import _
 from mailman.core.logging import get_handler
 from mailman.testing.helpers import SMTPServer
@@ -51,14 +52,21 @@ class ConfigLayer:
 
     @classmethod
     def setUp(cls):
-        initialize()
+        # Set up the basic configuration stuff.
+        initialize.initialize_1()
         assert cls.var_dir is None, 'Layer already set up'
         # Calculate a temporary VAR_DIR directory so that run-time artifacts
         # of the tests won't tread on the installation's data.  This also
         # makes it easier to clean up after the tests are done, and insures
         # isolation of test suite runs.
         cls.var_dir = tempfile.mkdtemp()
-        # Create a section with the var directory.
+        # We need a test configuration both for the foreground process and any
+        # child processes that get spawned.  lazr.config would allow us to do
+        # it all in a string that gets pushed, and we'll do that for the
+        # foreground, but because we may be spawning processes (such as queue
+        # runners) we'll need a file that we can specify to the with the -C
+        # option.  Craft the full test configuration string here, push it, and
+        # also write it out to a temp file for -C.
         test_config = dedent("""
         [mailman]
         var_dir: %s
@@ -66,10 +74,20 @@ class ConfigLayer:
         # Read the testing config and push it.
         test_config += resource_string('mailman.testing', 'testing.cfg')
         config.push('test config', test_config)
+        # Initialize everything else.
+        initialize.initialize_2()
+        initialize.initialize_3()
+        # When stderr debugging is enabled, subprocess root loggers should
+        # also be more verbose.
+        if cls.stderr:
+            test_config += dedent("""
+            [logging.root]
+            propagate: yes
+            level: debug
+            """)
         # Enable log message propagation and reset the log paths so that the
         # doctests can check the output.  XXX Switch to using the log support
         # in zope.testing.
-        os.makedirs(config.LOG_DIR)
         for logger_config in config.logger_configs:
             sub_name = logger_config.name.split('.')[-1]
             if sub_name == 'root':
@@ -84,6 +102,14 @@ class ConfigLayer:
             path = os.path.join(config.LOG_DIR, sub_name)
             get_handler(sub_name).reopen(path)
             log.setLevel(logging.DEBUG)
+            # If stderr debugging is enabled, make sure subprocesses are also
+            # more verbose.
+            if cls.stderr:
+                test_config += Template(dedent("""
+                [logging.$name]
+                propagate: yes
+                level: debug
+                """)).substitute(name=sub_name, path=path)
         # zope.testing sets up logging before we get to our own initialization
         # function.  This messes with the root logger, so explicitly set it to
         # go to stderr.
@@ -93,6 +119,13 @@ class ConfigLayer:
                                           config.logging.root.datefmt)
             console.setFormatter(formatter)
             logging.getLogger().addHandler(console)
+        # Write the configuration file for subprocesses and set up the config
+        # object to pass that properly on the -C option.
+        config_file = os.path.join(cls.var_dir, 'test.cfg')
+        with open(config_file, 'w') as fp:
+            fp.write(test_config)
+            print >> fp
+        config.filename = config_file
 
     @classmethod
     def tearDown(cls):