Rework the basic infrastructure for qrunner process control. Split out the

functionality of mailmanctl into a separate master watcher script.  mailmanctl
has not yet been updated but that'll happen next.

Fix DELIVERY_MODULE to name a handler instead of a module.

Change make_instance to use pkg_resources instead of module.__file__.

Change the qrunner and master processes coordination so that the qrunners are
not restarted on SIGINT, because otherwise C-c just doesn't work.  Now SIGUSR1
is how we'll implement 'mailman restart'.

Add a database commit so that initializing the schema doesn't lock the sqlite
database.  Also, don't try to initialize the schema if the tables already
exist.  Use some sqlite magic to do this test.

Move mailman.cfg.in into a new package Mailman/extras inside the tree.  Also,
MAILMAN_UID and MAILMAN_GID should be integers not strings.

Convert the command runner to use an IHandler instance instead of handler
module.  Similarly for the outgoing runner, DELIVERY_MODULE now names an
IHandler instance instead of a handler module.

13 files changed, 477 insertions, 77 deletions

diff --git a/Mailman/Defaults.py b/Mailman/Defaults.py
index 260f8d4ec..bc7c6b7a7 100644
--- a/Mailman/Defaults.py
+++ b/Mailman/Defaults.py
@@ -372,14 +372,8 @@ PUBLIC_MBOX = No
 
 # Final delivery module for outgoing mail.  This handler is used for message
 # delivery to the list via the smtpd, and to an individual user.  This value
-# must be a string naming a module in the Mailman.Handlers package.
-#
-# WARNING: Sendmail has security holes and should be avoided.  In fact, you
-# must read the Mailman/Handlers/Sendmail.py file before it will work for
-# you.
-#
-#DELIVERY_MODULE = 'Sendmail'
-DELIVERY_MODULE = 'SMTPDirect'
+# must be a string naming an IHandler.
+DELIVERY_MODULE = 'smtp-direct'
 
 # MTA should name a module in Mailman/MTA which provides the MTA specific
 # functionality for creating and removing lists.  Some MTAs like Exim can be
diff --git a/Mailman/bin/__init__.py b/Mailman/bin/__init__.py
index 4b94e05b0..f40c09d74 100644
--- a/Mailman/bin/__init__.py
+++ b/Mailman/bin/__init__.py
@@ -41,6 +41,7 @@ __all__ = [
     'list_owners',
     'mailmanctl',
     'make_instance',
+    'master',
     'mmsitepass',
     'newlist',
     'nightly_gzip',
diff --git a/Mailman/bin/mailmanctl.py b/Mailman/bin/mailmanctl.py
index 3be5d1849..2dc1905da 100644
--- a/Mailman/bin/mailmanctl.py
+++ b/Mailman/bin/mailmanctl.py
@@ -25,6 +25,8 @@ import socket
 import logging
 import optparse
 
+from datetime import timedelta
+from munepy import Enum
 from locknix import lockfile
 
 from Mailman import Defaults
@@ -62,17 +64,18 @@ sure that the various long-running qrunners are still alive and kicking.  It
 does this by forking and exec'ing the qrunners and waiting on their pids.
 When it detects a subprocess has exited, it may restart it.
 
-The qrunners respond to SIGINT, SIGTERM, and SIGHUP.  SIGINT and SIGTERM both
-cause the qrunners to exit cleanly, but the master will only restart qrunners
-that have exited due to a SIGINT.  SIGHUP causes the master and the qrunners
-to close their log files, and reopen then upon the next printed message.
+The qrunners respond to SIGINT, SIGTERM, SIGUSR1 and SIGHUP.  SIGINT, SIGTERM
+and SIGUSR1 all cause the qrunners to exit cleanly, but the master will only
+restart qrunners that have exited due to a SIGUSR1.  SIGHUP causes the master
+and the qrunners to close their log files, and reopen then upon the next
+printed message.
 
-The master also responds to SIGINT, SIGTERM, and SIGHUP, which it simply
-passes on to the qrunners (note that the master will close and reopen its own
-log files on receipt of a SIGHUP).  The master also leaves its own process id
-in the file data/master-qrunner.pid but you normally don't need to use this
-pid directly.  The `start', `stop', `restart', and `reopen' commands handle
-everything for you.
+The master also responds to SIGINT, SIGTERM, SIGUSR1 and SIGHUP, which it
+simply passes on to the qrunners (note that the master will close and reopen
+its own log files on receipt of a SIGHUP).  The master also leaves its own
+process id in the file data/master-qrunner.pid but you normally don't need to
+use this pid directly.  The `start', `stop', `restart', and `reopen' commands
+handle everything for you.
 
 Commands:
 
@@ -90,12 +93,6 @@ Commands:
               next time a message is written to them
 
 Usage: %prog [options] [ start | stop | restart | reopen ]"""))
-    parser.add_option('-n', '--no-restart',
-                      dest='restart', default=True, action='store_false',
-                      help=_("""\
-Don't restart the qrunners when they exit because of an error or a SIGINT.
-They are never restarted if they exit in response to a SIGTERM.  Use this only
-for debugging.  Only useful if the `start' command is given."""))
     parser.add_option('-u', '--run-as-user',
                       dest='checkprivs', default=True, action='store_false',
                       help=_("""\
@@ -202,13 +199,13 @@ def acquire_lock_1(force):
     # other master qrunner daemon is already going.
     lock = lockfile.Lock(config.LOCK_FILE, LOCK_LIFETIME)
     try:
-        lock.lock(0.1)
+        lock.lock(timedelta(seconds=0.1))
         return lock
     except lockfile.TimeOutError:
         if not force:
             raise
         # Force removal of lock first
-        lock._disown()
+        lock.disown()
         hostname, pid, tempfile = get_lock_data()
         os.unlink(config.LOCK_FILE)
         os.unlink(os.path.join(config.LOCK_DIR, tempfile))
@@ -319,19 +316,13 @@ def main():
     # Handle the commands
     command = args[0].lower()
     if command == 'stop':
-        # Sent the master qrunner process a SIGINT, which is equivalent to
-        # giving cron/qrunner a ctrl-c or KeyboardInterrupt.  This will
-        # effectively shut everything down.
         if not opts.quiet:
             print _("Shutting down Mailman's master qrunner")
         kill_watcher(signal.SIGTERM)
     elif command == 'restart':
-        # Sent the master qrunner process a SIGHUP.  This will cause the
-        # master qrunner to kill and restart all the worker qrunners, and to
-        # close and re-open its log files.
         if not opts.quiet:
             print _("Restarting Mailman's master qrunner")
-        kill_watcher(signal.SIGINT)
+        kill_watcher(signal.SIGUSR1)
     elif command == 'reopen':
         if not opts.quiet:
             print _('Re-opening all log files')
@@ -372,10 +363,10 @@ def main():
             # Give up the lock "ownership".  This just means the foreground
             # process won't close/unlock the lock when it finalizes this lock
             # instance.  We'll let the mater watcher subproc own the lock.
-            lock._transfer_to(pid)
+            lock.transfer_to(pid)
             return
         # child
-        lock._take_possession()
+        lock.take_possession()
         # Save our pid in a file for "mailmanctl stop" rendezvous.
         fp = open(config.PIDFILE, 'w')
         try:
@@ -413,8 +404,13 @@ def main():
             qlog.info('Master watcher caught SIGHUP.  Re-opening log files.')
         signal.signal(signal.SIGHUP, sighup_handler)
         # We also need to install a SIGTERM handler because that's what init
-        # will kill this process with when changing run levels.
+        # will kill this process with when changing run levels.  It's also the
+        # signal 'mailmanctl stop' uses.
         def sigterm_handler(signum, frame):
+            # Make sure we never try to restart our children, no matter why
+            # the child exited.
+            opts.restart = False
+            qlog.info('I AM NEVER RESTARTING AGAIN: %d', pid)
             for pid in kids.keys():
                 try:
                     os.kill(pid, signal.SIGTERM)
diff --git a/Mailman/bin/make_instance.py b/Mailman/bin/make_instance.py
index 26e799470..1f93c39bb 100644
--- a/Mailman/bin/make_instance.py
+++ b/Mailman/bin/make_instance.py
@@ -25,9 +25,9 @@ import errno
 import shutil
 import optparse
 import setuptools
-from string import Template
 
-import Mailman.data
+from pkg_resources import resource_string
+from string import Template
 
 from Mailman import Defaults
 from Mailman.Version import MAILMAN_VERSION
@@ -35,7 +35,6 @@ from Mailman.i18n import _
 
 
 SPACE = ' '
-DATA_DIR = os.path.dirname(Mailman.data.__file__)
 
 
 
@@ -131,15 +130,13 @@ def instantiate(var_dir, user, group, languages, force):
     # Create an etc/mailman.cfg file which contains just a few configuration
     # variables about the run-time environment that can't be calculated.
     # Don't overwrite mailman.cfg unless the -f flag was given.
-    in_file_path  = os.path.join(DATA_DIR, 'mailman.cfg.in')
     out_file_path = os.path.join(etc_dir, 'mailman.cfg')
     if os.path.exists(out_file_path) and not force:
         # The logging subsystem isn't up yet, so just print this to stderr.
         print >> sys.stderr, 'File exists:', out_file_path
         print >> sys.stderr, 'Use --force to override.'
     else:
-        with open(in_file_path) as fp:
-            raw = Template(fp.read())
+        raw = Template(resource_string('Mailman.extras', 'mailman.cfg.in'))
         processed = raw.safe_substitute(var_dir=var_dir,
                                         user_id=uid,
                                         user_name=user_name,
diff --git a/Mailman/bin/master.py b/Mailman/bin/master.py
new file mode 100644
index 000000000..6e7c5408d
--- /dev/null
+++ b/Mailman/bin/master.py
@@ -0,0 +1,393 @@
+# Copyright (C) 2001-2008 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+
+from __future__ import with_statement
+
+import os
+import grp
+import pwd
+import sys
+import errno
+import signal
+import socket
+import logging
+import optparse
+
+from datetime import timedelta
+from locknix import lockfile
+from munepy import Enum
+
+from Mailman import Defaults
+from Mailman import Version
+from Mailman import loginit
+from Mailman.configuration import config
+from Mailman.i18n import _
+from Mailman.initialize import initialize
+
+
+COMMASPACE = ', '
+DOT = '.'
+# Calculate this here and now, because we're going to do a chdir later on, and
+# if the path is relative, the qrunner script won't be found.
+BIN_DIR = os.path.abspath(os.path.dirname(sys.argv[0]))
+
+# Since we wake up once per day and refresh the lock, the LOCK_LIFETIME
+# needn't be (much) longer than SNOOZE.  We pad it 6 hours just to be safe.
+LOCK_LIFETIME = Defaults.days(1) + Defaults.hours(6)
+SNOOZE = Defaults.days(1)
+
+log = None
+parser = None
+
+
+
+def parseargs():
+    parser = optparse.OptionParser(version=Version.MAILMAN_VERSION,
+                                   usage=_("""\
+Master queue runner watcher.
+
+Start and watch the configured queue runners and ensure that they stay alive
+and kicking.  Each are fork and exec'd in turn, with the master waiting on
+their process ids.  When it detects a child queue runner has exited, it may
+restart it.
+
+The queue runners respond to SIGINT, SIGTERM, SIGUSR1 and SIGHUP.  SIGINT,
+SIGTERM and SIGUSR1 all cause the qrunners to exit cleanly.  The master will
+restart qrunners that have exited due to a SIGUSR1 or some kind of other exit
+condition (say because of an exception).  SIGHUP causes the master and the
+qrunners to close their log files, and reopen then upon the next printed
+message.
+
+The master also responds to SIGINT, SIGTERM, SIGUSR1 and SIGHUP, which it
+simply passes on to the qrunners.  Note that the master will close and reopen
+its own log files on receipt of a SIGHUP.  The master also leaves its own
+process id in the file `data/master-qrunner.pid` but you normally don't need
+to use this pid directly.
+
+Usage: %prog [options]"""))
+    parser.add_option('-n', '--no-restart',
+                      dest='restartable', default=True, action='store_false',
+                      help=_("""\
+Don't restart the qrunners when they exit because of an error or a SIGUSR1.
+Use this only for debugging."""))
+    parser.add_option('-C', '--config',
+                      help=_('Alternative configuration file to use'))
+    options, arguments = parser.parse_args()
+    if len(arguments) > 0:
+        parser.error(_('Too many arguments'))
+    parser.options = options
+    parser.arguments = arguments
+    return parser
+
+
+
+def get_lock_data():
+    """Get information from the master lock file.
+
+    :return: A 3-tuple of the hostname, integer process id, and file name of
+        the lock file.
+    """
+    with open(config.LOCK_FILE) as fp:
+        filename = os.path.split(fp.read().strip())[1]
+    parts = filename.split('.')
+    hostname = DOT.join(parts[1:-1])
+    pid = int(parts[-1])
+    return hostname, int(pid), filename
+
+
+class WatcherState(Enum):
+    # Another master watcher is running.
+    conflict = 1
+    # No conflicting process exists.
+    stale_lock = 2
+    # Hostname from lock file doesn't match.
+    host_mismatch = 3
+
+
+def master_state():
+    """Get the state of the master watcher.
+
+    :return: WatcherState describing the state of the lock file.
+    """
+
+    # 1 if proc exists on host (but is it qrunner? ;)
+    # 0 if host matches but no proc
+    # hostname if hostname doesn't match
+    hostname, pid, tempfile = get_lock_data()
+    if hostname <> socket.gethostname():
+        return WatcherState.host_mismatch
+    # Find out if the process exists by calling kill with a signal 0.
+    try:
+        os.kill(pid, 0)
+        return WatcherState.conflict
+    except OSError, e:
+        if e.errno == errno.ESRCH:
+            # No matching process id.
+            return WatcherState.stale_lock
+        # Some other error occurred.
+        raise
+
+
+def acquire_lock_1(force):
+    """Try to acquire the master queue runner lock.
+
+    :param force: Flag that controls whether to force acquisition of the lock.
+    :return: The master queue runner lock.
+    :raises: `TimeOutError` if the lock could not be acquired.
+    """
+    lock = lockfile.Lock(config.LOCK_FILE, LOCK_LIFETIME)
+    try:
+        lock.lock(timedelta(seconds=0.1))
+        return lock
+    except lockfile.TimeOutError:
+        if not force:
+            raise
+        # Force removal of lock first.
+        lock.disown()
+        hostname, pid, tempfile = get_lock_data()
+        os.unlink(config.LOCK_FILE)
+        os.unlink(os.path.join(config.LOCK_DIR, tempfile))
+        return acquire_lock_1(force=False)
+
+
+def acquire_lock(force):
+    """Acquire the master queue runner lock.
+
+    :param force: Flag that controls whether to force acquisition of the lock.
+    :return: The master queue runner lock or None if the lock couldn't be
+        acquired.  In that case, an error messages is also printed to standard
+        error.
+    """
+    try:
+        lock = acquire_lock_1(force)
+        return lock
+    except lockfile.TimeOutError:
+        status = master_state()
+        if status == WatcherState.conflict:
+            # Hostname matches and process exists.
+            print >> sys.stderr, _("""\
+The master qrunner lock could not be acquired because it appears as if another
+master qrunner is already running.
+""")
+        elif status == WatcherState.stale_lock:
+            # Hostname matches but the process does not exist.
+            print >> sys.stderr, _("""\
+The master qrunner lock could not be acquired.  It appears as though there is
+a stale master qrunner lock.  Try re-running mailmanctl with the -s flag.
+""")
+        else:
+            assert status == WatcherState.host_mismatch, (
+                'Invalid enum value: %s' % status)
+            # Hostname doesn't even match.
+            print >> sys.stderr, _("""\
+The master qrunner lock could not be acquired, because it appears as if some
+process on some other host may have acquired it.  We can't test for stale
+locks across host boundaries, so you'll have to do this manually.  Or, if you
+know the lock is stale, re-run mailmanctl with the -s flag.
+
+Lock file: $config.LOCK_FILE
+Lock host: $status
+
+Exiting.""")
+        return None
+
+
+
+def start_runner(qrname, slice, count):
+    """Start a queue runner.
+
+    All arguments are passed to the qrunner process.
+
+    :param qrname: The name of the queue runner.
+    :param slice: The slice number.
+    :param count: The total number of slices.
+    :return: The process id of the child queue runner.
+    """
+    pid = os.fork()
+    if pid:
+        # Parent.
+        return pid
+    # Child.
+    #
+    # Craft the command line arguments for the exec() call.
+    rswitch = '--runner=%s:%d:%d' % (qrname, slice, count)
+    # Wherever mailmanctl lives, so too must live the qrunner script.
+    exe = os.path.join(BIN_DIR, 'qrunner')
+    # config.PYTHON, which is the absolute path to the Python interpreter,
+    # must be given as argv[0] due to Python's library search algorithm.
+    args = [sys.executable, sys.executable, exe, rswitch, '-s']
+    if parser.options.config:
+        args.extend(['-C', parser.options.config])
+    log.debug('starting: %s', args)
+    os.execl(*args)
+    # We should never get here.
+    raise RuntimeError('os.execl() failed')
+
+
+
+def control_loop(lock):
+    """The main control loop.
+
+    This starts up the queue runners, watching for their exit and restarting
+    them if need be.
+    """
+    restartable = parser.options.restartable
+    # Start all the qrunners.  Keep a dictionary mapping process ids to
+    # information about the child processes.
+    kids = {}
+    # Set up our signal handlers.  Also set up a SIGALRM handler to refresh
+    # the lock once per day.  The lock lifetime is 1 day + 6 hours so this
+    # should be plenty.
+    def sigalrm_handler(signum, frame):
+        lock.refresh()
+        signal.alarm(int(Defaults.days(1)))
+    signal.signal(signal.SIGALRM, sigalrm_handler)
+    signal.alarm(int(Defaults.days(1)))
+    # SIGHUP tells the qrunners to close and reopen their log files.
+    def sighup_handler(signum, frame):
+        loginit.reopen()
+        for pid in kids:
+            os.kill(pid, signal.SIGHUP)
+        log.info('Master watcher caught SIGHUP.  Re-opening log files.')
+    signal.signal(signal.SIGHUP, sighup_handler)
+    # SIGUSR1 is used by 'mailman restart'.
+    def sigusr1_handler(signum, frame):
+        for pid in kids:
+            os.kill(pid, signal.SIGUSR1)
+        log.info('Master watcher caught SIGUSR1.  Exiting.')
+    signal.signal(signal.SIGUSR1, sigusr1_handler)
+    # SIGTERM is what init will kill this process with when changing run
+    # levels.  It's also the signal 'mailmanctl stop' uses.
+    def sigterm_handler(signum, frame):
+        for pid in kids:
+            os.kill(pid, signal.SIGTERM)
+        log.info('Master watcher caught SIGTERM.  Exiting.')
+    signal.signal(signal.SIGTERM, sigterm_handler)
+    # SIGINT is what control-C gives.
+    def sigint_handler(signum, frame):
+        for pid in kids:
+            os.kill(pid, signal.SIGINT)
+        log.info('Master watcher caught SIGINT.  Restarting.')
+    signal.signal(signal.SIGINT, sigint_handler)
+    # Start all the child qrunners.
+    for qrname, count in config.qrunners.items():
+        for slice_number in range(count):
+            # queue runner name, slice number, number of slices, restart count
+            info = (qrname, slice_number, count, 0)
+            pid = start_runner(qrname, slice_number, count)
+            kids[pid] = info
+    # Enter the main wait loop.
+    try:
+        while True:
+            try:
+                pid, status = os.wait()
+            except OSError, error:
+                # No children?  We're done.
+                if error.errno == errno.ECHILD:
+                    break
+                # If the system call got interrupted, just restart it.
+                elif error.errno == errno.EINTR:
+                    continue
+                else:
+                    raise
+            # Find out why the subprocess exited by getting the signal
+            # received or exit status.
+            if os.WIFSIGNALED(status):
+                why = os.WTERMSIG(status)
+            elif os.WIFEXITED(status):
+                why = os.WEXITSTATUS(status)
+            else:
+                why = None
+            # We'll restart the subprocess if it exited with a SIGUSR1 or
+            # because of a failure (i.e. no exit signal), and the no-restart
+            # command line switch was not given.  This lets us better handle
+            # runaway restarts (e.g.  if the subprocess had a syntax error!)
+            qrname, slice, count, restarts = kids.pop(pid)
+            restart = False
+            if why == signal.SIGUSR1 and restartable:
+                restart = True
+            # Have we hit the maximum number of restarts?
+            restarts += 1
+            if restarts > config.MAX_RESTARTS:
+                restart = False
+            # Are we permanently non-restartable?
+            log.debug("""\
+Master detected subprocess exit
+(pid: %d, why: %s, class: %s, slice: %d/%d) %s""",
+                     pid, why, qrname, slice+1, count,
+                     ('[restarting]' if restart else ''))
+            # See if we've reached the maximum number of allowable restarts
+            if restarts > config.MAX_RESTARTS:
+                log.info("""\
+qrunner %s reached maximum restart limit of %d, not restarting.""",
+                         qrname, config.MAX_RESTARTS)
+            # Now perhaps restart the process unless it exited with a
+            # SIGTERM or we aren't restarting.
+            if restart:
+                newpid = start_runner(qrname, slice, count)
+                kids[newpid] = (qrname, slice, count, restarts)
+    finally:
+        # Should we leave the main loop for any reason, we want to be sure
+        # all of our children are exited cleanly.  Send SIGTERMs to all
+        # the child processes and wait for them all to exit.
+        for pid in kids:
+            try:
+                os.kill(pid, signal.SIGTERM)
+            except OSError, error:
+                if error.errno == errno.ESRCH:
+                    # The child has already exited.
+                    log.info('ESRCH on pid: %d', pid)
+        # Wait for all the children to go away.
+        while True:
+            try:
+                pid, status = os.wait()
+            except OSError, e:
+                if e.errno == errno.ECHILD:
+                    break
+                elif e.errno == errno.EINTR:
+                    continue
+                raise
+
+
+
+def main():
+    """Main process."""
+    global log, parser
+
+    parser = parseargs()
+    initialize(parser.options.config)
+
+    # We can't grab the logger until after everything's been initialized.
+    log = logging.getLogger('mailman.qrunner')
+
+    # Acquire the master lock, exiting if we can't acquire it.  We'll let the
+    # caller handle any clean up or lock breaking.
+    with lockfile.Lock(config.LOCK_FILE, LOCK_LIFETIME) as lock:
+        with open(config.PIDFILE, 'w') as fp:
+            print >> fp, os.getpid()
+        try:
+            control_loop(lock)
+        finally:
+            os.remove(config.PIDFILE)
+
+
+
+if __name__ == '__main__':
+    try:
+        main()
+    except KeyboardInterrupt:
+        pass
diff --git a/Mailman/bin/qrunner.py b/Mailman/bin/qrunner.py
index 91ecb2ca3..b73643c9d 100644
--- a/Mailman/bin/qrunner.py
+++ b/Mailman/bin/qrunner.py
@@ -95,7 +95,7 @@ each listed by the -l option."""))
     parser.add_option('-o', '--once',
                       default=False, action='store_true', help=_("""\
 Run each named qrunner exactly once through its main loop.  Otherwise, each
-qrunner runs indefinitely, until the process receives a SIGTERM or SIGINT."""))
+qrunner runs indefinitely, until the process receives signal."""))
     parser.add_option('-l', '--list',
                       default=False, action='store_true',
                       help=_('List the available qrunner names and exit.'))
@@ -134,13 +134,13 @@ def make_qrunner(name, slice, range, once=False):
         __import__(modulename)
     except ImportError, e:
         if opts.subproc:
-            # Exit with SIGTERM exit code so mailmanctl won't try to restart us
+            # Exit with SIGTERM exit code so the master watcher won't try to
+            # restart us.
             print >> sys.stderr, _('Cannot import runner module: $modulename')
             print >> sys.stderr, e
             sys.exit(signal.SIGTERM)
         else:
-            print >> sys.stderr, e
-            sys.exit(1)
+            raise
     qrclass = getattr(sys.modules[modulename], classname)
     if once:
         # Subclass to hack in the setting of the stop flag in _doperiodic()
@@ -155,22 +155,34 @@ def make_qrunner(name, slice, range, once=False):
 
 
 def set_signals(loop):
-    # Set up the SIGTERM handler for stopping the loop
+    """Set up the signal handlers.
+
+    Signals caught are: SIGTERM, SIGINT, SIGUSR1 and SIGHUP.  The latter is
+    used to re-open the log files.  SIGTERM and SIGINT are treated exactly the
+    same -- they cause qrunner to exit with no restart from the master.
+    SIGUSR1 also causes qrunner to exit, but the master watcher will restart
+    it in that case.
+
+    :param loop: A loop queue runner instance.
+    """
     def sigterm_handler(signum, frame):
         # Exit the qrunner cleanly
         loop.stop()
         loop.status = signal.SIGTERM
         log.info('%s qrunner caught SIGTERM.  Stopping.', loop.name())
     signal.signal(signal.SIGTERM, sigterm_handler)
-    # Set up the SIGINT handler for stopping the loop.  For us, SIGINT is
-    # the same as SIGTERM, but our parent treats the exit statuses
-    # differently (it restarts a SIGINT but not a SIGTERM).
     def sigint_handler(signum, frame):
         # Exit the qrunner cleanly
         loop.stop()
         loop.status = signal.SIGINT
         log.info('%s qrunner caught SIGINT.  Stopping.', loop.name())
     signal.signal(signal.SIGINT, sigint_handler)
+    def sigusr1_handler(signum, frame):
+        # Exit the qrunner cleanly
+        loop.stop()
+        loop.status = signal.SIGUSR1
+        log.info('%s qrunner caught SIGUSR1.  Stopping.', loop.name())
+    signal.signal(signal.SIGUSR1, sigusr1_handler)
     # SIGHUP just tells us to rotate our log files.
     def sighup_handler(signum, frame):
         loginit.reopen()
diff --git a/Mailman/database/__init__.py b/Mailman/database/__init__.py
index 7ca60f145..63bdd110c 100644
--- a/Mailman/database/__init__.py
+++ b/Mailman/database/__init__.py
@@ -23,7 +23,6 @@ __all__ = [
     ]
 
 import os
-import Mailman.Version
 
 from locknix.lockfile import Lock
 from storm.locals import create_database, Store
@@ -31,6 +30,9 @@ from string import Template
 from urlparse import urlparse
 from zope.interface import implements
 
+import Mailman.Version
+import Mailman.database
+
 from Mailman.configuration import config
 from Mailman.database.listmanager import ListManager
 from Mailman.database.messagestore import MessageStore
@@ -87,17 +89,23 @@ class StockDatabase:
         store = Store(database)
         database.DEBUG = (config.DEFAULT_DATABASE_ECHO
                           if debug is None else debug)
+        # Check the sqlite master database to see if the version file exists.
+        # If so, then we assume the database schema is correctly initialized.
         # Storm does not currently have schema creation.  This is not an ideal
         # way to handle creating the database, but it's cheap and easy for
         # now.
-        import Mailman.database
-        schema_file = os.path.join(
-            os.path.dirname(Mailman.database.__file__),
-            'mailman.sql')
-        with open(schema_file) as fp:
-            sql = fp.read()
-        for statement in sql.split(';'):
-            store.execute(statement + ';')
+        table_names = [item[0] for item in 
+                       store.execute('select tbl_name from sqlite_master;')]
+        if 'version' not in table_names:
+            # Initialize the database.
+            schema_file = os.path.join(
+                os.path.dirname(Mailman.database.__file__),
+                'mailman.sql')
+            with open(schema_file) as fp:
+                sql = fp.read()
+            for statement in sql.split(';'):
+                store.execute(statement + ';')
+            store.commit()
         # Validate schema version.
         v = store.find(Version, component=u'schema').one()
         if not v:
diff --git a/Mailman/extras/__init__.py b/Mailman/extras/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/Mailman/extras/__init__.py
diff --git a/data/mailman.cfg.in b/Mailman/extras/mailman.cfg.in
index 3e24acc46..bb51be97c 100644
--- a/data/mailman.cfg.in
+++ b/Mailman/extras/mailman.cfg.in
@@ -1,6 +1,6 @@
 # -*- python -*-
 
-# Copyright (C) 2006-2007 by the Free Software Foundation, Inc.
+# Copyright (C) 2006-2008 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -27,7 +27,8 @@ automatically be available in this module's namespace.
 You should consult `Defaults.py` though for a complete listing of
 configuration variables that you can change.
 
-Mailman's installation procedure will never overwrite `mailman.cfg`.
+Mailman's installation procedure will never overwrite `mailman.cfg` unless you
+tell it to.
 """
 
 # This is the top-level run-time data directory.  All other runtime data by
@@ -35,11 +36,11 @@ Mailman's installation procedure will never overwrite `mailman.cfg`.
 VAR_DIR = '$var_dir'
 
 # User name and id that owns the Mailman process and files.
-MAILMAN_UID  = '$user_id'
+MAILMAN_UID  = $user_id
 MAILMAN_USER = '$user_name'
 
 # Group name and id that owns the Mailman process and files.
-MAILMAN_GID   = '$group_id'
+MAILMAN_GID   = $group_id
 MAILMAN_GROUP = '$group_name'
 
 LANGUAGES = '$languages'
diff --git a/Mailman/queue/command.py b/Mailman/queue/command.py
index 8217acad6..a0db2f776 100644
--- a/Mailman/queue/command.py
+++ b/Mailman/queue/command.py
@@ -33,7 +33,6 @@ from email.MIMEText import MIMEText
 
 from Mailman import Message
 from Mailman import Utils
-from Mailman.Handlers import Replybot
 from Mailman.app.replybot import autorespond_to_sender
 from Mailman.configuration import config
 from Mailman.i18n import _
@@ -206,7 +205,8 @@ class CommandRunner(Runner):
             return False
         # Do replybot for commands
         mlist.Load()
-        Replybot.process(mlist, msg, msgdata)
+        replybot = config.handlers['replybot']
+        replybot.process(mlist, msg, msgdata)
         if mlist.autorespond_requests == 1:
             log.info('replied and discard')
             # w/discard
diff --git a/Mailman/queue/outgoing.py b/Mailman/queue/outgoing.py
index 09572ba9b..a3d60f0a0 100644
--- a/Mailman/queue/outgoing.py
+++ b/Mailman/queue/outgoing.py
@@ -46,14 +46,13 @@ class OutgoingRunner(Runner, BounceMixin):
         Runner.__init__(self, slice, numslices)
         BounceMixin.__init__(self)
         # We look this function up only at startup time
-        modname = 'Mailman.Handlers.' + config.DELIVERY_MODULE
-        mod = __import__(modname)
-        self._func = getattr(sys.modules[modname], 'process')
+        handler = config.handlers[config.DELIVERY_MODULE]
+        self._func = handler.process
         # This prevents smtp server connection problems from filling up the
         # error log.  It gets reset if the message was successfully sent, and
         # set if there was a socket.error.
-        self.__logged = False
-        self.__retryq = Switchboard(config.RETRYQUEUE_DIR)
+        self._logged = False
+        self._retryq = Switchboard(config.RETRYQUEUE_DIR)
 
     def _dispose(self, mlist, msg, msgdata):
         # See if we should retry delivery of this message again.
@@ -69,7 +68,7 @@ class OutgoingRunner(Runner, BounceMixin):
             if pid <> os.getpid():
                 log.error('child process leaked thru: %s', modname)
                 os._exit(1)
-            self.__logged = False
+            self._logged = False
         except socket.error:
             # There was a problem connecting to the SMTP server.  Log this
             # once, but crank up our sleep time so we don't fill the error
@@ -78,10 +77,10 @@ class OutgoingRunner(Runner, BounceMixin):
             if port == 0:
                 port = 'smtp'
             # Log this just once.
-            if not self.__logged:
+            if not self._logged:
                 log.error('Cannot connect to SMTP server %s on port %s',
                           config.SMTPHOST, port)
-                self.__logged = True
+                self._logged = True
             return True
         except Errors.SomeRecipientsFailed, e:
             # Handle local rejects of probe messages differently.
@@ -120,7 +119,7 @@ class OutgoingRunner(Runner, BounceMixin):
                     msgdata['last_recip_count'] = len(recips)
                     msgdata['deliver_until'] = deliver_until
                     msgdata['recips'] = recips
-                    self.__retryq.enqueue(msg, msgdata)
+                    self._retryq.enqueue(msg, msgdata)
         # We've successfully completed handling of this message
         return False
 
diff --git a/Mailman/queue/pipeline.py b/Mailman/queue/pipeline.py
index 14bfea56c..47c2f9e6f 100644
--- a/Mailman/queue/pipeline.py
+++ b/Mailman/queue/pipeline.py
@@ -22,9 +22,9 @@ through the 'preparation pipeline'.  This pipeline adds, deletes and modifies
 headers, calculates message recipients, and more.
 """
 
-from Mailman.app.pipeline import process
+from Mailman.app.pipelines import process
 from Mailman.configuration import config
-from Mailman.queue import runner
+from Mailman.queue import Runner
 
 
 
diff --git a/setup.py b/setup.py
index 9cb6370b1..983e7a2a7 100644
--- a/setup.py
+++ b/setup.py
@@ -82,7 +82,6 @@ Any other spelling is incorrect.""",
     include_package_data = True,
     entry_points    = {
         'console_scripts': list(scripts),
-        'setuptools.file_finders': 'bzr = setuptools_bzr:find_files_for_bzr',
         # Entry point for plugging in different database backends.
         'mailman.archiver'  : 'stock = Mailman.app.archiving:Pipermail',
         'mailman.database'  : 'stock = Mailman.database:StockDatabase',