13 files changed, 477 insertions, 77 deletions

diff --git a/Mailman/Defaults.py b/Mailman/Defaults.py
index 260f8d4ec..bc7c6b7a7 100644
--- a/Mailman/Defaults.py
+++ b/Mailman/Defaults.py
@@ -372,14 +372,8 @@ PUBLIC_MBOX = No
 
 # Final delivery module for outgoing mail.  This handler is used for message
 # delivery to the list via the smtpd, and to an individual user.  This value
-# must be a string naming a module in the Mailman.Handlers package.
-#
-# WARNING: Sendmail has security holes and should be avoided.  In fact, you
-# must read the Mailman/Handlers/Sendmail.py file before it will work for
-# you.
-#
-#DELIVERY_MODULE = 'Sendmail'
-DELIVERY_MODULE = 'SMTPDirect'
+# must be a string naming an IHandler.
+DELIVERY_MODULE = 'smtp-direct'
 
 # MTA should name a module in Mailman/MTA which provides the MTA specific
 # functionality for creating and removing lists.  Some MTAs like Exim can be
diff --git a/Mailman/bin/__init__.py b/Mailman/bin/__init__.py
index 4b94e05b0..f40c09d74 100644
--- a/Mailman/bin/__init__.py
+++ b/Mailman/bin/__init__.py
@@ -41,6 +41,7 @@ __all__ = [
     'list_owners',
     'mailmanctl',
     'make_instance',
+    'master',
     'mmsitepass',
     'newlist',
     'nightly_gzip',
diff --git a/Mailman/bin/mailmanctl.py b/Mailman/bin/mailmanctl.py
index 3be5d1849..2dc1905da 100644
--- a/Mailman/bin/mailmanctl.py
+++ b/Mailman/bin/mailmanctl.py
@@ -25,6 +25,8 @@ import socket
 import logging
 import optparse
 
+from datetime import timedelta
+from munepy import Enum
 from locknix import lockfile
 
 from Mailman import Defaults
@@ -62,17 +64,18 @@ sure that the various long-running qrunners are still alive and kicking.  It
 does this by forking and exec'ing the qrunners and waiting on their pids.
 When it detects a subprocess has exited, it may restart it.
 
-The qrunners respond to SIGINT, SIGTERM, and SIGHUP.  SIGINT and SIGTERM both
-cause the qrunners to exit cleanly, but the master will only restart qrunners
-that have exited due to a SIGINT.  SIGHUP causes the master and the qrunners
-to close their log files, and reopen then upon the next printed message.
+The qrunners respond to SIGINT, SIGTERM, SIGUSR1 and SIGHUP.  SIGINT, SIGTERM
+and SIGUSR1 all cause the qrunners to exit cleanly, but the master will only
+restart qrunners that have exited due to a SIGUSR1.  SIGHUP causes the master
+and the qrunners to close their log files, and reopen then upon the next
+printed message.
 
-The master also responds to SIGINT, SIGTERM, and SIGHUP, which it simply
-passes on to the qrunners (note that the master will close and reopen its own
-log files on receipt of a SIGHUP).  The master also leaves its own process id
-in the file data/master-qrunner.pid but you normally don't need to use this
-pid directly.  The `start', `stop', `restart', and `reopen' commands handle
-everything for you.
+The master also responds to SIGINT, SIGTERM, SIGUSR1 and SIGHUP, which it
+simply passes on to the qrunners (note that the master will close and reopen
+its own log files on receipt of a SIGHUP).  The master also leaves its own
+process id in the file data/master-qrunner.pid but you normally don't need to
+use this pid directly.  The `start', `stop', `restart', and `reopen' commands
+handle everything for you.
 
 Commands:
 
@@ -90,12 +93,6 @@ Commands:
               next time a message is written to them
 
 Usage: %prog [options] [ start | stop | restart | reopen ]"""))
-    parser.add_option('-n', '--no-restart',
-                      dest='restart', default=True, action='store_false',
-                      help=_("""\
-Don't restart the qrunners when they exit because of an error or a SIGINT.
-They are never restarted if they exit in response to a SIGTERM.  Use this only
-for debugging.  Only useful if the `start' command is given."""))
     parser.add_option('-u', '--run-as-user',
                       dest='checkprivs', default=True, action='store_false',
                       help=_("""\
@@ -202,13 +199,13 @@ def acquire_lock_1(force):
     # other master qrunner daemon is already going.
     lock = lockfile.Lock(config.LOCK_FILE, LOCK_LIFETIME)
     try:
-        lock.lock(0.1)
+        lock.lock(timedelta(seconds=0.1))
         return lock
     except lockfile.TimeOutError:
         if not force:
             raise
         # Force removal of lock first
-        lock._disown()
+        lock.disown()
         hostname, pid, tempfile = get_lock_data()
         os.unlink(config.LOCK_FILE)
         os.unlink(os.path.join(config.LOCK_DIR, tempfile))
@@ -319,19 +316,13 @@ def main():
     # Handle the commands
     command = args[0].lower()
     if command == 'stop':
-        # Sent the master qrunner process a SIGINT, which is equivalent to
-        # giving cron/qrunner a ctrl-c or KeyboardInterrupt.  This will
-        # effectively shut everything down.
         if not opts.quiet:
             print _("Shutting down Mailman's master qrunner")
         kill_watcher(signal.SIGTERM)
     elif command == 'restart':
-        # Sent the master qrunner process a SIGHUP.  This will cause the
-        # master qrunner to kill and restart all the worker qrunners, and to
-        # close and re-open its log files.
         if not opts.quiet:
             print _("Restarting Mailman's master qrunner")
-        kill_watcher(signal.SIGINT)
+        kill_watcher(signal.SIGUSR1)
     elif command == 'reopen':
         if not opts.quiet:
             print _('Re-opening all log files')
@@ -372,10 +363,10 @@ def main():
             # Give up the lock "ownership".  This just means the foreground
             # process won't close/unlock the lock when it finalizes this lock
             # instance.  We'll let the mater watcher subproc own the lock.
-            lock._transfer_to(pid)
+            lock.transfer_to(pid)
             return
         # child
-        lock._take_possession()
+        lock.take_possession()
         # Save our pid in a file for "mailmanctl stop" rendezvous.
         fp = open(config.PIDFILE, 'w')
         try:
@@ -413,8 +404,13 @@ def main():
             qlog.info('Master watcher caught SIGHUP.  Re-opening log files.')
         signal.signal(signal.SIGHUP, sighup_handler)
         # We also need to install a SIGTERM handler because that's what init
-        # will kill this process with when changing run levels.
+        # will kill this process with when changing run levels.  It's also the
+        # signal 'mailmanctl stop' uses.
         def sigterm_handler(signum, frame):
+            # Make sure we never try to restart our children, no matter why
+            # the child exited.
+            opts.restart = False
+            qlog.info('I AM NEVER RESTARTING AGAIN: %d', pid)
             for pid in kids.keys():
                 try:
                     os.kill(pid, signal.SIGTERM)
diff --git a/Mailman/bin/make_instance.py b/Mailman/bin/make_instance.py
index 26e799470..1f93c39bb 100644
--- a/Mailman/bin/make_instance.py
+++ b/Mailman/bin/make_instance.py
@@ -25,9 +25,9 @@ import errno
 import shutil
 import optparse
 import setuptools
-from string import Template
 
-import Mailman.data
+from pkg_resources import resource_string
+from string import Template
 
 from Mailman import Defaults
 from Mailman.Version import MAILMAN_VERSION
@@ -35,7 +35,6 @@ from Mailman.i18n import _
 
 
 SPACE = ' '
-DATA_DIR = os.path.dirname(Mailman.data.__file__)
 
 
 
@@ -131,15 +130,13 @@ def instantiate(var_dir, user, group, languages, force):
     # Create an etc/mailman.cfg file which contains just a few configuration
     # variables about the run-time environment that can't be calculated.
     # Don't overwrite mailman.cfg unless the -f flag was given.
-    in_file_path  = os.path.join(DATA_DIR, 'mailman.cfg.in')
     out_file_path = os.path.join(etc_dir, 'mailman.cfg')
     if os.path.exists(out_file_path) and not force:
         # The logging subsystem isn't up yet, so just print this to stderr.
         print >> sys.stderr, 'File exists:', out_file_path
         print >> sys.stderr, 'Use --force to override.'
     else:
-        with open(in_file_path) as fp:
-            raw = Template(fp.read())
+        raw = Template(resource_string('Mailman.extras', 'mailman.cfg.in'))
         processed = raw.safe_substitute(var_dir=var_dir,
                                         user_id=uid,
                                         user_name=user_name,
diff --git a/Mailman/bin/master.py b/Mailman/bin/master.py
new file mode 100644
index 000000000..6e7c5408d
--- /dev/null
+++ b/Mailman/bin/master.py
@@ -0,0 +1,393 @@
+# Copyright (C) 2001-2008 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+
+from __future__ import with_statement
+
+import os
+import grp
+import pwd
+import sys
+import errno
+import signal
+import socket
+import logging
+import optparse
+
+from datetime import timedelta
+from locknix import lockfile
+from munepy import Enum
+
+from Mailman import Defaults
+from Mailman import Version
+from Mailman import loginit
+from Mailman.configuration import config
+from Mailman.i18n import _
+from Mailman.initialize import initialize
+
+
+COMMASPACE = ', '
+DOT = '.'
+# Calculate this here and now, because we're going to do a chdir later on, and
+# if the path is relative, the qrunner script won't be found.
+BIN_DIR = os.path.abspath(os.path.dirname(sys.argv[0]))
+
+# Since we wake up once per day and refresh the lock, the LOCK_LIFETIME
+# needn't be (much) longer than SNOOZE.  We pad it 6 hours just to be safe.
+LOCK_LIFETIME = Defaults.days(1) + Defaults.hours(6)
+SNOOZE = Defaults.days(1)
+
+log = None
+parser = None
+
+
+
+def parseargs():
+    parser = optparse.OptionParser(version=Version.MAILMAN_VERSION,
+                                   usage=_("""\
+Master queue runner watcher.
+
+Start and watch the configured queue runners and ensure that they stay alive
+and kicking.  Each are fork and exec'd in turn, with the master waiting on
+their process ids.  When it detects a child queue runner has exited, it may
+restart it.
+
+The queue runners respond to SIGINT, SIGTERM, SIGUSR1 and SIGHUP.  SIGINT,
+SIGTERM and SIGUSR1 all cause the qrunners to exit cleanly.  The master will
+restart qrunners that have exited due to a SIGUSR1 or some kind of other exit
+condition (say because of an exception).  SIGHUP causes the master and the
+qrunners to close their log files, and reopen then upon the next printed
+message.
+
+The master also responds to SIGINT, SIGTERM, SIGUSR1 and SIGHUP, which it
+simply passes on to the qrunners.  Note that the master will close and reopen
+its own log files on receipt of a SIGHUP.  The master also leaves its own
+process id in the file `data/master-qrunner.pid` but you normally don't need
+to use this pid directly.
+
+Usage: %prog [options]"""))
+    parser.add_option('-n', '--no-restart',
+                      dest='restartable', default=True, action='store_false',
+                      help=_("""\
+Don't restart the qrunners when they exit because of an error or a SIGUSR1.
+Use this only for debugging."""))
+    parser.add_option('-C', '--config',
+                      help=_('Alternative configuration file to use'))
+    options, arguments = parser.parse_args()
+    if len(arguments) > 0:
+        parser.error(_('Too many arguments'))
+    parser.options = options
+    parser.arguments = arguments
+    return parser
+
+
+
+def get_lock_data():
+    """Get information from the master lock file.
+
+    :return: A 3-tuple of the hostname, integer process id, and file name of
+        the lock file.
+    """
+    with open(config.LOCK_FILE) as fp:
+        filename = os.path.split(fp.read().strip())[1]
+    parts = filename.split('.')
+    hostname = DOT.join(parts[1:-1])
+    pid = int(parts[-1])
+    return hostname, int(pid), filename
+
+
+class WatcherState(Enum):
+    # Another master watcher is running.
+    conflict = 1
+    # No conflicting process exists.
+    stale_lock = 2
+    # Hostname from lock file doesn't match.
+    host_mismatch = 3
+
+
+def master_state():
+    """Get the state of the master watcher.
+
+    :return: WatcherState describing the state of the lock file.
+    """
+
+    # 1 if proc exists on host (but is it qrunner? ;)
+    # 0 if host matches but no proc
+    # hostname if hostname doesn't match
+    hostname, pid, tempfile = get_lock_data()
+    if hostname <> socket.gethostname():
+        return WatcherState.host_mismatch
+    # Find out if the process exists by calling kill with a signal 0.
+    try:
+        os.kill(pid, 0)
+        return WatcherState.conflict
+    except OSError, e:
+        if e.errno == errno.ESRCH:
+            # No matching process id.
+            return WatcherState.stale_lock
+        # Some other error occurred.
+        raise
+
+
+def acquire_lock_1(force):
+    """Try to acquire the master queue runner lock.
+
+    :param force: Flag that controls whether to force acquisition of the lock.
+    :return: The master queue runner lock.
+    :raises: `TimeOutError` if the lock could not be acquired.
+    """
+    lock = lockfile.Lock(config.LOCK_FILE, LOCK_LIFETIME)
+    try:
+        lock.lock(timedelta(seconds=0.1))
+        return lock
+    except lockfile.TimeOutError:
+        if not force:
+            raise
+        # Force removal of lock first.
+        lock.disown()
+        hostname, pid, tempfile = get_lock_data()
+        os.unlink(config.LOCK_FILE)
+        os.unlink(os.path.join(config.LOCK_DIR, tempfile))
+        return acquire_lock_1(force=False)
+
+
+def acquire_lock(force):
+    """Acquire the master queue runner lock.
+
+    :param force: Flag that controls whether to force acquisition of the lock.
+    :return: The master queue runner lock or None if the lock couldn't be
+        acquired.  In that case, an error messages is also printed to standard
+        error.
+    """
+    try:
+        lock = acquire_lock_1(force)
+        return lock
+    except lockfile.TimeOutError:
+        status = master_state()
+        if status == WatcherState.conflict:
+            # Hostname matches and process exists.
+            print >> sys.stderr, _("""\
+The master qrunner lock could not be acquired because it appears as if another
+master qrunner is already running.
+""")
+        elif status == WatcherState.stale_lock:
+            # Hostname matches but the process does not exist.
+            print >> sys.stderr, _("""\
+The master qrunner lock could not be acquired.  It appears as though there is
+a stale master qrunner lock.  Try re-running mailmanctl with the -s flag.
+""")
+        else:
+            assert status == WatcherState.host_mismatch, (
+                'Invalid enum value: %s' % status)
+            # Hostname doesn't even match.
+            print >> sys.stderr, _("""\
+The master qrunner lock could not be acquired, because it appears as if some
+process on some other host may have acquired it.  We can't test for stale
+locks across host boundaries, so you'll have to do this manually.  Or, if you
+know the lock is stale, re-run mailmanctl with the -s flag.
+
+Lock file: $config.LOCK_FILE
+Lock host: $status
+
+Exiting.""")
+        return None
+
+
+
+def start_runner(qrname, slice, count):
+    """Start a queue runner.
+
+    All arguments are passed to the qrunner process.
+
+    :param qrname: The name of the queue runner.
+    :param slice: The slice number.
+    :param count: The total number of slices.
+    :return: The process id of the child queue runner.
+    """
+    pid = os.fork()
+    if pid:
+        # Parent.
+        return pid
+    # Child.
+    #
+    # Craft the command line arguments for the exec() call.
+    rswitch = '--runner=%s:%d:%d' % (qrname, slice, count)
+    # Wherever mailmanctl lives, so too must live the qrunner script.
+    exe = os.path.join(BIN_DIR, 'qrunner')
+    # config.PYTHON, which is the absolute path to the Python interpreter,
+    # must be given as argv[0] due to Python's library search algorithm.
+    args = [sys.executable, sys.executable, exe, rswitch, '-s']
+    if parser.options.config:
+        args.extend(['-C', parser.options.config])
+    log.debug('starting: %s', args)
+    os.execl(*args)
+    # We should never get here.
+    raise RuntimeError('os.execl() failed')
+
+
+
+def control_loop(lock):
+    """The main control loop.
+
+    This starts up the queue runners, watching for their exit and restarting
+    them if need be.
+    """
+    restartable = parser.options.restartable
+    # Start all the qrunners.  Keep a dictionary mapping process ids to
+    # information about the child processes.
+    kids = {}
+    # Set up our signal handlers.  Also set up a SIGALRM handler to refresh
+    # the lock once per day.  The lock lifetime is 1 day + 6 hours so this
+    # should be plenty.
+    def sigalrm_handler(signum, frame):
+        lock.refresh()
+        signal.alarm(int(Defaults.days(1)))
+    signal.signal(signal.SIGALRM, sigalrm_handler)
+    signal.alarm(int(Defaults.days(1)))
+    # SIGHUP tells the qrunners to close and reopen their log files.
+    def sighup_handler(signum, frame):
+        loginit.reopen()
+        for pid in kids:
+            os.kill(pid, signal.SIGHUP)
+        log.info('Master watcher caught SIGHUP.  Re-opening log files.')
+    signal.signal(signal.SIGHUP, sighup_handler)
+    # SIGUSR1 is used by 'mailman restart'.
+    def sigusr1_handler(signum, frame):
+        for pid in kids:
+            os.kill(pid, signal.SIGUSR1)
+        log.info('Master watcher caught SIGUSR1.  Exiting.')
+    signal.signal(signal.SIGUSR1, sigusr1_handler)
+    # SIGTERM is what init will kill this process with when changing run
+    # levels.  It's also the signal 'mailmanctl stop' uses.
+    def sigterm_handler(signum, frame):
+        for pid in kids:
+            os.kill(pid, signal.SIGTERM)
+        log.info('Master watcher caught SIGTERM.  Exiting.')
+    signal.signal(signal.SIGTERM, sigterm_handler)
+    # SIGINT is what control-C gives.
+    def sigint_handler(signum, frame):
+        for pid in kids:
+            os.kill(pid, signal.SIGINT)
+        log.info('Master watcher caught SIGINT.  Restarting.')
+    signal.signal(signal.SIGINT, sigint_handler)
+    # Start all the child qrunners.
+    for qrname, count in config.qrunners.items():
+        for slice_number in range(count):
+            # queue runner name, slice number, number of slices, restart count
+            info = (qrname, slice_number, count, 0)
+            pid = start_runner(qrname, slice_number, count)
+            kids[pid] = info
+    # Enter the main wait loop.
+    try:
+        while True:
+            try:
+                pid, status = os.wait()
+            except OSError, error:
+                # No children?  We're done.
+                if error.errno == errno.ECHILD:
+                    break
+                # If the system call got interrupted, just restart it.
+                elif error.errno == errno.EINTR:
+                    continue
+                else:
+                    raise
+            # Find out why the subprocess exited by getting the signal
+            # received or exit status.
+            if os.WIFSIGNALED(status):
+                why = os.WTERMSIG(status)
+            elif os.WIFEXITED(status):
+                why = os.WEXITSTATUS(status)
+            else:
+                why = None
+            # We'll restart the subprocess if it exited with a SIGUSR1 or
+            # because of a failure (i.e. no exit signal), and the no-restart
+            # command line switch was not given.  This lets us better handle
+            # runaway restarts (e.g.  if the subprocess had a syntax error!)
+            qrname, slice, count, restarts = kids.pop(pid)
+            restart = False
+            if why == signal.SIGUSR1 and restartable:
+                restart = True
+            # Have we hit the maximum number of restarts?
+            restarts += 1
+            if restarts > config.MAX_RESTARTS:
+                restart = False
+            # Are we permanently non-restartable?
+            log.debug("""\
+Master detected subprocess exit
+(pid: %d, why: %s, class: %s, slice: %d/%d) %s""",
+                     pid, why, qrname, slice+1, count,
+                     ('[restarting]' if restart else ''))
+            # See if we've reached the maximum number of allowable restarts
+            if restarts > config.MAX_RESTARTS:
+                log.info("""\
+qrunner %s reached maximum restart limit of %d, not restarting.""",
+                         qrname, config.MAX_RESTARTS)
+            # Now perhaps restart the process unless it exited with a
+            # SIGTERM or we aren't restarting.
+            if restart:
+                newpid = start_runner(qrname, slice, count)
+                kids[newpid] = (qrname, slice, count, restarts)
+    finally:
+        # Should we leave the main loop for any reason, we want to be sure
+        # all of our children are exited cleanly.  Send SIGTERMs to all
+        # the child processes and wait for them all to exit.
+        for pid in kids:
+            try:
+                os.kill(pid, signal.SIGTERM)
+            except OSError, error:
+                if error.errno == errno.ESRCH:
+                    # The child has already exited.
+                    log.info('ESRCH on pid: %d', pid)
+        # Wait for all the children to go away.
+        while True:
+            try:
+                pid, status = os.wait()
+            except OSError, e:
+                if e.errno == errno.ECHILD:
+                    break
+                elif e.errno == errno.EINTR:
+                    continue
+                raise
+
+
+
+def main():
+    """Main process."""
+    global log, parser
+
+    parser = parseargs()
+    initialize(parser.options.config)
+
+    # We can't grab the logger until after everything's been initialized.
+    log = logging.getLogger('mailman.qrunner')
+
+    # Acquire the master lock, exiting if we can't acquire it.  We'll let the
+    # caller handle any clean up or lock breaking.
+    with lockfile.Lock(config.LOCK_FILE, LOCK_LIFETIME) as lock:
+        with open(config.PIDFILE, 'w') as fp:
+            print >> fp, os.getpid()
+        try:
+            control_loop(lock)
+        finally:
+            os.remove(config.PIDFILE)
+
+
+
+if __name__ == '__main__':
+    try:
+        main()
+    except KeyboardInterrupt:
+        pass
diff --git a/Mailman/bin/qrunner.py b/Mailman/bin/qrunner.py
index 91ecb2ca3..b73643c9d 100644
--- a/Mailman/bin/qrunner.py
+++ b/Mailman/bin/qrunner.py
@@ -95,7 +95,7 @@ each listed by the -l option."""))
     parser.add_option('-o', '--once',
                       default=False, action='store_true', help=_("""\
 Run each named qrunner exactly once through its main loop.  Otherwise, each
-qrunner runs indefinitely, until the process receives a SIGTERM or SIGINT."""))
+qrunner runs indefinitely, until the process receives signal."""))
     parser.add_option('-l', '--list',
                       default=False, action='store_true',
                       help=_('List the available qrunner names and exit.'))
@@ -134,13 +134,13 @@ def make_qrunner(name, slice, range, once=False):
         __import__(modulename)
     except ImportError, e:
         if opts.subproc:
-            # Exit with SIGTERM exit code so mailmanctl won't try to restart us
+            # Exit with SIGTERM exit code so the master watcher won't try to
+            # restart us.
             print >> sys.stderr, _('Cannot import runner module: $modulename')
             print >> sys.stderr, e
             sys.exit(signal.SIGTERM)
         else:
-            print >> sys.stderr, e
-            sys.exit(1)
+            raise
     qrclass = getattr(sys.modules[modulename], classname)
     if once:
         # Subclass to hack in the setting of the stop flag in _doperiodic()
@@ -155,22 +155,34 @@ def make_qrunner(name, slice, range, once=False):
 
 
 def set_signals(loop):
-    # Set up the SIGTERM handler for stopping the loop
+    """Set up the signal handlers.
+
+    Signals caught are: SIGTERM, SIGINT, SIGUSR1 and SIGHUP.  The latter is
+    used to re-open the log files.  SIGTERM and SIGINT are treated exactly the
+    same -- they cause qrunner to exit with no restart from the master.
+    SIGUSR1 also causes qrunner to exit, but the master watcher will restart
+    it in that case.
+
+    :param loop: A loop queue runner instance.
+    """
     def sigterm_handler(signum, frame):
         # Exit the qrunner cleanly
         loop.stop()
         loop.status = signal.SIGTERM
         log.info('%s qrunner caught SIGTERM.  Stopping.', loop.name())
     signal.signal(signal.SIGTERM, sigterm_handler)
-    # Set up the SIGINT handler for stopping the loop.  For us, SIGINT is
-    # the same as SIGTERM, but our parent treats the exit statuses
-    # differently (it restarts a SIGINT but not a SIGTERM).
     def sigint_handler(signum, frame):
         # Exit the qrunner cleanly
         loop.stop()
         loop.status = signal.SIGINT
         log.info('%s qrunner caught SIGINT.  Stopping.', loop.name())
     signal.signal(signal.SIGINT, sigint_handler)
+    def sigusr1_handler(signum, frame):
+        # Exit the qrunner cleanly
+        loop.stop()
+        loop.status = signal.SIGUSR1
+        log.info('%s qrunner caught SIGUSR1.  Stopping.', loop.name())
+    signal.signal(signal.SIGUSR1, sigusr1_handler)
     # SIGHUP just tells us to rotate our log files.
     def sighup_handler(signum, frame):
         loginit.reopen()
diff --git a/Mailman/database/__init__.py b/Mailman/database/__init__.py
index 7ca60f145..63bdd110c 100644
--- a/Mailman/database/__init__.py
+++ b/Mailman/database/__init__.py
@@ -23,7 +23,6 @@ __all__ = [
     ]
 
 import os
-import Mailman.Version
 
 from locknix.lockfile import Lock
 from storm.locals import create_database, Store
@@ -31,6 +30,9 @@ from string import Template
 from urlparse import urlparse
 from zope.interface import implements
 
+import Mailman.Version
+import Mailman.database
+
 from Mailman.configuration import config
 from Mailman.database.listmanager import ListManager
 from Mailman.database.messagestore import MessageStore
@@ -87,17 +89,23 @@ class StockDatabase:
         store = Store(database)
         database.DEBUG = (config.DEFAULT_DATABASE_ECHO
                           if debug is None else debug)
+        # Check the sqlite master database to see if the version file exists.
+        # If so, then we assume the database schema is correctly initialized.
         # Storm does not currently have schema creation.  This is not an ideal
         # way to handle creating the database, but it's cheap and easy for
         # now.
-        import Mailman.database
-        schema_file = os.path.join(
-            os.path.dirname(Mailman.database.__file__),
-            'mailman.sql')
-        with open(schema_file) as fp:
-            sql = fp.read()
-        for statement in sql.split(';'):
-            store.execute(statement + ';')
+        table_names = [item[0] for item in 
+                       store.execute('select tbl_name from sqlite_master;')]
+        if 'version' not in table_names:
+            # Initialize the database.
+            schema_file = os.path.join(
+                os.path.dirname(Mailman.database.__file__),
+                'mailman.sql')
+            with open(schema_file) as fp:
+                sql = fp.read()
+            for statement in sql.split(';'):
+                store.execute(statement + ';')
+            store.commit()
         # Validate schema version.
         v = store.find(Version, component=u'schema').one()
         if not v:
diff --git a/Mailman/extras/__init__.py b/Mailman/extras/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/Mailman/extras/__init__.py
diff --git a/data/mailman.cfg.in b/Mailman/extras/mailman.cfg.in
index 3e24acc46..bb51be97c 100644
--- a/data/mailman.cfg.in
+++ b/Mailman/extras/mailman.cfg.in
@@ -1,6 +1,6 @@
 # -*- python -*-
 
-# Copyright (C) 2006-2007 by the Free Software Foundation, Inc.
+# Copyright (C) 2006-2008 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -27,7 +27,8 @@ automatically be available in this module's namespace.
 You should consult `Defaults.py` though for a complete listing of
 configuration variables that you can change.
 
-Mailman's installation procedure will never overwrite `mailman.cfg`.
+Mailman's installation procedure will never overwrite `mailman.cfg` unless you
+tell it to.
 """
 
 # This is the top-level run-time data directory.  All other runtime data by
@@ -35,11 +36,11 @@ Mailman's installation procedure will never overwrite `mailman.cfg`.
 VAR_DIR = '$var_dir'
 
 # User name and id that owns the Mailman process and files.
-MAILMAN_UID  = '$user_id'
+MAILMAN_UID  = $user_id
 MAILMAN_USER = '$user_name'
 
 # Group name and id that owns the Mailman process and files.
-MAILMAN_GID   = '$group_id'
+MAILMAN_GID   = $group_id
 MAILMAN_GROUP = '$group_name'
 
 LANGUAGES = '$languages'
diff --git a/Mailman/queue/command.py b/Mailman/queue/command.py
index 8217acad6..a0db2f776 100644
--- a/Mailman/queue/command.py
+++ b/Mailman/queue/command.py
@@ -33,7 +33,6 @@ from email.MIMEText import MIMEText
 
 from Mailman import Message
 from Mailman import Utils
-from Mailman.Handlers import Replybot
 from Mailman.app.replybot import autorespond_to_sender
 from Mailman.configuration import config
 from Mailman.i18n import _
@@ -206,7 +205,8 @@ class CommandRunner(Runner):
             return False
         # Do replybot for commands
         mlist.Load()
-        Replybot.process(mlist, msg, msgdata)
+        replybot = config.handlers['replybot']
+        replybot.process(mlist, msg, msgdata)
         if mlist.autorespond_requests == 1:
             log.info('replied and discard')
             # w/discard
diff --git a/Mailman/queue/outgoing.py b/Mailman/queue/outgoing.py
index 09572ba9b..a3d60f0a0 100644
--- a/Mailman/queue/outgoing.py
+++ b/Mailman/queue/outgoing.py
@@ -46,14 +46,13 @@ class OutgoingRunner(Runner, BounceMixin):
         Runner.__init__(self, slice, numslices)
         BounceMixin.__init__(self)
         # We look this function up only at startup time
-        modname = 'Mailman.Handlers.' + config.DELIVERY_MODULE
-        mod = __import__(modname)
-        self._func = getattr(sys.modules[modname], 'process')
+        handler = config.handlers[config.DELIVERY_MODULE]
+        self._func = handler.process
         # This prevents smtp server connection problems from filling up the
         # error log.  It gets reset if the message was successfully sent, and
         # set if there was a socket.error.
-        self.__logged = False
-        self.__retryq = Switchboard(config.RETRYQUEUE_DIR)
+        self._logged = False
+        self._retryq = Switchboard(config.RETRYQUEUE_DIR)
 
     def _dispose(self, mlist, msg, msgdata):
         # See if we should retry delivery of this message again.
@@ -69,7 +68,7 @@ class OutgoingRunner(Runner, BounceMixin):
             if pid <> os.getpid():
                 log.error('child process leaked thru: %s', modname)
                 os._exit(1)
-            self.__logged = False
+            self._logged = False
         except socket.error:
             # There was a problem connecting to the SMTP server.  Log this
             # once, but crank up our sleep time so we don't fill the error
@@ -78,10 +77,10 @@ class OutgoingRunner(Runner, BounceMixin):
             if port == 0:
                 port = 'smtp'
             # Log this just once.
-            if not self.__logged:
+            if not self._logged:
                 log.error('Cannot connect to SMTP server %s on port %s',
                           config.SMTPHOST, port)
-                self.__logged = True
+                self._logged = True
             return True
         except Errors.SomeRecipientsFailed, e:
             # Handle local rejects of probe messages differently.
@@ -120,7 +119,7 @@ class OutgoingRunner(Runner, BounceMixin):
                     msgdata['last_recip_count'] = len(recips)
                     msgdata['deliver_until'] = deliver_until
                     msgdata['recips'] = recips
-                    self.__retryq.enqueue(msg, msgdata)
+                    self._retryq.enqueue(msg, msgdata)
         # We've successfully completed handling of this message
         return False
 
diff --git a/Mailman/queue/pipeline.py b/Mailman/queue/pipeline.py
index 14bfea56c..47c2f9e6f 100644
--- a/Mailman/queue/pipeline.py
+++ b/Mailman/queue/pipeline.py
@@ -22,9 +22,9 @@ through the 'preparation pipeline'.  This pipeline adds, deletes and modifies
 headers, calculates message recipients, and more.
 """
 
-from Mailman.app.pipeline import process
+from Mailman.app.pipelines import process
 from Mailman.configuration import config
-from Mailman.queue import runner
+from Mailman.queue import Runner
 
 
 
diff --git a/setup.py b/setup.py
index 9cb6370b1..983e7a2a7 100644
--- a/setup.py
+++ b/setup.py
@@ -82,7 +82,6 @@ Any other spelling is incorrect.""",
     include_package_data = True,
     entry_points    = {
         'console_scripts': list(scripts),
-        'setuptools.file_finders': 'bzr = setuptools_bzr:find_files_for_bzr',
         # Entry point for plugging in different database backends.
         'mailman.archiver'  : 'stock = Mailman.app.archiving:Pipermail',
         'mailman.database'  : 'stock = Mailman.database:StockDatabase',