6 files changed, 682 insertions, 0 deletions

diff --git a/src/mailman/testing/__init__.py b/src/mailman/testing/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/src/mailman/testing/__init__.py
diff --git a/src/mailman/testing/helpers.py b/src/mailman/testing/helpers.py
new file mode 100644
index 000000000..f92c5f012
--- /dev/null
+++ b/src/mailman/testing/helpers.py
@@ -0,0 +1,248 @@
+# Copyright (C) 2008-2009 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Various test helpers."""
+
+from __future__ import absolute_import, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    'TestableMaster',
+    'digest_mbox',
+    'get_lmtp_client',
+    'get_queue_messages',
+    'make_testable_runner',
+    ]
+
+
+import os
+import time
+import errno
+import signal
+import socket
+import logging
+import mailbox
+import smtplib
+import threading
+
+from Queue import Empty, Queue
+
+from mailman.bin.master import Loop as Master
+from mailman.config import config
+from mailman.testing.smtplistener import Server
+
+
+log = logging.getLogger('mailman.debug')
+
+
+
+def make_testable_runner(runner_class, name=None):
+    """Create a queue runner that runs until its queue is empty.
+
+    :param runner_class: The queue runner's class.
+    :type runner_class: class
+    :param name: Optional queue name; if not given, it is calculated from the
+        class name.
+    :type name: string or None
+    :return: A runner instance.
+    """
+
+    if name is None:
+        assert runner_class.__name__.endswith('Runner'), (
+            'Unparseable runner class name: %s' % runner_class.__name__)
+        name = runner_class.__name__[:-6].lower()
+
+    class EmptyingRunner(runner_class):
+        """Stop processing when the queue is empty."""
+
+        def _do_periodic(self):
+            """Stop when the queue is empty."""
+            self._stop = (len(self.switchboard.files) == 0)
+
+    return EmptyingRunner(name)
+
+
+
+class _Bag:
+    def __init__(self, **kws):
+        for key, value in kws.items():
+            setattr(self, key, value)
+
+
+def get_queue_messages(queue_name):
+    """Return and clear all the messages in the given queue.
+
+    :param queue_name: A string naming a queue.
+    :return: A list of 2-tuples where each item contains the message and
+        message metadata.
+    """
+    queue = config.switchboards[queue_name]
+    messages = []
+    for filebase in queue.files:
+        msg, msgdata = queue.dequeue(filebase)
+        messages.append(_Bag(msg=msg, msgdata=msgdata))
+        queue.finish(filebase)
+    return messages
+
+
+
+def digest_mbox(mlist):
+    """The mailing list's pending digest as a mailbox.
+
+    :param mlist: The mailing list.
+    :return: The mailing list's pending digest as a mailbox.
+    """
+    path = os.path.join(mlist.data_path, 'digest.mbox')
+    return mailbox.mbox(path)
+
+
+
+class TestableMaster(Master):
+    """A testable master loop watcher."""
+
+    def __init__(self):
+        super(TestableMaster, self).__init__(
+            restartable=False, config_file=config.filename)
+        self.event = threading.Event()
+        self.thread = threading.Thread(target=self.loop)
+        self._started_kids = None
+
+    def start(self, *qrunners):
+        """Start the master."""
+        self.start_qrunners(qrunners)
+        self.thread.start()
+        # Wait until all the children are definitely started.
+        self.event.wait()
+
+    def stop(self):
+        """Stop the master by killing all the children."""
+        for pid in self.qrunner_pids:
+            os.kill(pid, signal.SIGTERM)
+        self.cleanup()
+        self.thread.join()
+
+    def loop(self):
+        """Wait until all the qrunners are actually running before looping."""
+        starting_kids = set(self._kids)
+        while starting_kids:
+            for pid in self._kids:
+                try:
+                    os.kill(pid, 0)
+                    starting_kids.remove(pid)
+                except OSError, error:
+                    if error.errno == errno.ESRCH:
+                        # The child has not yet started.
+                        pass
+                    raise
+        # Keeping a copy of all the started child processes for use by the
+        # testing environment, even after all have exited.
+        self._started_kids = set(self._kids)
+        # Let the blocking thread know everything's running.
+        self.event.set()
+        super(TestableMaster, self).loop()
+
+    @property
+    def qrunner_pids(self):
+        """The pids of all the child qrunner processes."""
+        for pid in self._started_kids:
+            yield pid
+
+
+
+class SMTPServer:
+    """An smtp server for testing."""
+
+    def __init__(self):
+        self._messages = []
+        self._queue = Queue()
+        self.host = config.mta.smtp_host
+        self.port = int(config.mta.smtp_port)
+        self._server = Server((self.host, self.port), self._queue)
+        self._thread = threading.Thread(target=self._server.start)
+
+    def start(self):
+        """Start the smtp server in a thread."""
+        log.info('test SMTP server starting')
+        self._thread.start()
+        smtpd = smtplib.SMTP()
+        log.info('connecting to %s:%s', self.host, self.port)
+        smtpd.connect(self.host, self.port)
+        response = smtpd.helo('test.localhost')
+        smtpd.quit()
+        log.info('SMTP server is running: %s', response)
+
+    def stop(self):
+        """Stop the smtp server."""
+        smtpd = smtplib.SMTP()
+        smtpd.connect(self.host, self.port)
+        smtpd.docmd('EXIT')
+        self.clear()
+        # Wait for the thread to exit.
+        self._thread.join()
+        log.info('test SMTP server stopped')
+
+    @property
+    def messages(self):
+        """Return all the messages received by the smtp server."""
+        # Look at the thread queue and append any messages from there to our
+        # internal list of messages.
+        while True:
+            try:
+                message = self._queue.get_nowait()
+            except Empty:
+                break
+            else:
+                self._messages.append(message)
+        # Now return all the messages we know about.
+        for message in self._messages:
+            yield message
+
+    def clear(self):
+        """Clear all messages from the queue."""
+        # Just throw these away.
+        list(self._messages)
+        self._messages = []
+
+
+
+class LMTP(smtplib.SMTP):
+    """Like a normal SMTP client, but for LMTP."""
+    def lhlo(self, name=''):
+        self.putcmd('lhlo', name or self.local_hostname)
+        code, msg = self.getreply()
+        self.helo_resp = msg
+        return code, msg
+
+
+def get_lmtp_client():
+    """Return a connected LMTP client."""
+    # It's possible the process has started but is not yet accepting
+    # connections.  Wait a little while.
+    lmtp = LMTP()
+    for attempts in range(3):
+        try:
+            response = lmtp.connect(
+                config.mta.lmtp_host, int(config.mta.lmtp_port))
+            print response
+            return lmtp
+        except socket.error, error:
+            if error[0] == errno.ECONNREFUSED:
+                time.sleep(1)
+            else:
+                raise
+    else:
+        raise RuntimeError('Connection refused')
diff --git a/src/mailman/testing/layers.py b/src/mailman/testing/layers.py
new file mode 100644
index 000000000..19300ba1e
--- /dev/null
+++ b/src/mailman/testing/layers.py
@@ -0,0 +1,204 @@
+# Copyright (C) 2008-2009 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Mailman test layers."""
+
+from __future__ import absolute_import, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    'ConfigLayer',
+    'SMTPLayer',
+    ]
+
+
+import os
+import sys
+import shutil
+import logging
+import tempfile
+
+from pkg_resources import resource_string
+from textwrap import dedent
+
+from mailman.config import config
+from mailman.core import initialize
+from mailman.core.logging import get_handler
+from mailman.i18n import _
+from mailman.testing.helpers import SMTPServer
+from mailman.utilities.string import expand
+
+
+NL = '\n'
+
+
+
+class ConfigLayer:
+    """Layer for pushing and popping test configurations."""
+
+    var_dir = None
+    styles = None
+
+    @classmethod
+    def setUp(cls):
+        # Set up the basic configuration stuff.
+        initialize.initialize_1()
+        assert cls.var_dir is None, 'Layer already set up'
+        # Calculate a temporary VAR_DIR directory so that run-time artifacts
+        # of the tests won't tread on the installation's data.  This also
+        # makes it easier to clean up after the tests are done, and insures
+        # isolation of test suite runs.
+        cls.var_dir = tempfile.mkdtemp()
+        # We need a test configuration both for the foreground process and any
+        # child processes that get spawned.  lazr.config would allow us to do
+        # it all in a string that gets pushed, and we'll do that for the
+        # foreground, but because we may be spawning processes (such as queue
+        # runners) we'll need a file that we can specify to the with the -C
+        # option.  Craft the full test configuration string here, push it, and
+        # also write it out to a temp file for -C.
+        test_config = dedent("""
+        [mailman]
+        var_dir: %s
+        """ % cls.var_dir)
+        # Read the testing config and push it.
+        test_config += resource_string('mailman.testing', 'testing.cfg')
+        config.push('test config', test_config)
+        # Initialize everything else.
+        initialize.initialize_2()
+        initialize.initialize_3()
+        # When stderr debugging is enabled, subprocess root loggers should
+        # also be more verbose.
+        if cls.stderr:
+            test_config += dedent("""
+            [logging.root]
+            propagate: yes
+            level: debug
+            """)
+        # Enable log message propagation and reset the log paths so that the
+        # doctests can check the output.
+        for logger_config in config.logger_configs:
+            sub_name = logger_config.name.split('.')[-1]
+            if sub_name == 'root':
+                continue
+            logger_name = 'mailman.' + sub_name
+            log = logging.getLogger(logger_name)
+            log.propagate = True
+            # Reopen the file to a new path that tests can get at.  Instead of
+            # using the configuration file path though, use a path that's
+            # specific to the logger so that tests can find expected output
+            # more easily.
+            path = os.path.join(config.LOG_DIR, sub_name)
+            get_handler(sub_name).reopen(path)
+            log.setLevel(logging.DEBUG)
+            # If stderr debugging is enabled, make sure subprocesses are also
+            # more verbose.
+            if cls.stderr:
+                test_config += expand(dedent("""
+                [logging.$name]
+                propagate: yes
+                level: debug
+                """), dict(name=sub_name, path=path))
+        # zope.testing sets up logging before we get to our own initialization
+        # function.  This messes with the root logger, so explicitly set it to
+        # go to stderr.
+        if cls.stderr:
+            console = logging.StreamHandler(sys.stderr)
+            formatter = logging.Formatter(config.logging.root.format,
+                                          config.logging.root.datefmt)
+            console.setFormatter(formatter)
+            logging.getLogger().addHandler(console)
+        # Write the configuration file for subprocesses and set up the config
+        # object to pass that properly on the -C option.
+        config_file = os.path.join(cls.var_dir, 'test.cfg')
+        with open(config_file, 'w') as fp:
+            fp.write(test_config)
+            print >> fp
+        config.filename = config_file
+
+    @classmethod
+    def tearDown(cls):
+        assert cls.var_dir is not None, 'Layer not set up'
+        config.pop('test config')
+        shutil.rmtree(cls.var_dir)
+        cls.var_dir = None
+
+    @classmethod
+    def testSetUp(cls):
+        pass
+
+    @classmethod
+    def testTearDown(cls):
+        # Reset the database between tests.
+        config.db._reset()
+        # Remove all residual queue files.
+        for dirpath, dirnames, filenames in os.walk(config.QUEUE_DIR):
+            for filename in filenames:
+                os.remove(os.path.join(dirpath, filename))
+        # Clear out messages in the message store.
+        for message in config.db.message_store.messages:
+            config.db.message_store.delete_message(message['message-id'])
+        config.db.commit()
+        # Reset the global style manager.
+        config.style_manager.populate()
+
+    # Flag to indicate that loggers should propagate to the console.
+    stderr = False
+
+    @classmethod
+    def handle_stderr(cls, *ignore):
+        cls.stderr = True
+
+    @classmethod
+    def hack_options_parser(cls):
+        """Hack our way into the zc.testing framework.
+
+        Add our custom command line option parsing into zc.testing's.  We do
+        the imports here so that if zc.testing isn't invoked, this stuff never
+        gets in the way.  This is pretty fragile, depend on changes in the
+        zc.testing package.  There should be a better way!
+        """
+        from zope.testing.testrunner.options import parser
+        parser.add_option(str('-e'), str('--stderr'),
+                          action='callback', callback=cls.handle_stderr,
+                          help=_('Propagate log errors to stderr.'))
+
+
+
+class SMTPLayer(ConfigLayer):
+    """Layer for starting, stopping, and accessing a test SMTP server."""
+
+    smtpd = None
+
+    @classmethod
+    def setUp(cls):
+        assert cls.smtpd is None, 'Layer already set up'
+        cls.smtpd = SMTPServer()
+        cls.smtpd.start()
+
+    @classmethod
+    def tearDown(cls):
+        assert cls.smtpd is not None, 'Layer not set up'
+        cls.smtpd.clear()
+        cls.smtpd.stop()
+
+    @classmethod
+    def testSetUp(cls):
+        pass
+
+    @classmethod
+    def testTearDown(cls):
+        pass
diff --git a/src/mailman/testing/mta.py b/src/mailman/testing/mta.py
new file mode 100644
index 000000000..a10ba3c81
--- /dev/null
+++ b/src/mailman/testing/mta.py
@@ -0,0 +1,46 @@
+# Copyright (C) 2009 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Fake MTA for testing purposes."""
+
+from __future__ import absolute_import, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    'FakeMTA',
+    ]
+
+
+from zope.interface import implements
+
+from mailman.interfaces.mta import IMailTransportAgent
+
+
+
+class FakeMTA:
+    """Fake MTA for testing purposes."""
+
+    implements(IMailTransportAgent)
+
+    def create(self, mlist):
+        pass
+
+    def delete(self, mlist):
+        pass
+
+    def regenerate(self):
+        pass
diff --git a/src/mailman/testing/smtplistener.py b/src/mailman/testing/smtplistener.py
new file mode 100644
index 000000000..2094e20de
--- /dev/null
+++ b/src/mailman/testing/smtplistener.py
@@ -0,0 +1,97 @@
+# Copyright (C) 2007-2009 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""A test SMTP listener."""
+
+from __future__ import absolute_import, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    'Server',
+    ]
+
+
+import smtpd
+import socket
+import logging
+import asyncore
+
+from email import message_from_string
+
+
+COMMASPACE = ', '
+log = logging.getLogger('mailman.debug')
+
+
+
+class Channel(smtpd.SMTPChannel):
+    """A channel that can reset the mailbox."""
+
+    def __init__(self, server, conn, addr):
+        smtpd.SMTPChannel.__init__(self, server, conn, addr)
+        # Stash this here since the subclass uses private attributes. :(
+        self._server = server
+
+    def smtp_EXIT(self, arg):
+        """Respond to a new command EXIT by exiting the server."""
+        self.push('250 Ok')
+        self._server.stop()
+
+    def send(self, data):
+        """Silence the bloody asynchat/asyncore broken pipe errors!"""
+        try:
+            return smtpd.SMTPChannel.send(self, data)
+        except socket.error:
+            # Nothing here can affect the outcome, and these messages are just
+            # plain annoying!  So ignore them.
+            pass
+
+
+
+class Server(smtpd.SMTPServer):
+    """An SMTP server that stores messages to a mailbox."""
+
+    def __init__(self, localaddr, queue):
+        smtpd.SMTPServer.__init__(self, localaddr, None)
+        log.info('[SMTPServer] listening: %s', localaddr)
+        self._queue = queue
+
+    def handle_accept(self):
+        """Handle connections by creating our own Channel object."""
+        conn, addr = self.accept()
+        log.info('[SMTPServer] accepted: %s', addr)
+        Channel(self, conn, addr)
+
+    def process_message(self, peer, mailfrom, rcpttos, data):
+        """Process a message by adding it to the mailbox."""
+        message = message_from_string(data)
+        message['X-Peer'] = '{0}:{1}'.format(*peer)
+        message['X-MailFrom'] = mailfrom
+        message['X-RcptTo'] = COMMASPACE.join(rcpttos)
+        log.info('[SMTPServer] processed message: %s',
+                 message.get('message-id', 'n/a'))
+        self._queue.put(message)
+
+    def start(self):
+        """Start the asyncore loop."""
+        asyncore.loop()
+
+    def stop(self):
+        """Stop the asyncore loop."""
+        asyncore.socket_map.clear()
+        asyncore.close_all()
+        self.close()
diff --git a/src/mailman/testing/testing.cfg b/src/mailman/testing/testing.cfg
new file mode 100644
index 000000000..107db86ed
--- /dev/null
+++ b/src/mailman/testing/testing.cfg
@@ -0,0 +1,87 @@
+# Copyright (C) 2008-2009 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+# A testing configuration.
+
+[mta]
+smtp_port: 9025
+incoming: mailman.testing.mta.FakeMTA
+
+[qrunner.archive]
+max_restarts: 1
+
+[qrunner.bounces]
+max_restarts: 1
+
+[qrunner.command]
+max_restarts: 1
+
+[qrunner.in]
+max_restarts: 1
+
+[qrunner.lmtp]
+max_restarts: 1
+
+[qrunner.maildir]
+max_restarts: 1
+
+[qrunner.news]
+max_restarts: 1
+
+[qrunner.out]
+max_restarts: 1
+
+[qrunner.pipeline]
+max_restarts: 1
+
+[qrunner.retry]
+max_restarts: 1
+
+[qrunner.shunt]
+max_restarts: 1
+
+[qrunner.virgin]
+max_restarts: 1
+
+[archiver.prototype]
+enable: yes
+
+[archiver.mail_archive]
+enable: yes
+base_url: http://go.mail-archive.dev/
+recipient: archive@mail-archive.dev
+
+[archiver.pipermail]
+enable: yes
+base_url: http://www.example.com/pipermail/$listname
+
+[archiver.mhonarc]
+enable: yes
+command: /bin/echo "/usr/bin/mhonarc -add -dbfile $PRIVATE_ARCHIVE_FILE_DIR/${listname}.mbox/mhonarc.db -outdir $VAR_DIR/mhonarc/${listname} -stderr $LOG_DIR/mhonarc -stdout $LOG_DIR/mhonarc -spammode -umask 022"
+
+[domain.example_dot_com]
+email_host: example.com
+base_url: http://lists.example.com
+contact_address: postmaster@example.com
+
+[language.ja]
+description: Japanese
+charset: euc-jp
+
+[language.fr]
+description: French
+charset: iso-8859-1