diff options
Diffstat (limited to 'src/mailman/testing')
| -rw-r--r-- | src/mailman/testing/__init__.py | 0 | ||||
| -rw-r--r-- | src/mailman/testing/helpers.py | 248 | ||||
| -rw-r--r-- | src/mailman/testing/layers.py | 204 | ||||
| -rw-r--r-- | src/mailman/testing/mta.py | 46 | ||||
| -rw-r--r-- | src/mailman/testing/smtplistener.py | 97 | ||||
| -rw-r--r-- | src/mailman/testing/testing.cfg | 87 |
6 files changed, 682 insertions, 0 deletions
diff --git a/src/mailman/testing/__init__.py b/src/mailman/testing/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/mailman/testing/__init__.py diff --git a/src/mailman/testing/helpers.py b/src/mailman/testing/helpers.py new file mode 100644 index 000000000..f92c5f012 --- /dev/null +++ b/src/mailman/testing/helpers.py @@ -0,0 +1,248 @@ +# Copyright (C) 2008-2009 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Various test helpers.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestableMaster', + 'digest_mbox', + 'get_lmtp_client', + 'get_queue_messages', + 'make_testable_runner', + ] + + +import os +import time +import errno +import signal +import socket +import logging +import mailbox +import smtplib +import threading + +from Queue import Empty, Queue + +from mailman.bin.master import Loop as Master +from mailman.config import config +from mailman.testing.smtplistener import Server + + +log = logging.getLogger('mailman.debug') + + + +def make_testable_runner(runner_class, name=None): + """Create a queue runner that runs until its queue is empty. + + :param runner_class: The queue runner's class. + :type runner_class: class + :param name: Optional queue name; if not given, it is calculated from the + class name. + :type name: string or None + :return: A runner instance. + """ + + if name is None: + assert runner_class.__name__.endswith('Runner'), ( + 'Unparseable runner class name: %s' % runner_class.__name__) + name = runner_class.__name__[:-6].lower() + + class EmptyingRunner(runner_class): + """Stop processing when the queue is empty.""" + + def _do_periodic(self): + """Stop when the queue is empty.""" + self._stop = (len(self.switchboard.files) == 0) + + return EmptyingRunner(name) + + + +class _Bag: + def __init__(self, **kws): + for key, value in kws.items(): + setattr(self, key, value) + + +def get_queue_messages(queue_name): + """Return and clear all the messages in the given queue. + + :param queue_name: A string naming a queue. + :return: A list of 2-tuples where each item contains the message and + message metadata. + """ + queue = config.switchboards[queue_name] + messages = [] + for filebase in queue.files: + msg, msgdata = queue.dequeue(filebase) + messages.append(_Bag(msg=msg, msgdata=msgdata)) + queue.finish(filebase) + return messages + + + +def digest_mbox(mlist): + """The mailing list's pending digest as a mailbox. + + :param mlist: The mailing list. + :return: The mailing list's pending digest as a mailbox. + """ + path = os.path.join(mlist.data_path, 'digest.mbox') + return mailbox.mbox(path) + + + +class TestableMaster(Master): + """A testable master loop watcher.""" + + def __init__(self): + super(TestableMaster, self).__init__( + restartable=False, config_file=config.filename) + self.event = threading.Event() + self.thread = threading.Thread(target=self.loop) + self._started_kids = None + + def start(self, *qrunners): + """Start the master.""" + self.start_qrunners(qrunners) + self.thread.start() + # Wait until all the children are definitely started. + self.event.wait() + + def stop(self): + """Stop the master by killing all the children.""" + for pid in self.qrunner_pids: + os.kill(pid, signal.SIGTERM) + self.cleanup() + self.thread.join() + + def loop(self): + """Wait until all the qrunners are actually running before looping.""" + starting_kids = set(self._kids) + while starting_kids: + for pid in self._kids: + try: + os.kill(pid, 0) + starting_kids.remove(pid) + except OSError, error: + if error.errno == errno.ESRCH: + # The child has not yet started. + pass + raise + # Keeping a copy of all the started child processes for use by the + # testing environment, even after all have exited. + self._started_kids = set(self._kids) + # Let the blocking thread know everything's running. + self.event.set() + super(TestableMaster, self).loop() + + @property + def qrunner_pids(self): + """The pids of all the child qrunner processes.""" + for pid in self._started_kids: + yield pid + + + +class SMTPServer: + """An smtp server for testing.""" + + def __init__(self): + self._messages = [] + self._queue = Queue() + self.host = config.mta.smtp_host + self.port = int(config.mta.smtp_port) + self._server = Server((self.host, self.port), self._queue) + self._thread = threading.Thread(target=self._server.start) + + def start(self): + """Start the smtp server in a thread.""" + log.info('test SMTP server starting') + self._thread.start() + smtpd = smtplib.SMTP() + log.info('connecting to %s:%s', self.host, self.port) + smtpd.connect(self.host, self.port) + response = smtpd.helo('test.localhost') + smtpd.quit() + log.info('SMTP server is running: %s', response) + + def stop(self): + """Stop the smtp server.""" + smtpd = smtplib.SMTP() + smtpd.connect(self.host, self.port) + smtpd.docmd('EXIT') + self.clear() + # Wait for the thread to exit. + self._thread.join() + log.info('test SMTP server stopped') + + @property + def messages(self): + """Return all the messages received by the smtp server.""" + # Look at the thread queue and append any messages from there to our + # internal list of messages. + while True: + try: + message = self._queue.get_nowait() + except Empty: + break + else: + self._messages.append(message) + # Now return all the messages we know about. + for message in self._messages: + yield message + + def clear(self): + """Clear all messages from the queue.""" + # Just throw these away. + list(self._messages) + self._messages = [] + + + +class LMTP(smtplib.SMTP): + """Like a normal SMTP client, but for LMTP.""" + def lhlo(self, name=''): + self.putcmd('lhlo', name or self.local_hostname) + code, msg = self.getreply() + self.helo_resp = msg + return code, msg + + +def get_lmtp_client(): + """Return a connected LMTP client.""" + # It's possible the process has started but is not yet accepting + # connections. Wait a little while. + lmtp = LMTP() + for attempts in range(3): + try: + response = lmtp.connect( + config.mta.lmtp_host, int(config.mta.lmtp_port)) + print response + return lmtp + except socket.error, error: + if error[0] == errno.ECONNREFUSED: + time.sleep(1) + else: + raise + else: + raise RuntimeError('Connection refused') diff --git a/src/mailman/testing/layers.py b/src/mailman/testing/layers.py new file mode 100644 index 000000000..19300ba1e --- /dev/null +++ b/src/mailman/testing/layers.py @@ -0,0 +1,204 @@ +# Copyright (C) 2008-2009 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Mailman test layers.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'ConfigLayer', + 'SMTPLayer', + ] + + +import os +import sys +import shutil +import logging +import tempfile + +from pkg_resources import resource_string +from textwrap import dedent + +from mailman.config import config +from mailman.core import initialize +from mailman.core.logging import get_handler +from mailman.i18n import _ +from mailman.testing.helpers import SMTPServer +from mailman.utilities.string import expand + + +NL = '\n' + + + +class ConfigLayer: + """Layer for pushing and popping test configurations.""" + + var_dir = None + styles = None + + @classmethod + def setUp(cls): + # Set up the basic configuration stuff. + initialize.initialize_1() + assert cls.var_dir is None, 'Layer already set up' + # Calculate a temporary VAR_DIR directory so that run-time artifacts + # of the tests won't tread on the installation's data. This also + # makes it easier to clean up after the tests are done, and insures + # isolation of test suite runs. + cls.var_dir = tempfile.mkdtemp() + # We need a test configuration both for the foreground process and any + # child processes that get spawned. lazr.config would allow us to do + # it all in a string that gets pushed, and we'll do that for the + # foreground, but because we may be spawning processes (such as queue + # runners) we'll need a file that we can specify to the with the -C + # option. Craft the full test configuration string here, push it, and + # also write it out to a temp file for -C. + test_config = dedent(""" + [mailman] + var_dir: %s + """ % cls.var_dir) + # Read the testing config and push it. + test_config += resource_string('mailman.testing', 'testing.cfg') + config.push('test config', test_config) + # Initialize everything else. + initialize.initialize_2() + initialize.initialize_3() + # When stderr debugging is enabled, subprocess root loggers should + # also be more verbose. + if cls.stderr: + test_config += dedent(""" + [logging.root] + propagate: yes + level: debug + """) + # Enable log message propagation and reset the log paths so that the + # doctests can check the output. + for logger_config in config.logger_configs: + sub_name = logger_config.name.split('.')[-1] + if sub_name == 'root': + continue + logger_name = 'mailman.' + sub_name + log = logging.getLogger(logger_name) + log.propagate = True + # Reopen the file to a new path that tests can get at. Instead of + # using the configuration file path though, use a path that's + # specific to the logger so that tests can find expected output + # more easily. + path = os.path.join(config.LOG_DIR, sub_name) + get_handler(sub_name).reopen(path) + log.setLevel(logging.DEBUG) + # If stderr debugging is enabled, make sure subprocesses are also + # more verbose. + if cls.stderr: + test_config += expand(dedent(""" + [logging.$name] + propagate: yes + level: debug + """), dict(name=sub_name, path=path)) + # zope.testing sets up logging before we get to our own initialization + # function. This messes with the root logger, so explicitly set it to + # go to stderr. + if cls.stderr: + console = logging.StreamHandler(sys.stderr) + formatter = logging.Formatter(config.logging.root.format, + config.logging.root.datefmt) + console.setFormatter(formatter) + logging.getLogger().addHandler(console) + # Write the configuration file for subprocesses and set up the config + # object to pass that properly on the -C option. + config_file = os.path.join(cls.var_dir, 'test.cfg') + with open(config_file, 'w') as fp: + fp.write(test_config) + print >> fp + config.filename = config_file + + @classmethod + def tearDown(cls): + assert cls.var_dir is not None, 'Layer not set up' + config.pop('test config') + shutil.rmtree(cls.var_dir) + cls.var_dir = None + + @classmethod + def testSetUp(cls): + pass + + @classmethod + def testTearDown(cls): + # Reset the database between tests. + config.db._reset() + # Remove all residual queue files. + for dirpath, dirnames, filenames in os.walk(config.QUEUE_DIR): + for filename in filenames: + os.remove(os.path.join(dirpath, filename)) + # Clear out messages in the message store. + for message in config.db.message_store.messages: + config.db.message_store.delete_message(message['message-id']) + config.db.commit() + # Reset the global style manager. + config.style_manager.populate() + + # Flag to indicate that loggers should propagate to the console. + stderr = False + + @classmethod + def handle_stderr(cls, *ignore): + cls.stderr = True + + @classmethod + def hack_options_parser(cls): + """Hack our way into the zc.testing framework. + + Add our custom command line option parsing into zc.testing's. We do + the imports here so that if zc.testing isn't invoked, this stuff never + gets in the way. This is pretty fragile, depend on changes in the + zc.testing package. There should be a better way! + """ + from zope.testing.testrunner.options import parser + parser.add_option(str('-e'), str('--stderr'), + action='callback', callback=cls.handle_stderr, + help=_('Propagate log errors to stderr.')) + + + +class SMTPLayer(ConfigLayer): + """Layer for starting, stopping, and accessing a test SMTP server.""" + + smtpd = None + + @classmethod + def setUp(cls): + assert cls.smtpd is None, 'Layer already set up' + cls.smtpd = SMTPServer() + cls.smtpd.start() + + @classmethod + def tearDown(cls): + assert cls.smtpd is not None, 'Layer not set up' + cls.smtpd.clear() + cls.smtpd.stop() + + @classmethod + def testSetUp(cls): + pass + + @classmethod + def testTearDown(cls): + pass diff --git a/src/mailman/testing/mta.py b/src/mailman/testing/mta.py new file mode 100644 index 000000000..a10ba3c81 --- /dev/null +++ b/src/mailman/testing/mta.py @@ -0,0 +1,46 @@ +# Copyright (C) 2009 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Fake MTA for testing purposes.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'FakeMTA', + ] + + +from zope.interface import implements + +from mailman.interfaces.mta import IMailTransportAgent + + + +class FakeMTA: + """Fake MTA for testing purposes.""" + + implements(IMailTransportAgent) + + def create(self, mlist): + pass + + def delete(self, mlist): + pass + + def regenerate(self): + pass diff --git a/src/mailman/testing/smtplistener.py b/src/mailman/testing/smtplistener.py new file mode 100644 index 000000000..2094e20de --- /dev/null +++ b/src/mailman/testing/smtplistener.py @@ -0,0 +1,97 @@ +# Copyright (C) 2007-2009 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""A test SMTP listener.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'Server', + ] + + +import smtpd +import socket +import logging +import asyncore + +from email import message_from_string + + +COMMASPACE = ', ' +log = logging.getLogger('mailman.debug') + + + +class Channel(smtpd.SMTPChannel): + """A channel that can reset the mailbox.""" + + def __init__(self, server, conn, addr): + smtpd.SMTPChannel.__init__(self, server, conn, addr) + # Stash this here since the subclass uses private attributes. :( + self._server = server + + def smtp_EXIT(self, arg): + """Respond to a new command EXIT by exiting the server.""" + self.push('250 Ok') + self._server.stop() + + def send(self, data): + """Silence the bloody asynchat/asyncore broken pipe errors!""" + try: + return smtpd.SMTPChannel.send(self, data) + except socket.error: + # Nothing here can affect the outcome, and these messages are just + # plain annoying! So ignore them. + pass + + + +class Server(smtpd.SMTPServer): + """An SMTP server that stores messages to a mailbox.""" + + def __init__(self, localaddr, queue): + smtpd.SMTPServer.__init__(self, localaddr, None) + log.info('[SMTPServer] listening: %s', localaddr) + self._queue = queue + + def handle_accept(self): + """Handle connections by creating our own Channel object.""" + conn, addr = self.accept() + log.info('[SMTPServer] accepted: %s', addr) + Channel(self, conn, addr) + + def process_message(self, peer, mailfrom, rcpttos, data): + """Process a message by adding it to the mailbox.""" + message = message_from_string(data) + message['X-Peer'] = '{0}:{1}'.format(*peer) + message['X-MailFrom'] = mailfrom + message['X-RcptTo'] = COMMASPACE.join(rcpttos) + log.info('[SMTPServer] processed message: %s', + message.get('message-id', 'n/a')) + self._queue.put(message) + + def start(self): + """Start the asyncore loop.""" + asyncore.loop() + + def stop(self): + """Stop the asyncore loop.""" + asyncore.socket_map.clear() + asyncore.close_all() + self.close() diff --git a/src/mailman/testing/testing.cfg b/src/mailman/testing/testing.cfg new file mode 100644 index 000000000..107db86ed --- /dev/null +++ b/src/mailman/testing/testing.cfg @@ -0,0 +1,87 @@ +# Copyright (C) 2008-2009 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +# A testing configuration. + +[mta] +smtp_port: 9025 +incoming: mailman.testing.mta.FakeMTA + +[qrunner.archive] +max_restarts: 1 + +[qrunner.bounces] +max_restarts: 1 + +[qrunner.command] +max_restarts: 1 + +[qrunner.in] +max_restarts: 1 + +[qrunner.lmtp] +max_restarts: 1 + +[qrunner.maildir] +max_restarts: 1 + +[qrunner.news] +max_restarts: 1 + +[qrunner.out] +max_restarts: 1 + +[qrunner.pipeline] +max_restarts: 1 + +[qrunner.retry] +max_restarts: 1 + +[qrunner.shunt] +max_restarts: 1 + +[qrunner.virgin] +max_restarts: 1 + +[archiver.prototype] +enable: yes + +[archiver.mail_archive] +enable: yes +base_url: http://go.mail-archive.dev/ +recipient: archive@mail-archive.dev + +[archiver.pipermail] +enable: yes +base_url: http://www.example.com/pipermail/$listname + +[archiver.mhonarc] +enable: yes +command: /bin/echo "/usr/bin/mhonarc -add -dbfile $PRIVATE_ARCHIVE_FILE_DIR/${listname}.mbox/mhonarc.db -outdir $VAR_DIR/mhonarc/${listname} -stderr $LOG_DIR/mhonarc -stdout $LOG_DIR/mhonarc -spammode -umask 022" + +[domain.example_dot_com] +email_host: example.com +base_url: http://lists.example.com +contact_address: postmaster@example.com + +[language.ja] +description: Japanese +charset: euc-jp + +[language.fr] +description: French +charset: iso-8859-1 |