diff options
| -rw-r--r-- | mailman/Defaults.py | 16 | ||||
| -rw-r--r-- | mailman/archiving/__init__.py | 31 | ||||
| -rw-r--r-- | mailman/archiving/mailarchive.py | 87 | ||||
| -rw-r--r-- | mailman/archiving/mhonarc.py | 95 | ||||
| -rw-r--r-- | mailman/archiving/pipermail.py (renamed from mailman/app/archiving.py) | 49 | ||||
| -rw-r--r-- | mailman/archiving/prototype.py | 75 | ||||
| -rw-r--r-- | mailman/bin/testall.py | 10 | ||||
| -rw-r--r-- | mailman/configuration.py | 1 | ||||
| -rw-r--r-- | mailman/docs/archivers.txt | 140 | ||||
| -rw-r--r-- | mailman/docs/pipelines.txt | 3 | ||||
| -rw-r--r-- | mailman/initialize.py | 2 | ||||
| -rw-r--r-- | mailman/interfaces/mailinglist.py | 2 | ||||
| -rw-r--r-- | mailman/loginit.py | 1 | ||||
| -rw-r--r-- | mailman/pipeline/docs/cook-headers.txt | 1 | ||||
| -rw-r--r-- | mailman/queue/docs/outgoing.txt | 20 | ||||
| -rw-r--r-- | mailman/testing/helpers.py | 6 | ||||
| -rw-r--r-- | mailman/testing/smtplistener.py | 6 | ||||
| -rw-r--r-- | mailman/testing/testing.cfg.in | 5 | ||||
| -rw-r--r-- | mailman/tests/test_documentation.py | 11 | ||||
| -rw-r--r-- | setup.py | 8 |
20 files changed, 483 insertions, 86 deletions
diff --git a/mailman/Defaults.py b/mailman/Defaults.py index 43bf7a3c3..6fa457425 100644 --- a/mailman/Defaults.py +++ b/mailman/Defaults.py @@ -220,6 +220,22 @@ CGIEXT = '' # - $fqdn_listname -- the long name of the list being accessed PUBLIC_ARCHIVE_URL = 'http://$hostname/pipermail/$fqdn_listname' +# The public Mail-Archive.com service's base url. +MAIL_ARCHIVE_BASEURL = 'http://go.mail-archive.com/' +# The posting address for the Mail-Archive.com service +MAIL_ARCHIVE_RECIPIENT = 'archive@mail-archive.com' + +# The command for archiving to a local MHonArc instance. +MHONARC_COMMAND = """\ +/usr/bin/mhonarc \ +-add \ +-dbfile $PRIVATE_ARCHIVE_FILE_DIR/${listname}.mbox/mhonarc.db \ +-outdir $VAR_DIR/mhonarc/${listname} \ +-stderr $LOG_DIR/mhonarc \ +-stdout $LOG_DIR/mhonarc \ +-spammode \ +-umask 022""" + # Are archives on or off by default? DEFAULT_ARCHIVE = On diff --git a/mailman/archiving/__init__.py b/mailman/archiving/__init__.py new file mode 100644 index 000000000..b9ef686a1 --- /dev/null +++ b/mailman/archiving/__init__.py @@ -0,0 +1,31 @@ +# Copyright (C) 2008 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +__metaclass__ = type +__all__ = [ + 'initialize', + ] + + +from mailman.app.plugins import get_plugins +from mailman.configuration import config + + +def initialize(): + """Initialize archivers.""" + for archiver in get_plugins('mailman.archiver'): + config.archivers[archiver.name] = archiver diff --git a/mailman/archiving/mailarchive.py b/mailman/archiving/mailarchive.py new file mode 100644 index 000000000..fb78a2257 --- /dev/null +++ b/mailman/archiving/mailarchive.py @@ -0,0 +1,87 @@ +# Copyright (C) 2008 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +"""The Mail-Archive.com archiver.""" + +__metaclass__ = type +__all__ = [ + 'MailArchive', + ] + + +import hashlib + +from base64 import urlsafe_b64encode +from urllib import quote +from urlparse import urljoin +from zope.interface import implements + +from mailman.configuration import config +from mailman.interfaces.archiver import IArchiver +from mailman.queue import Switchboard + + + +class MailArchive: + """Public archiver at the Mail-Archive.com. + + Messages get archived at http://go.mail-archive.com. + """ + + implements(IArchiver) + + name = 'mail-archive' + is_enabled = False + + @staticmethod + def list_url(mlist): + """See `IArchiver`.""" + if mlist.archive_private: + return None + return urljoin(config.MAIL_ARCHIVE_BASEURL, + quote(mlist.posting_address)) + + @staticmethod + def permalink(mlist, msg): + """See `IArchiver`.""" + if mlist.archive_private: + return None + message_id = msg.get('message-id') + # It is not the archiver's job to ensure the message has a Message-ID. + assert message_id is not None, 'No Message-ID found' + # The angle brackets are not part of the Message-ID. See RFC 2822. + if message_id.startswith('<') and message_id.endswith('>'): + message_id = message_id[1:-1] + else: + message_id = message_id.strip() + sha = hashlib.sha1(message_id) + sha.update(str(mlist.posting_address)) + message_id_hash = urlsafe_b64encode(sha.digest()) + del msg['x-message-id-hash'] + msg['X-Message-ID-Hash'] = message_id_hash + return urljoin(config.MAIL_ARCHIVE_BASEURL, message_id_hash) + + @staticmethod + def archive_message(mlist, msg): + """See `IArchiver`.""" + if mlist.archive_private: + return + outq = Switchboard(config.OUTQUEUE_DIR) + outq.enqueue( + msg, + listname=mlist.fqdn_listname, + recips=[config.MAIL_ARCHIVE_RECIPIENT]) diff --git a/mailman/archiving/mhonarc.py b/mailman/archiving/mhonarc.py new file mode 100644 index 000000000..cc549dee8 --- /dev/null +++ b/mailman/archiving/mhonarc.py @@ -0,0 +1,95 @@ +# Copyright (C) 2008 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +"""MHonArc archiver.""" + +__metaclass__ = type +__all__ = [ + 'MHonArc', + ] + + +import hashlib +import logging +import subprocess + +from base64 import b32encode +from string import Template +from urlparse import urljoin +from zope.interface import implements + +from mailman.configuration import config +from mailman.interfaces.archiver import IArchiver + + +log = logging.getLogger('mailman.archiver') + + + +class MHonArc: + """Local MHonArc archiver.""" + + implements(IArchiver) + + name = 'mhonarc' + is_enabled = False + + @staticmethod + def list_url(mlist): + """See `IArchiver`.""" + # XXX What about private MHonArc archives? + web_host = config.domains.get(mlist.host_name, mlist.host_name) + return Template(config.PUBLIC_ARCHIVE_URL).safe_substitute( + listname=mlist.fqdn_listname, + hostname=web_host, + fqdn_listname=mlist.fqdn_listname, + ) + + @staticmethod + def permalink(mlist, msg): + """See `IArchiver`.""" + # XXX What about private MHonArc archives? + message_id = msg.get('message-id') + # It is not the archiver's job to ensure the message has a Message-ID. + assert message_id is not None, 'No Message-ID found' + # The angle brackets are not part of the Message-ID. See RFC 2822. + if message_id.startswith('<') and message_id.endswith('>'): + message_id = message_id[1:-1] + else: + message_id = message_id.strip() + sha = hashlib.sha1(message_id) + message_id_hash = b32encode(sha.digest()) + del msg['x-message-id-hash'] + msg['X-Message-ID-Hash'] = message_id_hash + return urljoin(MHonArc.list_url(mlist), message_id_hash) + + @staticmethod + def archive_message(mlist, msg): + """See `IArchiver`.""" + substitutions = config.__dict__.copy() + substitutions['listname'] = mlist.fqdn_listname + command = Template(config.MHONARC_COMMAND).safe_substitute( + substitutions) + proc = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + shell=True) + stdout, stderr = proc.communicate(msg.as_string()) + if proc.returncode <> 0: + log.error('%s: mhonarc subprocess had non-zero exit code: %s' % + (msg['message-id'], proc.returncode)) + log.info(stdout) + log.error(stderr) diff --git a/mailman/app/archiving.py b/mailman/archiving/pipermail.py index 15e987daf..1e8f4f28e 100644 --- a/mailman/app/archiving.py +++ b/mailman/archiving/pipermail.py @@ -15,23 +15,18 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, # USA. -"""Application level archiving support.""" +"""Pipermail archiver.""" __metaclass__ = type __all__ = [ 'Pipermail', - 'Prototype', ] import os -import hashlib -from base64 import b32encode from cStringIO import StringIO -from email.utils import make_msgid from string import Template -from urlparse import urljoin from zope.interface import implements from zope.interface.interface import adapter_hooks @@ -79,7 +74,7 @@ class Pipermail: implements(IArchiver) name = 'pipermail' - is_enabled = True + is_enabled = False @staticmethod def list_url(mlist): @@ -112,43 +107,3 @@ class Pipermail: fileobj.close() # There's no good way to know the url for the archived message. return None - - - -class Prototype: - """A prototype of a third party archiver. - - Mailman proposes a draft specification for interoperability between list - servers and archivers: <http://wiki.list.org/display/DEV/Stable+URLs>. - """ - - implements(IArchiver) - - name = 'prototype' - is_enabled = False - - @staticmethod - def list_url(mlist): - """See `IArchiver`.""" - web_host = config.domains.get(mlist.host_name, mlist.host_name) - return 'http://' + web_host - - @staticmethod - def permalink(mlist, msg): - """See `IArchiver`.""" - message_id = msg.get('message-id') - # It is not the archiver's job to ensure the message has a Message-ID. - assert message_id is not None, 'No Message-ID found' - # The angle brackets are not part of the Message-ID. See RFC 2822. - if message_id.startswith('<') and message_id.endswith('>'): - message_id = message_id[1:-1] - digest = hashlib.sha1(message_id).digest() - message_id_hash = b32encode(digest) - del msg['x-message-id-hash'] - msg['X-Message-ID-Hash'] = message_id_hash - return urljoin(Prototype.list_url(mlist), message_id_hash) - - @staticmethod - def archive_message(mlist, message): - """See `IArchiver`.""" - raise NotImplementedError diff --git a/mailman/archiving/prototype.py b/mailman/archiving/prototype.py new file mode 100644 index 000000000..deeaaa624 --- /dev/null +++ b/mailman/archiving/prototype.py @@ -0,0 +1,75 @@ +# Copyright (C) 2008 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +"""Prototypical permalinking archiver.""" + +__metaclass__ = type +__all__ = [ + 'Prototype', + ] + + +import hashlib + +from base64 import b32encode +from urlparse import urljoin +from zope.interface import implements + +from mailman.configuration import config +from mailman.interfaces.archiver import IArchiver + + + +class Prototype: + """A prototype of a third party archiver. + + Mailman proposes a draft specification for interoperability between list + servers and archivers: <http://wiki.list.org/display/DEV/Stable+URLs>. + """ + + implements(IArchiver) + + name = 'prototype' + is_enabled = False + + @staticmethod + def list_url(mlist): + """See `IArchiver`.""" + web_host = config.domains.get(mlist.host_name, mlist.host_name) + return 'http://' + web_host + + @staticmethod + def permalink(mlist, msg): + """See `IArchiver`.""" + message_id = msg.get('message-id') + # It is not the archiver's job to ensure the message has a Message-ID. + assert message_id is not None, 'No Message-ID found' + # The angle brackets are not part of the Message-ID. See RFC 2822. + if message_id.startswith('<') and message_id.endswith('>'): + message_id = message_id[1:-1] + else: + message_id = message_id.strip() + digest = hashlib.sha1(message_id).digest() + message_id_hash = b32encode(digest) + del msg['x-message-id-hash'] + msg['X-Message-ID-Hash'] = message_id_hash + return urljoin(Prototype.list_url(mlist), message_id_hash) + + @staticmethod + def archive_message(mlist, message): + """See `IArchiver`.""" + raise NotImplementedError diff --git a/mailman/bin/testall.py b/mailman/bin/testall.py index e7b3c445b..ed90980ae 100644 --- a/mailman/bin/testall.py +++ b/mailman/bin/testall.py @@ -31,9 +31,11 @@ import tempfile import unittest import pkg_resources +from mailman import Defaults from mailman.configuration import config from mailman.i18n import _ from mailman.initialize import initialize_1, initialize_2 +from mailman.testing.helpers import SMTPServer from mailman.version import MAILMAN_VERSION @@ -226,10 +228,14 @@ def main(): with open(cfg_out, 'a') as fp: print >> fp, 'VAR_DIR = "%s"' % var_dir print >> fp, 'MAILMAN_USER = "%s"' % user_name - print >> fp, 'MAILMAN_UID = %d' % user_id + print >> fp, 'MAILMAN_UID =', user_id print >> fp, 'MAILMAN_GROUP = "%s"' % group_name - print >> fp, 'MAILMAN_GID = %d' % group_id + print >> fp, 'MAILMAN_GID =', group_id print >> fp, "LANGUAGES = 'en'" + print >> fp, 'SMTPPORT =', SMTPServer.port + # A fake MHonArc command, for testing. + print >> fp, 'MHONARC_COMMAND = """/bin/echo', \ + Defaults.MHONARC_COMMAND, '"""' initialize_1(cfg_out, propagate_logs=parser.options.stderr) mailman_uid = pwd.getpwnam(config.MAILMAN_USER).pw_uid diff --git a/mailman/configuration.py b/mailman/configuration.py index 2f682d114..8a702c457 100644 --- a/mailman/configuration.py +++ b/mailman/configuration.py @@ -174,6 +174,7 @@ class Configuration(object): code = self.DEFAULT_SERVER_LANGUAGE self.languages.enable_language(code) # Create various registries. + self.archivers = {} self.chains = {} self.rules = {} self.handlers = {} diff --git a/mailman/docs/archivers.txt b/mailman/docs/archivers.txt index 9e4fbc121..c8ddb73e4 100644 --- a/mailman/docs/archivers.txt +++ b/mailman/docs/archivers.txt @@ -1,4 +1,5 @@ -= Archivers = +Archivers +========= Mailman supports pluggable archivers, and it comes with several default archivers. @@ -23,15 +24,17 @@ Pipermail does not support a permalink, so that interface returns None. Mailman defines a draft spec for how list servers and archivers can interoperate. - >>> from operator import attrgetter - >>> name = attrgetter('name') - >>> from mailman.app.plugins import get_plugins - >>> archivers = {} - >>> for archiver in sorted(get_plugins('mailman.archiver'), key=name): + >>> from mailman.configuration import config + >>> for archiver_name, archiver in sorted(config.archivers.items()): ... print archiver.name ... print ' ', archiver.list_url(mlist) ... print ' ', archiver.permalink(mlist, msg) - ... archivers[archiver.name] = archiver + mail-archive + http://go.mail-archive.dev/test%40example.com + http://go.mail-archive.dev/ZaXPPxRMM9_hFZL4vTRlQlBx8pc= + mhonarc + http://www.example.com/.../test@example.com + http://www.example.com/.../RSZCG7IGPHFIRW3EMTVMMDNJMNCVCOLE pipermail http://www.example.com/pipermail/test@example.com None @@ -40,12 +43,13 @@ interoperate. http://www.example.com/RSZCG7IGPHFIRW3EMTVMMDNJMNCVCOLE -== Sending the message to the archiver == +Sending the message to the archiver +----------------------------------- The archiver is also able to archive the message. >>> mlist.web_page_url = u'http://lists.example.com/' - >>> archivers['pipermail'].archive_message(mlist, msg) + >>> config.archivers['pipermail'].archive_message(mlist, msg) >>> import os >>> from mailman.interfaces.archiver import IPipermailMailingList @@ -57,7 +61,123 @@ The archiver is also able to archive the message. Note however that the prototype archiver can't archive messages. - >>> archivers['prototype'].archive_message(mlist, msg) + >>> config.archivers['prototype'].archive_message(mlist, msg) Traceback (most recent call last): ... NotImplementedError + + +The Mail-Archive.com +-------------------- + +The Mail-Archive <http://www.mail-archive.com> is a public archiver that can +be used to archive message for free. Mailman comes with a plugin for this +archiver; by enabling it messages to public lists will get sent there +automatically. + + >>> archiver = config.archivers['mail-archive'] + >>> archiver.list_url(mlist) + 'http://go.mail-archive.dev/test%40example.com' + >>> archiver.permalink(mlist, msg) + 'http://go.mail-archive.dev/ZaXPPxRMM9_hFZL4vTRlQlBx8pc=' + +To archive the message, the archiver actually mails the message to a special +address at the Mail-Archive. + + >>> archiver.archive_message(mlist, msg) + + >>> from mailman.queue.outgoing import OutgoingRunner + >>> from mailman.testing.helpers import make_testable_runner + >>> outgoing = make_testable_runner(OutgoingRunner) + >>> outgoing.run() + + >>> from operator import itemgetter + >>> messages = list(smtpd.messages) + >>> len(messages) + 1 + + >>> print messages[0].as_string() + From: aperson@example.org + To: test@example.com + Subject: An archived message + Message-ID: <12345> + X-Message-ID-Hash: ZaXPPxRMM9_hFZL4vTRlQlBx8pc= + MIME-Version: 1.0 + Content-Type: text/plain; charset="us-ascii" + Content-Transfer-Encoding: 7bit + Sender: test-bounces@example.com + Errors-To: test-bounces@example.com + X-Peer: 127.0.0.1:... + X-MailFrom: test-bounces@example.com + X-RcptTo: archive@mail-archive.dev + <BLANKLINE> + Here is an archived message. + _______________________________________________ + Test mailing list + test@example.com + http://lists.example.com/listinfo/test@example.com + + >>> smtpd.clear() + +However, if the mailing list is not public, the message will never be archived +at this service. + + >>> mlist.archive_private = True + >>> print archiver.list_url(mlist) + None + >>> print archiver.permalink(mlist, msg) + None + >>> archiver.archive_message(mlist, msg) + >>> list(smtpd.messages) + [] + +Additionally, this archiver can handle malformed Message-IDs. + + >>> mlist.archive_private = False + >>> del msg['message-id'] + >>> msg['Message-ID'] = '12345>' + >>> archiver.permalink(mlist, msg) + 'http://go.mail-archive.dev/bXvG32YzcDEIVDaDLaUSVQekfo8=' + + >>> del msg['message-id'] + >>> msg['Message-ID'] = '<12345' + >>> archiver.permalink(mlist, msg) + 'http://go.mail-archive.dev/9rockPrT1Mm-jOsLWS6_hseR_OY=' + + >>> del msg['message-id'] + >>> msg['Message-ID'] = '12345' + >>> archiver.permalink(mlist, msg) + 'http://go.mail-archive.dev/ZaXPPxRMM9_hFZL4vTRlQlBx8pc=' + + >>> del msg['message-id'] + >>> msg['Message-ID'] = ' 12345 ' + >>> archiver.permalink(mlist, msg) + 'http://go.mail-archive.dev/ZaXPPxRMM9_hFZL4vTRlQlBx8pc=' + + +MHonArc +------- + +The MHonArc archiver <http://www.mhonarc.org> is also available. + + >>> archiver = config.archivers['mhonarc'] + >>> archiver.name + 'mhonarc' + +Messages sent to a local MHonArc instance are added to its archive via a +subprocess call. + + >>> archiver.archive_message(mlist, msg) + >>> archive_log = open(os.path.join(config.LOG_DIR, 'archiver')) + >>> try: + ... contents = archive_log.read() + ... finally: + ... archive_log.close() + >>> print 'LOG:', contents + LOG: ... /usr/bin/mhonarc -add + -dbfile /.../private/test@example.com.mbox/mhonarc.db + -outdir /.../mhonarc/test@example.com + -stderr /.../logs/mhonarc + -stdout /.../logs/mhonarc + -spammode -umask 022 + ... diff --git a/mailman/docs/pipelines.txt b/mailman/docs/pipelines.txt index 0fe51e0e4..c4f8488a1 100644 --- a/mailman/docs/pipelines.txt +++ b/mailman/docs/pipelines.txt @@ -21,6 +21,8 @@ Processing a message Messages hit the pipeline after they've been accepted for posting. + >>> from mailman.configuration import config + >>> config.archivers['pipermail'].is_enabled = True >>> msg = message_from_string("""\ ... From: aperson@example.com ... To: xtest@example.com @@ -69,7 +71,6 @@ However there are currently no recipients for this message. And the message is now sitting in various other processing queues. >>> from mailman.testing.helpers import get_queue_messages - >>> from mailman.configuration import config >>> messages = get_queue_messages(config.ARCHQUEUE_DIR) >>> len(messages) 1 diff --git a/mailman/initialize.py b/mailman/initialize.py index 6c2a5a8f4..d8dc0d69d 100644 --- a/mailman/initialize.py +++ b/mailman/initialize.py @@ -65,10 +65,12 @@ def initialize_2(debug=False): mailman.configuration.config.db = database # Initialize the rules and chains. Do the imports here so as to avoid # circular imports. + from mailman.archiving import initialize as initialize_archivers from mailman.app.chains import initialize as initialize_chains from mailman.app.rules import initialize as initialize_rules from mailman.app.pipelines import initialize as initialize_pipelines from mailman.app.commands import initialize as initialize_commands + initialize_archivers() initialize_rules() initialize_chains() initialize_pipelines() diff --git a/mailman/interfaces/mailinglist.py b/mailman/interfaces/mailinglist.py index a5f6a9e9a..8a9967e89 100644 --- a/mailman/interfaces/mailinglist.py +++ b/mailman/interfaces/mailinglist.py @@ -147,7 +147,7 @@ class IMailingList(Interface): last_post_date = Attribute( """The date and time a message was last posted to the mailing list.""") - post_number = Attribute( + post_id = Attribute( """A monotonically increasing integer sequentially assigned to each list posting.""") diff --git a/mailman/loginit.py b/mailman/loginit.py index 844c2543e..68debff8d 100644 --- a/mailman/loginit.py +++ b/mailman/loginit.py @@ -35,6 +35,7 @@ FMT = '%(asctime)s (%(process)d) %(message)s' DATEFMT = '%b %d %H:%M:%S %Y' LOGGERS = ( + 'archiver', # All archiver output 'bounce', # All bounce processing logs go here 'config', # Configuration issues 'debug', # Only used for development diff --git a/mailman/pipeline/docs/cook-headers.txt b/mailman/pipeline/docs/cook-headers.txt index d764bd796..4fbdf58bb 100644 --- a/mailman/pipeline/docs/cook-headers.txt +++ b/mailman/pipeline/docs/cook-headers.txt @@ -184,6 +184,7 @@ But normally, a list will include these headers. >>> mlist.include_rfc2369_headers = True >>> mlist.include_list_post_header = True >>> mlist.preferred_language = u'en' + >>> config.archivers['pipermail'].is_enabled = True >>> msg = message_from_string("""\ ... From: aperson@example.com ... Message-ID: <12345> diff --git a/mailman/queue/docs/outgoing.txt b/mailman/queue/docs/outgoing.txt index ba6cc6d96..cfb6c6988 100644 --- a/mailman/queue/docs/outgoing.txt +++ b/mailman/queue/docs/outgoing.txt @@ -26,21 +26,12 @@ move messages to the 'retry queue' for handling delivery failures. ... u'password', DeliveryMode.regular, u'en', ... ack=False, admin_notif=False) - >>> from mailman.testing.helpers import SMTPServer - >>> smtpd = SMTPServer() - >>> smtpd.start() - >>> from mailman.configuration import config - >>> old_host = config.SMTPHOST - >>> old_port = config.SMTPPORT - >>> config.SMTPHOST = smtpd.host - >>> config.SMTPPORT = smtpd.port - By setting the mailing list to personalize messages, each recipient will get a unique copy of the message, with certain headers tailored for that recipient. >>> from mailman.interfaces import Personalization >>> mlist.personalize = Personalization.individual - >>> config.db.commit() + >>> commit() >>> msg = message_from_string("""\ ... From: aperson@example.com @@ -55,6 +46,7 @@ Normally, messages would show up in the outgoing queue after the message has been processed by the rule set and pipeline. But we can simulate that here by injecting a message directly into the outgoing queue. + >>> from mailman.configuration import config >>> msgdata = {} >>> handler = config.handlers['calculate-recipients'] >>> handler.process(mlist, msg, msgdata) @@ -86,11 +78,3 @@ Three messages have been delivered to our SMTP server, one for each recipient. test-bounces+aperson=example.com@example.com test-bounces+bperson=example.com@example.com test-bounces+cperson=example.com@example.com - - -Clean up --------- - - >>> smtpd.stop() - >>> config.SMTPHOST = old_host - >>> config.SMTPPORT = old_port diff --git a/mailman/testing/helpers.py b/mailman/testing/helpers.py index af687828f..706ef603f 100644 --- a/mailman/testing/helpers.py +++ b/mailman/testing/helpers.py @@ -162,7 +162,7 @@ class SMTPServer: """An smtp server for testing.""" host = 'localhost' - port = 9025 + port = 10825 def __init__(self): self._messages = [] @@ -174,6 +174,10 @@ class SMTPServer: """Start the smtp server in a thread.""" log.info('test SMTP server starting') self._thread.start() + smtpd = smtplib.SMTP() + smtpd.connect(self.host, self.port) + smtpd.helo('test.localhost') + smtpd.quit() def stop(self): """Stop the smtp server.""" diff --git a/mailman/testing/smtplistener.py b/mailman/testing/smtplistener.py index 1dc11e3e0..eaf9083e8 100644 --- a/mailman/testing/smtplistener.py +++ b/mailman/testing/smtplistener.py @@ -58,12 +58,13 @@ class Server(smtpd.SMTPServer): def __init__(self, localaddr, queue): smtpd.SMTPServer.__init__(self, localaddr, None) + log.info('[SMTPServer] listening: %s', localaddr) self._queue = queue def handle_accept(self): """Handle connections by creating our own Channel object.""" conn, addr = self.accept() - log.info('accepted: %s', addr) + log.info('[SMTPServer] accepted: %s', addr) Channel(self, conn, addr) def process_message(self, peer, mailfrom, rcpttos, data): @@ -72,7 +73,8 @@ class Server(smtpd.SMTPServer): message['X-Peer'] = '%s:%s' % peer message['X-MailFrom'] = mailfrom message['X-RcptTo'] = COMMASPACE.join(rcpttos) - log.info('processed message: %s', message.get('message-id', 'n/a')) + log.info('[SMTPServer] processed message: %s', + message.get('message-id', 'n/a')) self._queue.put(message) def start(self): diff --git a/mailman/testing/testing.cfg.in b/mailman/testing/testing.cfg.in index e7b23ac6d..c8e121079 100644 --- a/mailman/testing/testing.cfg.in +++ b/mailman/testing/testing.cfg.in @@ -4,11 +4,14 @@ # both the process running the tests and all sub-processes (e.g. qrunners) # must share the same configuration file. -SMTPPORT = 10825 MAX_RESTARTS = 1 MTA = None USE_LMTP = Yes +# Make sure these goes to fake domains. +MAIL_ARCHIVE_BASEURL = 'http://go.mail-archive.dev/' +MAIL_ARCHIVE_RECIPIENT = 'archive@mail-archive.dev' + add_domain('example.com', 'www.example.com') # bin/testall will add additional runtime configuration variables here. diff --git a/mailman/tests/test_documentation.py b/mailman/tests/test_documentation.py index e805b10fa..2d42f989d 100644 --- a/mailman/tests/test_documentation.py +++ b/mailman/tests/test_documentation.py @@ -29,6 +29,7 @@ import mailman from mailman.Message import Message from mailman.app.styles import style_manager from mailman.configuration import config +from mailman.testing.helpers import SMTPServer DOT = '.' @@ -54,12 +55,15 @@ def specialized_message_from_string(text): def setup(testobj): """Test setup.""" + smtpd = SMTPServer() + smtpd.start() # In general, I don't like adding convenience functions, since I think # doctests should do the imports themselves. It makes for better # documentation that way. However, a few are really useful, or help to # hide some icky test implementation details. testobj.globs['message_from_string'] = specialized_message_from_string testobj.globs['commit'] = config.db.commit + testobj.globs['smtpd'] = smtpd @@ -79,6 +83,13 @@ def cleaning_teardown(testobj): for message in config.db.message_store.messages: config.db.message_store.delete_message(message['message-id']) config.db.commit() + # Reset all archivers by disabling them. + for archiver in config.archivers.values(): + archiver.is_enabled = False + # Shutdown the smtp server. + smtpd = testobj.globs['smtpd'] + smtpd.clear() + smtpd.stop() @@ -91,10 +91,12 @@ Any other spelling is incorrect.""", 'console_scripts': list(scripts), # Entry point for plugging in different database backends. 'mailman.archiver' : [ - 'pipermail = mailman.app.archiving:Pipermail', - 'prototype = mailman.app.archiving:Prototype', + 'mail-archive = mailman.archiving.mailarchive:MailArchive', + 'mhonarc = mailman.archiving.mhonarc:MHonArc', + 'pipermail = mailman.archiving.pipermail:Pipermail', + 'prototype = mailman.archiving.prototype:Prototype', ], - 'mailman.scrubber' : 'stock = mailman.app.archiving:Pipermail', + 'mailman.scrubber' : 'stock = mailman.archiving.pipermail:Pipermail', 'mailman.commands' : list(commands), 'mailman.database' : 'stock = mailman.database:StockDatabase', 'mailman.mta' : 'stock = mailman.MTA:Manual', |