diff options
| -rw-r--r-- | mailman/Defaults.py | 5 | ||||
| -rw-r--r-- | mailman/app/archiving.py | 57 | ||||
| -rw-r--r-- | mailman/bin/testall.py | 6 | ||||
| -rw-r--r-- | mailman/docs/archivers.txt | 104 | ||||
| -rw-r--r-- | mailman/interfaces/mailinglist.py | 2 | ||||
| -rw-r--r-- | mailman/testing/helpers.py | 6 | ||||
| -rw-r--r-- | mailman/testing/smtplistener.py | 6 | ||||
| -rw-r--r-- | mailman/testing/testing.cfg.in | 5 | ||||
| -rw-r--r-- | setup.py | 1 |
9 files changed, 181 insertions, 11 deletions
diff --git a/mailman/Defaults.py b/mailman/Defaults.py index 43bf7a3c3..e665c1870 100644 --- a/mailman/Defaults.py +++ b/mailman/Defaults.py @@ -220,6 +220,11 @@ CGIEXT = '' # - $fqdn_listname -- the long name of the list being accessed PUBLIC_ARCHIVE_URL = 'http://$hostname/pipermail/$fqdn_listname' +# The public Mail-Archive.com service's base url. +MAIL_ARCHIVE_BASEURL = 'http://go.mail-archive.com/' +# The posting address for the Mail-Archive.com service +MAIL_ARCHIVE_RECIPIENT = 'archive@mail-archive.com' + # Are archives on or off by default? DEFAULT_ARCHIVE = On diff --git a/mailman/app/archiving.py b/mailman/app/archiving.py index 15e987daf..3a8e428d1 100644 --- a/mailman/app/archiving.py +++ b/mailman/app/archiving.py @@ -27,10 +27,11 @@ __all__ = [ import os import hashlib -from base64 import b32encode +from base64 import b32encode, urlsafe_b64encode from cStringIO import StringIO from email.utils import make_msgid from string import Template +from urllib import quote from urlparse import urljoin from zope.interface import implements from zope.interface.interface import adapter_hooks @@ -38,6 +39,7 @@ from zope.interface.interface import adapter_hooks from mailman.configuration import config from mailman.interfaces.archiver import IArchiver, IPipermailMailingList from mailman.interfaces.mailinglist import IMailingList +from mailman.queue import Switchboard from mailman.Archiver.HyperArch import HyperArchive @@ -79,7 +81,7 @@ class Pipermail: implements(IArchiver) name = 'pipermail' - is_enabled = True + is_enabled = False @staticmethod def list_url(mlist): @@ -152,3 +154,54 @@ class Prototype: def archive_message(mlist, message): """See `IArchiver`.""" raise NotImplementedError + + + +class MailArchive: + """Public archiver at the Mail-Archive.com. + + Messages get archived at http://go.mail-archive.com. + """ + + implements(IArchiver) + + name = 'mail-archive' + is_enabled = False + + @staticmethod + def list_url(mlist): + """See `IArchiver`.""" + if mlist.archive_private: + return None + return urljoin(config.MAIL_ARCHIVE_BASEURL, + quote(mlist.posting_address)) + + @staticmethod + def permalink(mlist, msg): + """See `IArchiver`.""" + if mlist.archive_private: + return None + message_id = msg.get('message-id') + # It is not the archiver's job to ensure the message has a Message-ID. + assert message_id is not None, 'No Message-ID found' + # The angle brackets are not part of the Message-ID. See RFC 2822. + start = (1 if message_id.startswith('<') else 0) + end = (-1 if message_id.endswith('>') else None) + message_id = message_id[start:end] + sha = hashlib.sha1(message_id) + sha.update(str(mlist.post_id)) + message_id_hash = urlsafe_b64encode(sha.digest()) + del msg['x-message-id-hash'] + msg['X-Message-ID-Hash'] = message_id_hash + return urljoin(config.MAIL_ARCHIVE_BASEURL, message_id_hash) + + @staticmethod + def archive_message(mlist, msg): + """See `IArchiver`.""" + if mlist.archive_private: + return + outq = Switchboard(config.OUTQUEUE_DIR) + outq.enqueue( + msg, + listname=mlist.fqdn_listname, + recips=[config.MAIL_ARCHIVE_RECIPIENT]) diff --git a/mailman/bin/testall.py b/mailman/bin/testall.py index e7b3c445b..880130172 100644 --- a/mailman/bin/testall.py +++ b/mailman/bin/testall.py @@ -34,6 +34,7 @@ import pkg_resources from mailman.configuration import config from mailman.i18n import _ from mailman.initialize import initialize_1, initialize_2 +from mailman.testing.helpers import SMTPServer from mailman.version import MAILMAN_VERSION @@ -226,10 +227,11 @@ def main(): with open(cfg_out, 'a') as fp: print >> fp, 'VAR_DIR = "%s"' % var_dir print >> fp, 'MAILMAN_USER = "%s"' % user_name - print >> fp, 'MAILMAN_UID = %d' % user_id + print >> fp, 'MAILMAN_UID =', user_id print >> fp, 'MAILMAN_GROUP = "%s"' % group_name - print >> fp, 'MAILMAN_GID = %d' % group_id + print >> fp, 'MAILMAN_GID =', group_id print >> fp, "LANGUAGES = 'en'" + print >> fp, 'SMTPPORT =', SMTPServer.port initialize_1(cfg_out, propagate_logs=parser.options.stderr) mailman_uid = pwd.getpwnam(config.MAILMAN_USER).pw_uid diff --git a/mailman/docs/archivers.txt b/mailman/docs/archivers.txt index 9e4fbc121..56f81b98a 100644 --- a/mailman/docs/archivers.txt +++ b/mailman/docs/archivers.txt @@ -1,4 +1,5 @@ -= Archivers = +Archivers +========= Mailman supports pluggable archivers, and it comes with several default archivers. @@ -32,6 +33,9 @@ interoperate. ... print ' ', archiver.list_url(mlist) ... print ' ', archiver.permalink(mlist, msg) ... archivers[archiver.name] = archiver + mail-archive + http://go.mail-archive.dev/test%40example.com + http://go.mail-archive.dev/7ekn-OQjGKjbAsD3StwtnhZ3Azk= pipermail http://www.example.com/pipermail/test@example.com None @@ -40,7 +44,8 @@ interoperate. http://www.example.com/RSZCG7IGPHFIRW3EMTVMMDNJMNCVCOLE -== Sending the message to the archiver == +Sending the message to the archiver +----------------------------------- The archiver is also able to archive the message. @@ -61,3 +66,98 @@ Note however that the prototype archiver can't archive messages. Traceback (most recent call last): ... NotImplementedError + + +The Mail-Archive.com +-------------------- + +The Mail-Archive <http://www.mail-archive.com> is a public archiver that can +be used to archive message for free. Mailman comes with a plugin for this +archiver; by enabling it messages to public lists will get sent there +automatically. + + >>> archiver = archivers['mail-archive'] + >>> archiver.list_url(mlist) + 'http://go.mail-archive.dev/test%40example.com' + + >>> mlist.post_id = 77 + >>> archiver.permalink(mlist, msg) + 'http://go.mail-archive.dev/8FhWFYqR61sszkBLk4rEc0YNWks=' + +To archive the message, the archiver actually mails the message to a special +address at the Mail-Archive. + + >>> archiver.archive_message(mlist, msg) + + >>> from mailman.testing.helpers import SMTPServer + >>> smtpd = SMTPServer() + >>> smtpd.start() + + >>> from mailman.queue.outgoing import OutgoingRunner + >>> from mailman.testing.helpers import make_testable_runner + >>> outgoing = make_testable_runner(OutgoingRunner) + >>> outgoing.run() + + >>> from operator import itemgetter + >>> messages = list(smtpd.messages) + >>> len(messages) + 1 + + >>> print messages[0].as_string() + From: aperson@example.org + To: test@example.com + Subject: An archived message + Message-ID: <12345> + X-Message-ID-Hash: 8FhWFYqR61sszkBLk4rEc0YNWks= + MIME-Version: 1.0 + Content-Type: text/plain; charset="us-ascii" + Content-Transfer-Encoding: 7bit + Sender: test-bounces@example.com + Errors-To: test-bounces@example.com + X-Peer: 127.0.0.1:... + X-MailFrom: test-bounces@example.com + X-RcptTo: archive@mail-archive.dev + <BLANKLINE> + Here is an archived message. + _______________________________________________ + Test mailing list + test@example.com + http://lists.example.com/listinfo/test@example.com + + >>> smtpd.clear() + +However, if the mailing list is not public, the message will never be archived +at this service. + + >>> mlist.archive_private = True + >>> print archiver.list_url(mlist) + None + >>> print archiver.permalink(mlist, msg) + None + >>> archiver.archive_message(mlist, msg) + >>> list(smtpd.messages) + [] + +Additionally, this archiver can handle malformed Message-IDs. + + >>> mlist.archive_private = False + >>> del msg['message-id'] + >>> msg['Message-ID'] = '12345>' + >>> archiver.permalink(mlist, msg) + 'http://go.mail-archive.dev/8FhWFYqR61sszkBLk4rEc0YNWks=' + + >>> del msg['message-id'] + >>> msg['Message-ID'] = '<12345' + >>> archiver.permalink(mlist, msg) + 'http://go.mail-archive.dev/8FhWFYqR61sszkBLk4rEc0YNWks=' + + >>> del msg['message-id'] + >>> msg['Message-ID'] = '12345' + >>> archiver.permalink(mlist, msg) + 'http://go.mail-archive.dev/8FhWFYqR61sszkBLk4rEc0YNWks=' + + +Clean up +-------- + + >>> smtpd.stop() diff --git a/mailman/interfaces/mailinglist.py b/mailman/interfaces/mailinglist.py index a5f6a9e9a..8a9967e89 100644 --- a/mailman/interfaces/mailinglist.py +++ b/mailman/interfaces/mailinglist.py @@ -147,7 +147,7 @@ class IMailingList(Interface): last_post_date = Attribute( """The date and time a message was last posted to the mailing list.""") - post_number = Attribute( + post_id = Attribute( """A monotonically increasing integer sequentially assigned to each list posting.""") diff --git a/mailman/testing/helpers.py b/mailman/testing/helpers.py index af687828f..706ef603f 100644 --- a/mailman/testing/helpers.py +++ b/mailman/testing/helpers.py @@ -162,7 +162,7 @@ class SMTPServer: """An smtp server for testing.""" host = 'localhost' - port = 9025 + port = 10825 def __init__(self): self._messages = [] @@ -174,6 +174,10 @@ class SMTPServer: """Start the smtp server in a thread.""" log.info('test SMTP server starting') self._thread.start() + smtpd = smtplib.SMTP() + smtpd.connect(self.host, self.port) + smtpd.helo('test.localhost') + smtpd.quit() def stop(self): """Stop the smtp server.""" diff --git a/mailman/testing/smtplistener.py b/mailman/testing/smtplistener.py index 1dc11e3e0..eaf9083e8 100644 --- a/mailman/testing/smtplistener.py +++ b/mailman/testing/smtplistener.py @@ -58,12 +58,13 @@ class Server(smtpd.SMTPServer): def __init__(self, localaddr, queue): smtpd.SMTPServer.__init__(self, localaddr, None) + log.info('[SMTPServer] listening: %s', localaddr) self._queue = queue def handle_accept(self): """Handle connections by creating our own Channel object.""" conn, addr = self.accept() - log.info('accepted: %s', addr) + log.info('[SMTPServer] accepted: %s', addr) Channel(self, conn, addr) def process_message(self, peer, mailfrom, rcpttos, data): @@ -72,7 +73,8 @@ class Server(smtpd.SMTPServer): message['X-Peer'] = '%s:%s' % peer message['X-MailFrom'] = mailfrom message['X-RcptTo'] = COMMASPACE.join(rcpttos) - log.info('processed message: %s', message.get('message-id', 'n/a')) + log.info('[SMTPServer] processed message: %s', + message.get('message-id', 'n/a')) self._queue.put(message) def start(self): diff --git a/mailman/testing/testing.cfg.in b/mailman/testing/testing.cfg.in index e7b23ac6d..c8e121079 100644 --- a/mailman/testing/testing.cfg.in +++ b/mailman/testing/testing.cfg.in @@ -4,11 +4,14 @@ # both the process running the tests and all sub-processes (e.g. qrunners) # must share the same configuration file. -SMTPPORT = 10825 MAX_RESTARTS = 1 MTA = None USE_LMTP = Yes +# Make sure these goes to fake domains. +MAIL_ARCHIVE_BASEURL = 'http://go.mail-archive.dev/' +MAIL_ARCHIVE_RECIPIENT = 'archive@mail-archive.dev' + add_domain('example.com', 'www.example.com') # bin/testall will add additional runtime configuration variables here. @@ -93,6 +93,7 @@ Any other spelling is incorrect.""", 'mailman.archiver' : [ 'pipermail = mailman.app.archiving:Pipermail', 'prototype = mailman.app.archiving:Prototype', + 'mail-archive = mailman.app.archiving:MailArchive', ], 'mailman.scrubber' : 'stock = mailman.app.archiving:Pipermail', 'mailman.commands' : list(commands), |