diff options
| author | Barry Warsaw | 2012-03-13 21:03:01 -0700 |
|---|---|---|
| committer | Barry Warsaw | 2012-03-13 21:03:01 -0700 |
| commit | 80ac803cb2816dde0503671fd117ed134680de53 (patch) | |
| tree | ed2d40855f6b45dedf421a7a98bd83d95b6e3b40 | |
| parent | 07685642934fe934098927a9a9d20c17080f3dab (diff) | |
| download | mailman-80ac803cb2816dde0503671fd117ed134680de53.tar.gz mailman-80ac803cb2816dde0503671fd117ed134680de53.tar.zst mailman-80ac803cb2816dde0503671fd117ed134680de53.zip | |
| -rw-r--r-- | src/mailman/app/docs/pipelines.rst | 31 | ||||
| -rw-r--r-- | src/mailman/app/inject.py | 14 | ||||
| -rw-r--r-- | src/mailman/app/tests/test_inject.py | 77 | ||||
| -rw-r--r-- | src/mailman/archiving/docs/common.rst | 25 | ||||
| -rw-r--r-- | src/mailman/archiving/mailarchive.py | 22 | ||||
| -rw-r--r-- | src/mailman/archiving/mhonarc.py | 20 | ||||
| -rw-r--r-- | src/mailman/archiving/prototype.py | 21 | ||||
| -rw-r--r-- | src/mailman/commands/docs/inject.rst | 53 | ||||
| -rw-r--r-- | src/mailman/docs/NEWS.rst | 9 | ||||
| -rw-r--r-- | src/mailman/pipeline/decorate.py | 3 | ||||
| -rw-r--r-- | src/mailman/pipeline/docs/rfc-2369.rst | 1 | ||||
| -rw-r--r-- | src/mailman/runners/docs/incoming.rst | 1 | ||||
| -rw-r--r-- | src/mailman/runners/docs/lmtp.rst | 27 | ||||
| -rw-r--r-- | src/mailman/runners/lmtp.py | 17 | ||||
| -rw-r--r-- | src/mailman/runners/tests/test_confirm.py | 6 | ||||
| -rw-r--r-- | src/mailman/runners/tests/test_lmtp.py | 98 | ||||
| -rw-r--r-- | src/mailman/testing/helpers.py | 5 | ||||
| -rw-r--r-- | src/mailman/testing/layers.py | 35 | ||||
| -rw-r--r-- | src/mailman/utilities/email.py | 33 | ||||
| -rw-r--r-- | src/mailman/utilities/tests/test_email.py | 75 | ||||
| -rw-r--r-- | src/mailman/version.py | 12 |
21 files changed, 445 insertions, 140 deletions
diff --git a/src/mailman/app/docs/pipelines.rst b/src/mailman/app/docs/pipelines.rst index fbd405f68..56fdeb5c6 100644 --- a/src/mailman/app/docs/pipelines.rst +++ b/src/mailman/app/docs/pipelines.rst @@ -24,6 +24,7 @@ Messages hit the pipeline after they've been accepted for posting. ... To: test@example.com ... Subject: My first post ... Message-ID: <first> + ... X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB ... ... First post! ... """) @@ -37,15 +38,15 @@ etc. From: aperson@example.com To: test@example.com Message-ID: <first> + X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB Subject: [Test] My first post X-Mailman-Version: ... Precedence: list List-Id: <test.example.com> - X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB List-Post: <mailto:test@example.com> List-Subscribe: <http://lists.example.com/listinfo/test@example.com>, <mailto:test-join@example.com> - Archived-At: http://lists.example.com/archives/4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB + Archived-At: http://lists.example.com/.../4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB List-Unsubscribe: <http://lists.example.com/listinfo/test@example.com>, <mailto:test-leave@example.com> List-Archive: <http://lists.example.com/archives/test@example.com> @@ -65,15 +66,18 @@ However there are currently no recipients for this message. After pipeline processing, the message is now sitting in various other processing queues. +:: >>> from mailman.testing.helpers import get_queue_messages >>> messages = get_queue_messages('archive') >>> len(messages) 1 + >>> print messages[0].msg.as_string() From: aperson@example.com To: test@example.com Message-ID: <first> + X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB Subject: [Test] My first post X-Mailman-Version: ... Precedence: list @@ -82,6 +86,7 @@ processing queues. <BLANKLINE> First post! <BLANKLINE> + >>> dump_msgdata(messages[0].msgdata) _parsemsg : False original_sender : aperson@example.com @@ -97,15 +102,19 @@ the outgoing nntp queue. >>> len(messages) 0 -This is the message that will actually get delivered to end recipients. +The outgoing queue will hold the copy of the message that will actually get +delivered to end recipients. +:: >>> messages = get_queue_messages('out') >>> len(messages) 1 + >>> print messages[0].msg.as_string() From: aperson@example.com To: test@example.com Message-ID: <first> + X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB Subject: [Test] My first post X-Mailman-Version: ... Precedence: list @@ -114,6 +123,7 @@ This is the message that will actually get delivered to end recipients. <BLANKLINE> First post! <BLANKLINE> + >>> dump_msgdata(messages[0].msgdata) _parsemsg : False listname : test@example.com @@ -124,15 +134,18 @@ This is the message that will actually get delivered to end recipients. version : 3 There's now one message in the digest mailbox, getting ready to be sent. +:: >>> from mailman.testing.helpers import digest_mbox >>> digest = digest_mbox(mlist) >>> sum(1 for mboxmsg in digest) 1 + >>> print list(digest)[0].as_string() From: aperson@example.com To: test@example.com Message-ID: <first> + X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB Subject: [Test] My first post X-Mailman-Version: ... Precedence: list @@ -143,10 +156,8 @@ There's now one message in the digest mailbox, getting ready to be sent. <BLANKLINE> -Clean up the digests -==================== - - >>> digest.clear() - >>> digest.flush() - >>> sum(1 for msg in digest_mbox(mlist)) - 0 +.. Clean up the digests + >>> digest.clear() + >>> digest.flush() + >>> sum(1 for msg in digest_mbox(mlist)) + 0 diff --git a/src/mailman/app/inject.py b/src/mailman/app/inject.py index 8bf907af0..5dcaf5584 100644 --- a/src/mailman/app/inject.py +++ b/src/mailman/app/inject.py @@ -17,7 +17,7 @@ """Inject a message into a queue.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -31,12 +31,16 @@ from email.utils import formatdate, make_msgid from mailman.config import config from mailman.email.message import Message +from mailman.utilities.email import add_message_hash def inject_message(mlist, msg, recipients=None, switchboard=None, **kws): """Inject a message into a queue. + If the message does not have a Message-ID header, one is added. An + X-Message-Id-Hash header is also always added. + :param mlist: The mailing list this message is destined for. :type mlist: IMailingList :param msg: The Message object to inject. @@ -56,12 +60,14 @@ def inject_message(mlist, msg, recipients=None, switchboard=None, **kws): # message has a Message-ID. if 'message-id' not in msg: msg['Message-ID'] = make_msgid() + add_message_hash(msg) # Ditto for Date: as required by RFC 2822. if 'date' not in msg: msg['Date'] = formatdate(localtime=True) + msg.original_size = len(msg.as_string()) msgdata = dict( listname=mlist.fqdn_listname, - original_size=getattr(msg, 'original_size', len(msg.as_string())), + original_size=msg.original_size, ) msgdata.update(kws) if recipients is not None: @@ -73,6 +79,9 @@ def inject_message(mlist, msg, recipients=None, switchboard=None, **kws): def inject_text(mlist, text, recipients=None, switchboard=None, **kws): """Turn text into a message and inject that into a queue. + If the text does not have a Message-ID header, one is added. An + X-Message-Id-Hash header is also always added. + :param mlist: The mailing list this message is destined for. :type mlist: IMailingList :param text: The text of the message to inject. This will be parsed into @@ -88,5 +97,4 @@ def inject_text(mlist, text, recipients=None, switchboard=None, **kws): :type kws: dictionary """ message = message_from_string(text, Message) - message.original_size = len(text) inject_message(mlist, message, recipients, switchboard, **kws) diff --git a/src/mailman/app/tests/test_inject.py b/src/mailman/app/tests/test_inject.py index de7e4bd29..4e08ff0a7 100644 --- a/src/mailman/app/tests/test_inject.py +++ b/src/mailman/app/tests/test_inject.py @@ -17,7 +17,7 @@ """Testing app.inject functions.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -108,14 +108,6 @@ Nothing. self.assertTrue('date' in items[0].msg) self.assertEqual(items[0].msg['date'], self.msg['date']) - def test_inject_message_trusts_original_size_attribute(self): - # If the message object has an `original_size` attribute, this is - # copied into the metadata. - self.msg.original_size = 3 - inject_message(self.mlist, self.msg) - items = get_queue_messages('in') - self.assertEqual(items[0].msgdata['original_size'], 3) - def test_inject_message_with_keywords(self): # Keyword arguments are copied into the metadata. inject_message(self.mlist, self.msg, foo='yes', bar='no') @@ -123,6 +115,26 @@ Nothing. self.assertEqual(items[0].msgdata['foo'], 'yes') self.assertEqual(items[0].msgdata['bar'], 'no') + def test_inject_message_id_hash(self): + # When the injected message has a Message-ID header, the injected + # message will also get an X-Message-ID-Hash header. + inject_message(self.mlist, self.msg) + items = get_queue_messages('in') + self.assertEqual(items[0].msg['x-message-id-hash'], + '4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB') + + def test_inject_message_id_hash_without_message_id(self): + # When the injected message does not have a Message-ID header, a + # Message-ID header will be added, and the injected message will also + # get an X-Message-ID-Hash header. + del self.msg['message-id'] + self.assertFalse('message-id' in self.msg) + self.assertFalse('x-message-id-hash' in self.msg) + inject_message(self.mlist, self.msg) + items = get_queue_messages('in') + self.assertTrue('message-id' in items[0].msg) + self.assertTrue('x-message-id-hash' in items[0].msg) + class TestInjectText(unittest.TestCase): @@ -155,9 +167,18 @@ Nothing. items = get_queue_messages('in') self.assertEqual(len(items), 1) self.assertTrue(isinstance(items[0].msg, Message)) + self.assertEqual(items[0].msg['x-message-id-hash'], + 'GUXXQKNCHBFQAHGBFMGCME6HKZCUUH3K') + # Delete that header because it is not in the original text. + del items[0].msg['x-message-id-hash'] self.eq(items[0].msg.as_string(), self.text) self.assertEqual(items[0].msgdata['listname'], 'test@example.com') - self.assertEqual(items[0].msgdata['original_size'], len(self.text)) + self.assertEqual(items[0].msgdata['original_size'], + # Add back the X-Message-ID-Header which was in the + # message contributing to the original_size, but + # wasn't in the original text. Don't forget the + # newline! + len(self.text) + 52) def test_inject_text_with_recipients(self): # Explicit recipients end up in the metadata. @@ -173,9 +194,16 @@ Nothing. self.assertEqual(len(items), 0) items = get_queue_messages('virgin') self.assertEqual(len(items), 1) + # Remove the X-Message-ID-Hash header which isn't in the original text. + del items[0].msg['x-message-id-hash'] self.eq(items[0].msg.as_string(), self.text) self.assertEqual(items[0].msgdata['listname'], 'test@example.com') - self.assertEqual(items[0].msgdata['original_size'], len(self.text)) + self.assertEqual(items[0].msgdata['original_size'], + # Add back the X-Message-ID-Header which was in the + # message contributing to the original_size, but + # wasn't in the original text. Don't forget the + # newline! + len(self.text) + 52) def test_inject_text_without_message_id(self): # inject_text() adds a Message-ID header if it's missing. @@ -198,7 +226,12 @@ Nothing. # the injected text. inject_text(self.mlist, self.text) items = get_queue_messages('in') - self.assertEqual(items[0].msgdata['original_size'], len(self.text)) + self.assertEqual(items[0].msgdata['original_size'], + # Add back the X-Message-ID-Header which was in the + # message contributing to the original_size, but + # wasn't in the original text. Don't forget the + # newline! + len(self.text) + 52) def test_inject_text_with_keywords(self): # Keyword arguments are copied into the metadata. @@ -206,3 +239,23 @@ Nothing. items = get_queue_messages('in') self.assertEqual(items[0].msgdata['foo'], 'yes') self.assertEqual(items[0].msgdata['bar'], 'no') + + def test_inject_message_id_hash(self): + # When the injected message has a Message-ID header, the injected + # message will also get an X-Message-ID-Hash header. + inject_text(self.mlist, self.text) + items = get_queue_messages('in') + self.assertEqual(items[0].msg['x-message-id-hash'], + 'GUXXQKNCHBFQAHGBFMGCME6HKZCUUH3K') + + def test_inject_message_id_hash_without_message_id(self): + # When the injected message does not have a Message-ID header, a + # Message-ID header will be added, and the injected message will also + # get an X-Message-ID-Hash header. + filtered = self._remove_line('message-id') + self.assertFalse('Message-ID' in filtered) + self.assertFalse('X-Message-ID-Hash' in filtered) + inject_text(self.mlist, filtered) + items = get_queue_messages('in') + self.assertTrue('message-id' in items[0].msg) + self.assertTrue('x-message-id-hash' in items[0].msg) diff --git a/src/mailman/archiving/docs/common.rst b/src/mailman/archiving/docs/common.rst index 330c8e307..45ec8f194 100644 --- a/src/mailman/archiving/docs/common.rst +++ b/src/mailman/archiving/docs/common.rst @@ -11,6 +11,7 @@ archivers. ... To: test@example.com ... Subject: An archived message ... Message-ID: <12345> + ... X-Message-ID-Hash: RSZCG7IGPHFIRW3EMTVMMDNJMNCVCOLE ... ... Here is an archived message. ... """) @@ -33,7 +34,7 @@ interoperate. ... archivers[archiver.name] = archiver mail-archive http://go.mail-archive.dev/test%40example.com - http://go.mail-archive.dev/ZaXPPxRMM9_hFZL4vTRlQlBx8pc= + http://go.mail-archive.dev/RSZCG7IGPHFIRW3EMTVMMDNJMNCVCOLE mhonarc http://lists.example.com/.../test@example.com http://lists.example.com/.../RSZCG7IGPHFIRW3EMTVMMDNJMNCVCOLE @@ -80,7 +81,7 @@ messages to public lists will get sent there automatically. >>> print archiver.list_url(mlist) http://go.mail-archive.dev/test%40example.com >>> print archiver.permalink(mlist, msg) - http://go.mail-archive.dev/ZaXPPxRMM9_hFZL4vTRlQlBx8pc= + http://go.mail-archive.dev/RSZCG7IGPHFIRW3EMTVMMDNJMNCVCOLE To archive the message, the archiver actually mails the message to a special address at The Mail Archive. The message gets no header or footer decoration. @@ -103,7 +104,7 @@ address at The Mail Archive. The message gets no header or footer decoration. To: test@example.com Subject: An archived message Message-ID: <12345> - X-Message-ID-Hash: ZaXPPxRMM9_hFZL4vTRlQlBx8pc= + X-Message-ID-Hash: RSZCG7IGPHFIRW3EMTVMMDNJMNCVCOLE X-Peer: 127.0.0.1:... X-MailFrom: test-bounces@example.com X-RcptTo: archive@mail-archive.dev @@ -127,26 +128,36 @@ at this service. Additionally, this archiver can handle malformed ``Message-IDs``. :: + >>> from mailman.utilities.email import add_message_hash >>> mlist.archive_private = False >>> del msg['message-id'] + >>> del msg['x-message-id-hash'] >>> msg['Message-ID'] = '12345>' + >>> add_message_hash(msg) >>> print archiver.permalink(mlist, msg) - http://go.mail-archive.dev/bXvG32YzcDEIVDaDLaUSVQekfo8= + http://go.mail-archive.dev/YJIGBYRWZFG5LZEBQ7NR25B5HBR2BVD6 >>> del msg['message-id'] + >>> del msg['x-message-id-hash'] >>> msg['Message-ID'] = '<12345' + >>> add_message_hash(msg) >>> print archiver.permalink(mlist, msg) - http://go.mail-archive.dev/9rockPrT1Mm-jOsLWS6_hseR_OY= + http://go.mail-archive.dev/XUFFJNJ2P2WC4NDPQRZFDJMV24POP64B >>> del msg['message-id'] + >>> del msg['x-message-id-hash'] >>> msg['Message-ID'] = '12345' + >>> add_message_hash(msg) >>> print archiver.permalink(mlist, msg) - http://go.mail-archive.dev/ZaXPPxRMM9_hFZL4vTRlQlBx8pc= + http://go.mail-archive.dev/RSZCG7IGPHFIRW3EMTVMMDNJMNCVCOLE >>> del msg['message-id'] + >>> del msg['x-message-id-hash'] + >>> add_message_hash(msg) >>> msg['Message-ID'] = ' 12345 ' + >>> add_message_hash(msg) >>> print archiver.permalink(mlist, msg) - http://go.mail-archive.dev/ZaXPPxRMM9_hFZL4vTRlQlBx8pc= + http://go.mail-archive.dev/RSZCG7IGPHFIRW3EMTVMMDNJMNCVCOLE MHonArc diff --git a/src/mailman/archiving/mailarchive.py b/src/mailman/archiving/mailarchive.py index 29c0e22c2..c72cde11c 100644 --- a/src/mailman/archiving/mailarchive.py +++ b/src/mailman/archiving/mailarchive.py @@ -25,9 +25,6 @@ __all__ = [ ] -import hashlib - -from base64 import urlsafe_b64encode from urllib import quote from urlparse import urljoin from zope.interface import implements @@ -60,21 +57,12 @@ class MailArchive: """See `IArchiver`.""" if mlist.archive_private: return None - message_id = msg.get('message-id') - # It is not the archiver's job to ensure the message has a Message-ID. - # If no Message-ID is available, there is no permalink. - if message_id is None: + # It is the LMTP server's responsibility to ensure that the message + # has a X-Message-ID-Hash header. If it doesn't then there's no + # permalink. + message_id_hash = msg.get('x-message-id-hash') + if message_id_hash is None: return None - # The angle brackets are not part of the Message-ID. See RFC 2822. - if message_id.startswith('<') and message_id.endswith('>'): - message_id = message_id[1:-1] - else: - message_id = message_id.strip() - sha = hashlib.sha1(message_id) - sha.update(str(mlist.posting_address)) - message_id_hash = urlsafe_b64encode(sha.digest()) - del msg['x-message-id-hash'] - msg['X-Message-ID-Hash'] = message_id_hash return urljoin(config.archiver.mail_archive.base_url, message_id_hash) @staticmethod diff --git a/src/mailman/archiving/mhonarc.py b/src/mailman/archiving/mhonarc.py index db0b12c3d..0beeed73e 100644 --- a/src/mailman/archiving/mhonarc.py +++ b/src/mailman/archiving/mhonarc.py @@ -25,11 +25,9 @@ __all__ = [ ] -import hashlib import logging import subprocess -from base64 import b32encode from urlparse import urljoin from zope.interface import implements @@ -63,20 +61,12 @@ class MHonArc: def permalink(mlist, msg): """See `IArchiver`.""" # XXX What about private MHonArc archives? - message_id = msg.get('message-id') - # It is not the archiver's job to ensure the message has a Message-ID. - # If no Message-ID is available, there is no permalink. - if message_id is None: + # It is the LMTP server's responsibility to ensure that the message + # has a X-Message-ID-Hash header. If it doesn't then there's no + # permalink. + message_id_hash = msg.get('x-message-id-hash') + if message_id_hash is None: return None - # The angle brackets are not part of the Message-ID. See RFC 2822. - if message_id.startswith('<') and message_id.endswith('>'): - message_id = message_id[1:-1] - else: - message_id = message_id.strip() - sha = hashlib.sha1(message_id) - message_id_hash = b32encode(sha.digest()) - del msg['x-message-id-hash'] - msg['X-Message-ID-Hash'] = message_id_hash return urljoin(MHonArc.list_url(mlist), message_id_hash) @staticmethod diff --git a/src/mailman/archiving/prototype.py b/src/mailman/archiving/prototype.py index 0a857fd77..55d78074e 100644 --- a/src/mailman/archiving/prototype.py +++ b/src/mailman/archiving/prototype.py @@ -25,9 +25,6 @@ __all__ = [ ] -import hashlib - -from base64 import b32encode from urlparse import urljoin from zope.interface import implements @@ -54,20 +51,12 @@ class Prototype: @staticmethod def permalink(mlist, msg): """See `IArchiver`.""" - message_id = msg.get('message-id') - # It is not the archiver's job to ensure the message has a Message-ID. - # If this header is missing, there is no permalink. - if message_id is None: + # It is the LMTP server's responsibility to ensure that the message + # has a X-Message-ID-Hash header. If it doesn't then there's no + # permalink. + message_id_hash = msg.get('x-message-id-hash') + if message_id_hash is None: return None - # The angle brackets are not part of the Message-ID. See RFC 2822. - if message_id.startswith('<') and message_id.endswith('>'): - message_id = message_id[1:-1] - else: - message_id = message_id.strip() - digest = hashlib.sha1(message_id).digest() - message_id_hash = b32encode(digest) - del msg['x-message-id-hash'] - msg['X-Message-ID-Hash'] = message_id_hash return urljoin(Prototype.list_url(mlist), message_id_hash) @staticmethod diff --git a/src/mailman/commands/docs/inject.rst b/src/mailman/commands/docs/inject.rst index e8fc088c0..7150beac7 100644 --- a/src/mailman/commands/docs/inject.rst +++ b/src/mailman/commands/docs/inject.rst @@ -54,6 +54,7 @@ Usually, the text of the message to inject is in a file. ... From: aperson@example.com ... To: test@example.com ... Subject: testing + ... Message-ID: <aardvark> ... ... This is a test message. ... """ @@ -69,22 +70,20 @@ Let's provide a list name and try again. >>> mlist = create_list('test@example.com') >>> transaction.commit() + >>> from mailman.testing.helpers import get_queue_messages - >>> in_queue = config.switchboards['in'] - >>> len(in_queue.files) - 0 + >>> get_queue_messages('in') + [] >>> args.listname = ['test@example.com'] >>> command.process(args) By default, the incoming queue is used. :: - >>> len(in_queue.files) + >>> items = get_queue_messages('in') + >>> len(items) 1 - - >>> from mailman.testing.helpers import get_queue_messages - >>> item = get_queue_messages('in')[0] - >>> print item.msg.as_string() + >>> print items[0].msg.as_string() From: aperson@example.com To: test@example.com Subject: testing @@ -95,10 +94,10 @@ By default, the incoming queue is used. <BLANKLINE> <BLANKLINE> - >>> dump_msgdata(item.msgdata) + >>> dump_msgdata(items[0].msgdata) _parsemsg : False listname : test@example.com - original_size: 90 + original_size: 203 version : 3 But a different queue can be specified on the command line. @@ -107,13 +106,12 @@ But a different queue can be specified on the command line. >>> args.queue = 'virgin' >>> command.process(args) - >>> len(in_queue.files) - 0 - >>> virgin_queue = config.switchboards['virgin'] - >>> len(virgin_queue.files) + >>> get_queue_messages('in') + [] + >>> items = get_queue_messages('virgin') + >>> len(items) 1 - >>> item = get_queue_messages('virgin')[0] - >>> print item.msg.as_string() + >>> print items[0].msg.as_string() From: aperson@example.com To: test@example.com Subject: testing @@ -124,10 +122,10 @@ But a different queue can be specified on the command line. <BLANKLINE> <BLANKLINE> - >>> dump_msgdata(item.msgdata) + >>> dump_msgdata(items[0].msgdata) _parsemsg : False listname : test@example.com - original_size: 90 + original_size: 203 version : 3 @@ -144,6 +142,7 @@ The message text can also be provided on standard input. ... From: bperson@example.com ... To: test@example.com ... Subject: another test + ... Message-ID: <badger> ... ... This is another test message. ... """)) @@ -154,10 +153,10 @@ The message text can also be provided on standard input. >>> args.queue = None >>> command.process(args) - >>> len(in_queue.files) + >>> items = get_queue_messages('in') + >>> len(items) 1 - >>> item = get_queue_messages('in')[0] - >>> print item.msg.as_string() + >>> print items[0].msg.as_string() From: bperson@example.com To: test@example.com Subject: another test @@ -168,15 +167,15 @@ The message text can also be provided on standard input. <BLANKLINE> <BLANKLINE> - >>> dump_msgdata(item.msgdata) + >>> dump_msgdata(items[0].msgdata) _parsemsg : False listname : test@example.com - original_size: 100 + original_size: 211 version : 3 - # Clean up. - >>> sys.stdin = sys.__stdin__ - >>> args.filename = filename +.. Clean up. + >>> sys.stdin = sys.__stdin__ + >>> args.filename = filename Metadata @@ -199,7 +198,7 @@ injected. bar : two foo : one listname : test@example.com - original_size: 90 + original_size: 203 version : 3 diff --git a/src/mailman/docs/NEWS.rst b/src/mailman/docs/NEWS.rst index b065c0ce3..6869e2889 100644 --- a/src/mailman/docs/NEWS.rst +++ b/src/mailman/docs/NEWS.rst @@ -37,6 +37,15 @@ Architecture `default-posting-chain` while the `built-in` pipeline is renamed `default-posting-pipeline`. * The experimental `maildir` runner is removed. Use LMTP. + * The LMTP server now requires that the incoming message have a `Message-ID`, + otherwise it rejects the message with a 550 error. Also, the LMTP server + adds the `X-Message-ID-Hash` header automatically. The `inject` cli + command will also add the `X-Message-ID-Hash` header, but it will craft a + `Message-ID` header first if one is missing from the injected text. Also, + `inject` will always set the correct value for the `original_size` + attribute on the message object, instead of trusting a possibly incorrect + value if it's already set. The individual `IArchiver` implementations no + longer set the `X-Message-ID-Hash` header. Database -------- diff --git a/src/mailman/pipeline/decorate.py b/src/mailman/pipeline/decorate.py index b785e5ecf..1291a2668 100644 --- a/src/mailman/pipeline/decorate.py +++ b/src/mailman/pipeline/decorate.py @@ -17,7 +17,7 @@ """Decorate a message by sticking the header and footer around it.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -37,7 +37,6 @@ from mailman.core.i18n import _ from mailman.email.message import Message from mailman.interfaces.handler import IHandler from mailman.interfaces.templates import ITemplateLoader -from mailman.interfaces.usermanager import IUserManager from mailman.utilities.string import expand diff --git a/src/mailman/pipeline/docs/rfc-2369.rst b/src/mailman/pipeline/docs/rfc-2369.rst index bb9276157..a1ba6c746 100644 --- a/src/mailman/pipeline/docs/rfc-2369.rst +++ b/src/mailman/pipeline/docs/rfc-2369.rst @@ -193,6 +193,7 @@ The *prototype* archiver can calculate this archive url given a `Message-ID`. >>> msg = message_from_string("""\ ... From: aperson@example.com ... Message-ID: <first> + ... X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB ... ... """) >>> process(mlist, msg, {}) diff --git a/src/mailman/runners/docs/incoming.rst b/src/mailman/runners/docs/incoming.rst index e5f544a85..4a9db778e 100644 --- a/src/mailman/runners/docs/incoming.rst +++ b/src/mailman/runners/docs/incoming.rst @@ -121,6 +121,7 @@ Now the message is in the pipeline queue. To: test@example.com Subject: My first post Message-ID: <first> + X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB Date: ... X-Mailman-Rule-Misses: approved; emergency; loop; member-moderation; administrivia; implicit-dest; max-recipients; max-size; diff --git a/src/mailman/runners/docs/lmtp.rst b/src/mailman/runners/docs/lmtp.rst index 1ab42410b..2b6c4b42b 100644 --- a/src/mailman/runners/docs/lmtp.rst +++ b/src/mailman/runners/docs/lmtp.rst @@ -7,7 +7,9 @@ support LMTP local delivery, so this is a very portable way to connect Mailman with your mail server. Our LMTP server is fairly simple though; all it does is make sure that the -message is destined for a valid endpoint, e.g. ``mylist-join@example.com``. +message is destined for a valid endpoint, e.g. ``mylist-join@example.com``, +that the message bytes can be parsed into a message object, and that the +message has a `Message-ID` header. Let's start a testable LMTP runner. @@ -62,6 +64,9 @@ Once the mailing list is created, the posting address is valid. ... """) {} +Since the message itself is valid, it gets parsed and lands in the incoming +queue. + >>> from mailman.testing.helpers import get_queue_messages >>> messages = get_queue_messages('in') >>> len(messages) @@ -71,6 +76,7 @@ Once the mailing list is created, the posting address is valid. To: mylist@example.com Subject: An interesting message Message-ID: <badger> + X-Message-ID-Hash: JYMZWSQ4IC2JPKK7ZUONRFRVC4ZYJGKJ X-MailFrom: anne.person@example.com <BLANKLINE> This is an interesting message. @@ -85,8 +91,8 @@ Once the mailing list is created, the posting address is valid. Sub-addresses ============= -The LMTP server understands each of the list's sub-addreses, such as -join, --leave, -request and so on. If the message is posted to an invalid +The LMTP server understands each of the list's sub-addreses, such as `-join`, +`-leave`, `-request` and so on. If the message is posted to an invalid sub-address though, it is rejected. >>> lmtp.sendmail( @@ -122,8 +128,8 @@ Request subaddress ------------------ Depending on the subaddress, there is a message in the appropriate queue for -later processing. For example, all -request messages are put into the command -queue for processing. +later processing. For example, all `-request` messages are put into the +command queue for processing. >>> messages = get_queue_messages('command') >>> len(messages) @@ -133,6 +139,7 @@ queue for processing. To: mylist-request@example.com Subject: Help Message-ID: <dog> + X-Message-ID-Hash: 4SKREUSPI62BHDMFBSOZ3BMXFETSQHNA X-MailFrom: anne.person@example.com <BLANKLINE> Please help me. @@ -147,7 +154,7 @@ queue for processing. Bounce processor ---------------- -A message to the -bounces address goes to the bounce processor. +A message to the `-bounces` address goes to the bounce processor. >>> lmtp.sendmail( ... 'mail-daemon@example.com', @@ -283,7 +290,7 @@ Confirmation messages go to the command processor... Incoming processor ------------------ -Messages to the -owner address go to the incoming processor. +Messages to the `-owner` address also go to the incoming processor. >>> lmtp.sendmail( ... 'anne.person@example.com', @@ -307,7 +314,5 @@ Messages to the -owner address go to the incoming processor. version : ... -Clean up -======== - - >>> master.stop() +.. Clean up + >>> master.stop() diff --git a/src/mailman/runners/lmtp.py b/src/mailman/runners/lmtp.py index 6824491eb..bee111ad1 100644 --- a/src/mailman/runners/lmtp.py +++ b/src/mailman/runners/lmtp.py @@ -44,6 +44,7 @@ from mailman.core.runner import Runner from mailman.database.transaction import txn from mailman.email.message import Message from mailman.interfaces.listmanager import IListManager +from mailman.utilities.email import add_message_hash elog = logging.getLogger('mailman.error') qlog = logging.getLogger('mailman.runner') @@ -82,6 +83,7 @@ ERR_451 = '451 Requested action aborted: error in processing' ERR_501 = '501 Message has defects' ERR_502 = '502 Error: command HELO not implemented' ERR_550 = '550 Requested action not taken: mailbox unavailable' +ERR_550_MID = '550 No Message-ID header provided' # XXX Blech smtpd.__version__ = 'Python LMTP runner 1.0' @@ -159,15 +161,20 @@ class LMTPRunner(Runner, smtpd.SMTPServer): # Parse the message data. If there are any defects in the # message, reject it right away; it's probably spam. msg = email.message_from_string(data, Message) - msg.original_size = len(data) - if msg.defects: - return ERR_501 - msg['X-MailFrom'] = mailfrom - message_id = msg['message-id'] except Exception: elog.exception('LMTP message parsing') config.db.abort() return CRLF.join(ERR_451 for to in rcpttos) + # Do basic post-processing of the message, checking it for defects or + # other missing information. + message_id = msg.get('message-id') + if message_id is None: + return ERR_550_MID + if msg.defects: + return ERR_501 + msg.original_size = len(data) + add_message_hash(msg) + msg['X-MailFrom'] = mailfrom # RFC 2033 requires us to return a status code for every recipient. status = [] # Now for each address in the recipients, parse the address to first diff --git a/src/mailman/runners/tests/test_confirm.py b/src/mailman/runners/tests/test_confirm.py index 34a2af523..ea972c17f 100644 --- a/src/mailman/runners/tests/test_confirm.py +++ b/src/mailman/runners/tests/test_confirm.py @@ -155,7 +155,7 @@ Franziskanerstra=C3=9Fe self.assertEqual(address.email, 'anne@example.org') messages = get_queue_messages('virgin') self.assertEqual(len(messages), 1) - self.assertEqual(messages[0].msgdata['recipients'], + self.assertEqual(messages[0].msgdata['recipients'], set(['anne@example.org'])) def test_confirm_with_no_command_in_utf8_body(self): @@ -188,7 +188,7 @@ Franziskanerstra=C3=9Fe self.assertEqual(address.email, 'anne@example.org') messages = get_queue_messages('virgin') self.assertEqual(len(messages), 1) - self.assertEqual(messages[0].msgdata['recipients'], + self.assertEqual(messages[0].msgdata['recipients'], set(['anne@example.org'])) def test_double_confirmation(self): @@ -259,5 +259,5 @@ From: Anne Person <anne@example.org> messages = get_queue_messages('virgin', sort_on='subject') self.assertEqual(len(messages), 2) message = messages[1].msg - self.assertEqual(str(message['subject']), + self.assertEqual(str(message['subject']), 'Welcome to the "Test" mailing list') diff --git a/src/mailman/runners/tests/test_lmtp.py b/src/mailman/runners/tests/test_lmtp.py new file mode 100644 index 000000000..2c4defe59 --- /dev/null +++ b/src/mailman/runners/tests/test_lmtp.py @@ -0,0 +1,98 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Tests for the LMTP server.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestLMTP', + ] + + +import smtplib +import unittest + +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.testing.helpers import get_lmtp_client, get_queue_messages +from mailman.testing.layers import LMTPLayer + + + +class TestLMTP(unittest.TestCase): + """Test various aspects of the LMTP server.""" + + layer = LMTPLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + config.db.commit() + self._lmtp = get_lmtp_client(quiet=True) + self._lmtp.lhlo('remote.example.org') + + def tearDown(self): + self._lmtp.close() + + def test_message_id_required(self): + # The message is rejected if it does not have a Message-ID header. + try: + self._lmtp.sendmail('anne@example.com', ['test@example.com'], """\ +From: anne@example.com +To: test@example.com +Subject: This has no Message-ID header + +""") + except smtplib.SMTPDataError as error: + pass + else: + raise AssertionError('SMTPDataError expected') + # LMTP returns a 550: Requested action not taken: mailbox unavailable + # (e.g., mailbox not found, no access, or command rejected for policy + # reasons) + self.assertEqual(error.smtp_code, 550) + self.assertEqual(error.smtp_error, 'No Message-ID header provided') + + def test_message_id_hash_is_added(self): + self._lmtp.sendmail('anne@example.com', ['test@example.com'], """\ +From: anne@example.com +To: test@example.com +Message-ID: <ant> +Subject: This has a Message-ID but no X-Message-ID-Hash + +""") + messages = get_queue_messages('in') + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0].msg['x-message-id-hash'], + 'MS6QLWERIJLGCRF44J7USBFDELMNT2BW') + + def test_original_message_id_hash_is_overwritten(self): + self._lmtp.sendmail('anne@example.com', ['test@example.com'], """\ +From: anne@example.com +To: test@example.com +Message-ID: <ant> +X-Message-ID-Hash: IGNOREME +Subject: This has a Message-ID but no X-Message-ID-Hash + +""") + messages = get_queue_messages('in') + self.assertEqual(len(messages), 1) + all_headers = messages[0].msg.get_all('x-message-id-hash') + self.assertEqual(len(all_headers), 1) + self.assertEqual(messages[0].msg['x-message-id-hash'], + 'MS6QLWERIJLGCRF44J7USBFDELMNT2BW') diff --git a/src/mailman/testing/helpers.py b/src/mailman/testing/helpers.py index e61aca605..58c72d6d9 100644 --- a/src/mailman/testing/helpers.py +++ b/src/mailman/testing/helpers.py @@ -223,7 +223,7 @@ class LMTP(smtplib.SMTP): return code, msg -def get_lmtp_client(): +def get_lmtp_client(quiet=False): """Return a connected LMTP client.""" # It's possible the process has started but is not yet accepting # connections. Wait a little while. @@ -233,7 +233,8 @@ def get_lmtp_client(): try: response = lmtp.connect( config.mta.lmtp_host, int(config.mta.lmtp_port)) - print response + if not quiet: + print response return lmtp except socket.error as error: if error[0] == errno.ECONNREFUSED: diff --git a/src/mailman/testing/layers.py b/src/mailman/testing/layers.py index 5cee8c8d2..04ab8f91f 100644 --- a/src/mailman/testing/layers.py +++ b/src/mailman/testing/layers.py @@ -22,6 +22,7 @@ from __future__ import absolute_import, unicode_literals __metaclass__ = type __all__ = [ 'ConfigLayer', + 'LMTPLayer', 'MockAndMonkeyLayer', 'RESTLayer', 'SMTPLayer', @@ -48,7 +49,8 @@ from mailman.core import initialize from mailman.core.initialize import INHIBIT_CONFIG_FILE from mailman.core.logging import get_handler from mailman.interfaces.domain import IDomainManager -from mailman.testing.helpers import TestableMaster, reset_the_world +from mailman.testing.helpers import ( + TestableMaster, get_lmtp_client, reset_the_world) from mailman.testing.mta import ConnectionCountingController from mailman.utilities.string import expand @@ -250,6 +252,8 @@ class SMTPLayer(ConfigLayer): @classmethod def testSetUp(cls): + # Make sure we don't call our superclass's testSetUp(), otherwise the + # example.com domain will get added twice. pass @classmethod @@ -259,6 +263,35 @@ class SMTPLayer(ConfigLayer): +class LMTPLayer(ConfigLayer): + """Layer for starting, stopping, and accessing a test LMTP server.""" + + lmtpd = None + + @staticmethod + def _wait_for_lmtp_server(): + get_lmtp_client(quiet=True) + + @classmethod + def setUp(cls): + assert cls.lmtpd is None, 'Layer already set up' + cls.lmtpd = TestableMaster(cls._wait_for_lmtp_server) + cls.lmtpd.start('lmtp') + + @classmethod + def tearDown(cls): + assert cls.lmtpd is not None, 'Layer not set up' + cls.lmtpd.stop() + cls.lmtpd = None + + @classmethod + def testSetUp(cls): + # Make sure we don't call our superclass's testSetUp(), otherwise the + # example.com domain will get added twice. + pass + + + class RESTLayer(SMTPLayer): """Layer for starting, stopping, and accessing the test REST layer.""" diff --git a/src/mailman/utilities/email.py b/src/mailman/utilities/email.py index 4b1023ce1..38c9980a2 100644 --- a/src/mailman/utilities/email.py +++ b/src/mailman/utilities/email.py @@ -21,10 +21,17 @@ from __future__ import absolute_import, unicode_literals __metaclass__ = type __all__ = [ + 'add_message_hash', 'split_email', ] +from base64 import b32encode +from hashlib import sha1 + + + + def split_email(address): """Split an email address into a user name and domain. @@ -39,3 +46,29 @@ def split_email(address): # There was no at-sign in the email address. return local_part, None return local_part, domain.split('.') + + +def add_message_hash(msg): + """Add a X-Message-ID-Hash header derived from Message-ID. + + This function works by side-effect; the original message is mutated. Any + existing X-Message-ID-Headers are deleted if a Message-ID header is + found. If no Message-ID header is found, the original message is not + modified. + + :param msg: An email message + :type msg: `email.message.Message` or derived + """ + message_id = msg.get('message-id') + if message_id is None: + return + # The angle brackets are not part of the Message-ID. See RFC 2822 + # and http://wiki.list.org/display/DEV/Stable+URLs + if message_id.startswith('<') and message_id.endswith('>'): + message_id = message_id[1:-1] + else: + message_id = message_id.strip() + digest = sha1(message_id).digest() + message_id_hash = b32encode(digest) + del msg['x-message-id-hash'] + msg['X-Message-ID-Hash'] = message_id_hash diff --git a/src/mailman/utilities/tests/test_email.py b/src/mailman/utilities/tests/test_email.py index 833d631b5..478322aaf 100644 --- a/src/mailman/utilities/tests/test_email.py +++ b/src/mailman/utilities/tests/test_email.py @@ -15,22 +15,27 @@ # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see <http://www.gnu.org/licenses/>. -"""Testing app.bounces functions.""" +"""Testing functions in the email utilities.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ + 'TestEmail', ] import unittest -from mailman.utilities.email import split_email +from mailman.testing.helpers import ( + specialized_message_from_string as mfs) +from mailman.utilities.email import add_message_hash, split_email class TestEmail(unittest.TestCase): + """Testing functions in the email utilities.""" + def test_normal_split(self): self.assertEqual(split_email('anne@example.com'), ('anne', ['example', 'com'])) @@ -39,3 +44,67 @@ class TestEmail(unittest.TestCase): def test_no_at_split(self): self.assertEqual(split_email('anne'), ('anne', None)) + + def test_adding_the_message_hash(self): + # When the message has a Message-ID header, this will add the + # X-Mailman-Hash-ID header. + msg = mfs("""\ +Message-ID: <aardvark> + +""") + add_message_hash(msg) + self.assertEqual(msg['x-message-id-hash'], + '75E2XSUXAFQGWANWEROVQ7JGYMNWHJBT') + + def test_remove_hash_headers_first(self): + # Any existing X-Mailman-Hash-ID header is removed first. + msg = mfs("""\ +Message-ID: <aardvark> +X-Message-ID-Hash: abc + +""") + add_message_hash(msg) + headers = msg.get_all('x-message-id-hash') + self.assertEqual(len(headers), 1) + self.assertEqual(headers[0], '75E2XSUXAFQGWANWEROVQ7JGYMNWHJBT') + + def test_hash_header_left_alone_if_no_message_id(self): + # If the original message has no Message-ID header, then any existing + # X-Message-ID-Hash headers are left intact. + msg = mfs("""\ +X-Message-ID-Hash: abc + +""") + add_message_hash(msg) + headers = msg.get_all('x-message-id-hash') + self.assertEqual(len(headers), 1) + self.assertEqual(headers[0], 'abc') + + def test_angle_brackets_dont_contribute_to_hash(self): + # According to RFC 5322, the [matching] angle brackets do not + # contribute to the hash. + msg = mfs("""\ +Message-ID: aardvark + +""") + add_message_hash(msg) + self.assertEqual(msg['x-message-id-hash'], + '75E2XSUXAFQGWANWEROVQ7JGYMNWHJBT') + + def test_mismatched_angle_brackets_do_contribute_to_hash(self): + # According to RFC 5322, the [matching] angle brackets do not + # contribute to the hash. + msg = mfs("""\ +Message-ID: <aardvark + +""") + add_message_hash(msg) + self.assertEqual(msg['x-message-id-hash'], + 'AOJ545GHRYD2Y3RUFG2EWMPHUABTG4SM') + msg = mfs("""\ +Message-ID: aardvark> + +""") + add_message_hash(msg) + self.assertEqual(msg['x-message-id-hash'], + '5KH3RA7ZM4VM6XOZXA7AST2XN2X4S3WY') diff --git a/src/mailman/version.py b/src/mailman/version.py index 00c5b6c7a..8f13f8f51 100644 --- a/src/mailman/version.py +++ b/src/mailman/version.py @@ -17,15 +17,15 @@ """Mailman version strings.""" -# Mailman version +# Mailman version. VERSION = '3.0.0a8+' CODENAME = "The Twilight Zone" -# And as a hex number in the manner of PY_VERSION_HEX +# And as a hex number in the manner of PY_VERSION_HEX. ALPHA = 0xa BETA = 0xb GAMMA = 0xc -# release candidates +# Release candidates. RC = GAMMA FINAL = 0xf @@ -33,16 +33,16 @@ MAJOR_REV = 3 MINOR_REV = 0 MICRO_REV = 0 REL_LEVEL = BETA -# at most 15 beta releases! +# At most 15 beta releases! REL_SERIAL = 1 HEX_VERSION = ((MAJOR_REV << 24) | (MINOR_REV << 16) | (MICRO_REV << 8) | (REL_LEVEL << 4) | (REL_SERIAL << 0)) -# qfile/*.db schema version number +# queue/*.pck schema version number. QFILE_SCHEMA_VERSION = 3 -# Printable version string used by command line scripts +# Printable version string used by command line scripts. MAILMAN_VERSION = 'GNU Mailman ' + VERSION MAILMAN_VERSION_FULL = MAILMAN_VERSION + ' (' + CODENAME + ')' |
