Merge with upstream

5 files changed, 130 insertions, 19 deletions

diff --git a/src/mailman/runners/docs/incoming.rst b/src/mailman/runners/docs/incoming.rst
index e5f544a85..4a9db778e 100644
--- a/src/mailman/runners/docs/incoming.rst
+++ b/src/mailman/runners/docs/incoming.rst
@@ -121,6 +121,7 @@ Now the message is in the pipeline queue.
     To: test@example.com
     Subject: My first post
     Message-ID: <first>
+    X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB
     Date: ...
     X-Mailman-Rule-Misses: approved; emergency; loop; member-moderation;
         administrivia; implicit-dest; max-recipients; max-size;
diff --git a/src/mailman/runners/docs/lmtp.rst b/src/mailman/runners/docs/lmtp.rst
index 1ab42410b..2b6c4b42b 100644
--- a/src/mailman/runners/docs/lmtp.rst
+++ b/src/mailman/runners/docs/lmtp.rst
@@ -7,7 +7,9 @@ support LMTP local delivery, so this is a very portable way to connect Mailman
 with your mail server.
 
 Our LMTP server is fairly simple though; all it does is make sure that the
-message is destined for a valid endpoint, e.g. ``mylist-join@example.com``.
+message is destined for a valid endpoint, e.g. ``mylist-join@example.com``,
+that the message bytes can be parsed into a message object, and that the
+message has a `Message-ID` header.
 
 Let's start a testable LMTP runner.
 
@@ -62,6 +64,9 @@ Once the mailing list is created, the posting address is valid.
     ... """)
     {}
 
+Since the message itself is valid, it gets parsed and lands in the incoming
+queue.
+
     >>> from mailman.testing.helpers import get_queue_messages
     >>> messages = get_queue_messages('in')
     >>> len(messages)
@@ -71,6 +76,7 @@ Once the mailing list is created, the posting address is valid.
     To: mylist@example.com
     Subject: An interesting message
     Message-ID: <badger>
+    X-Message-ID-Hash: JYMZWSQ4IC2JPKK7ZUONRFRVC4ZYJGKJ
     X-MailFrom: anne.person@example.com
     <BLANKLINE>
     This is an interesting message.
@@ -85,8 +91,8 @@ Once the mailing list is created, the posting address is valid.
 Sub-addresses
 =============
 
-The LMTP server understands each of the list's sub-addreses, such as -join,
--leave, -request and so on.  If the message is posted to an invalid
+The LMTP server understands each of the list's sub-addreses, such as `-join`,
+`-leave`, `-request` and so on.  If the message is posted to an invalid
 sub-address though, it is rejected.
 
     >>> lmtp.sendmail(
@@ -122,8 +128,8 @@ Request subaddress
 ------------------
 
 Depending on the subaddress, there is a message in the appropriate queue for
-later processing.  For example, all -request messages are put into the command
-queue for processing.
+later processing.  For example, all `-request` messages are put into the
+command queue for processing.
 
     >>> messages = get_queue_messages('command')
     >>> len(messages)
@@ -133,6 +139,7 @@ queue for processing.
     To: mylist-request@example.com
     Subject: Help
     Message-ID: <dog>
+    X-Message-ID-Hash: 4SKREUSPI62BHDMFBSOZ3BMXFETSQHNA
     X-MailFrom: anne.person@example.com
     <BLANKLINE>
     Please help me.
@@ -147,7 +154,7 @@ queue for processing.
 Bounce processor
 ----------------
 
-A message to the -bounces address goes to the bounce processor.
+A message to the `-bounces` address goes to the bounce processor.
 
     >>> lmtp.sendmail(
     ...     'mail-daemon@example.com',
@@ -283,7 +290,7 @@ Confirmation messages go to the command processor...
 Incoming processor
 ------------------
 
-Messages to the -owner address go to the incoming processor.
+Messages to the `-owner` address also go to the incoming processor.
 
     >>> lmtp.sendmail(
     ...     'anne.person@example.com',
@@ -307,7 +314,5 @@ Messages to the -owner address go to the incoming processor.
     version      : ...
 
 
-Clean up
-========
-
-    >>> master.stop()
+.. Clean up
+   >>> master.stop()
diff --git a/src/mailman/runners/lmtp.py b/src/mailman/runners/lmtp.py
index 6824491eb..bee111ad1 100644
--- a/src/mailman/runners/lmtp.py
+++ b/src/mailman/runners/lmtp.py
@@ -44,6 +44,7 @@ from mailman.core.runner import Runner
 from mailman.database.transaction import txn
 from mailman.email.message import Message
 from mailman.interfaces.listmanager import IListManager
+from mailman.utilities.email import add_message_hash
 
 elog = logging.getLogger('mailman.error')
 qlog = logging.getLogger('mailman.runner')
@@ -82,6 +83,7 @@ ERR_451 = '451 Requested action aborted: error in processing'
 ERR_501 = '501 Message has defects'
 ERR_502 = '502 Error: command HELO not implemented'
 ERR_550 = '550 Requested action not taken: mailbox unavailable'
+ERR_550_MID = '550 No Message-ID header provided'
 
 # XXX Blech
 smtpd.__version__ = 'Python LMTP runner 1.0'
@@ -159,15 +161,20 @@ class LMTPRunner(Runner, smtpd.SMTPServer):
             # Parse the message data.  If there are any defects in the
             # message, reject it right away; it's probably spam.
             msg = email.message_from_string(data, Message)
-            msg.original_size = len(data)
-            if msg.defects:
-                return ERR_501
-            msg['X-MailFrom'] = mailfrom
-            message_id = msg['message-id']
         except Exception:
             elog.exception('LMTP message parsing')
             config.db.abort()
             return CRLF.join(ERR_451 for to in rcpttos)
+        # Do basic post-processing of the message, checking it for defects or
+        # other missing information.
+        message_id = msg.get('message-id')
+        if message_id is None:
+            return ERR_550_MID
+        if msg.defects:
+            return ERR_501
+        msg.original_size = len(data)
+        add_message_hash(msg)
+        msg['X-MailFrom'] = mailfrom
         # RFC 2033 requires us to return a status code for every recipient.
         status = []
         # Now for each address in the recipients, parse the address to first
diff --git a/src/mailman/runners/tests/test_confirm.py b/src/mailman/runners/tests/test_confirm.py
index 34a2af523..ea972c17f 100644
--- a/src/mailman/runners/tests/test_confirm.py
+++ b/src/mailman/runners/tests/test_confirm.py
@@ -155,7 +155,7 @@ Franziskanerstra=C3=9Fe
         self.assertEqual(address.email, 'anne@example.org')
         messages = get_queue_messages('virgin')
         self.assertEqual(len(messages), 1)
-        self.assertEqual(messages[0].msgdata['recipients'], 
+        self.assertEqual(messages[0].msgdata['recipients'],
                          set(['anne@example.org']))
 
     def test_confirm_with_no_command_in_utf8_body(self):
@@ -188,7 +188,7 @@ Franziskanerstra=C3=9Fe
         self.assertEqual(address.email, 'anne@example.org')
         messages = get_queue_messages('virgin')
         self.assertEqual(len(messages), 1)
-        self.assertEqual(messages[0].msgdata['recipients'], 
+        self.assertEqual(messages[0].msgdata['recipients'],
                          set(['anne@example.org']))
 
     def test_double_confirmation(self):
@@ -259,5 +259,5 @@ From: Anne Person <anne@example.org>
         messages = get_queue_messages('virgin', sort_on='subject')
         self.assertEqual(len(messages), 2)
         message = messages[1].msg
-        self.assertEqual(str(message['subject']), 
+        self.assertEqual(str(message['subject']),
                          'Welcome to the "Test" mailing list')
diff --git a/src/mailman/runners/tests/test_lmtp.py b/src/mailman/runners/tests/test_lmtp.py
new file mode 100644
index 000000000..2c4defe59
--- /dev/null
+++ b/src/mailman/runners/tests/test_lmtp.py
@@ -0,0 +1,98 @@
+# Copyright (C) 2012 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Tests for the LMTP server."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    'TestLMTP',
+    ]
+
+
+import smtplib
+import unittest
+
+from mailman.app.lifecycle import create_list
+from mailman.config import config
+from mailman.testing.helpers import get_lmtp_client, get_queue_messages
+from mailman.testing.layers import LMTPLayer
+
+
+
+class TestLMTP(unittest.TestCase):
+    """Test various aspects of the LMTP server."""
+
+    layer = LMTPLayer
+
+    def setUp(self):
+        self._mlist = create_list('test@example.com')
+        config.db.commit()
+        self._lmtp = get_lmtp_client(quiet=True)
+        self._lmtp.lhlo('remote.example.org')
+
+    def tearDown(self):
+        self._lmtp.close()
+
+    def test_message_id_required(self):
+        # The message is rejected if it does not have a Message-ID header.
+        try:
+            self._lmtp.sendmail('anne@example.com', ['test@example.com'], """\
+From: anne@example.com
+To: test@example.com
+Subject: This has no Message-ID header
+
+""")
+        except smtplib.SMTPDataError as error:
+            pass
+        else:
+            raise AssertionError('SMTPDataError expected')
+        # LMTP returns a 550: Requested action not taken: mailbox unavailable
+        # (e.g., mailbox not found, no access, or command rejected for policy
+        # reasons)
+        self.assertEqual(error.smtp_code, 550)
+        self.assertEqual(error.smtp_error, 'No Message-ID header provided')
+
+    def test_message_id_hash_is_added(self):
+        self._lmtp.sendmail('anne@example.com', ['test@example.com'], """\
+From: anne@example.com
+To: test@example.com
+Message-ID: <ant>
+Subject: This has a Message-ID but no X-Message-ID-Hash
+
+""")
+        messages = get_queue_messages('in')
+        self.assertEqual(len(messages), 1)
+        self.assertEqual(messages[0].msg['x-message-id-hash'],
+                         'MS6QLWERIJLGCRF44J7USBFDELMNT2BW')
+
+    def test_original_message_id_hash_is_overwritten(self):
+        self._lmtp.sendmail('anne@example.com', ['test@example.com'], """\
+From: anne@example.com
+To: test@example.com
+Message-ID: <ant>
+X-Message-ID-Hash: IGNOREME
+Subject: This has a Message-ID but no X-Message-ID-Hash
+
+""")
+        messages = get_queue_messages('in')
+        self.assertEqual(len(messages), 1)
+        all_headers = messages[0].msg.get_all('x-message-id-hash')
+        self.assertEqual(len(all_headers), 1)
+        self.assertEqual(messages[0].msg['x-message-id-hash'],
+                         'MS6QLWERIJLGCRF44J7USBFDELMNT2BW')