6 files changed, 193 insertions, 43 deletions

diff --git a/src/mailman/core/tests/test_runner.py b/src/mailman/core/tests/test_runner.py
index 2875b3b10..1fb8f0b7b 100644
--- a/src/mailman/core/tests/test_runner.py
+++ b/src/mailman/core/tests/test_runner.py
@@ -31,9 +31,11 @@ from mailman.app.lifecycle import create_list
 from mailman.config import config
 from mailman.core.runner import Runner
 from mailman.interfaces.runner import RunnerCrashEvent
+from mailman.runners.virgin import VirginRunner
 from mailman.testing.helpers import (
-    configuration, event_subscribers, get_queue_messages,
-    make_testable_runner, specialized_message_from_string as mfs)
+    LogFileMark, configuration, event_subscribers, get_queue_messages,
+    make_digest_messages, make_testable_runner,
+    specialized_message_from_string as mfs)
 from mailman.testing.layers import ConfigLayer
 
 
@@ -87,3 +89,37 @@ Message-ID: <ant>
         shunted = get_queue_messages('shunt')
         self.assertEqual(len(shunted), 1)
         self.assertEqual(shunted[0].msg['message-id'], '<ant>')
+
+    def test_digest_messages(self):
+        # In LP: #1130697, the digest runner creates MIME digests using the
+        # stdlib MIMEMutlipart class, however this class does not have the
+        # extended attributes we require (e.g. .sender).  The fix is to use a
+        # subclass of MIMEMultipart and our own Message subclass; this adds
+        # back the required attributes.  (LP: #1130696)
+        #
+        # Start by creating the raw ingredients for the digests.  This also
+        # runs the digest runner, thus producing the digest messages into the
+        # virgin queue.
+        make_digest_messages(self._mlist)
+        # Run the virgin queue processor, which runs the cook-headers and
+        # to-outgoing handlers.  This should produce no error.
+        error_log = LogFileMark('mailman.error')
+        runner = make_testable_runner(VirginRunner, 'virgin')
+        runner.run()
+        error_text = error_log.read()
+        self.assertEqual(len(error_text), 0, error_text)
+        self.assertEqual(len(get_queue_messages('shunt')), 0)
+        messages = get_queue_messages('out')
+        self.assertEqual(len(messages), 2)
+        # Which one is the MIME digest?
+        mime_digest = None
+        for bag in messages:
+            if bag.msg.get_content_type() == 'multipart/mixed':
+                assert mime_digest is None, 'Found two MIME digests'
+                mime_digest = bag.msg
+        # The cook-headers handler ran.
+        self.assertIn('x-mailman-version', mime_digest)
+        self.assertEqual(mime_digest['precedence'], 'list')
+        # The list's -request address is the original sender.
+        self.assertEqual(bag.msgdata['original_sender'],
+                         'test-request@example.com')
diff --git a/src/mailman/email/message.py b/src/mailman/email/message.py
index 24c6ead9e..e653133ba 100644
--- a/src/mailman/email/message.py
+++ b/src/mailman/email/message.py
@@ -28,6 +28,7 @@ from __future__ import absolute_import, print_function, unicode_literals
 __metaclass__ = type
 __all__ = [
     'Message',
+    'MultipartDigestMessage',
     'OwnerNotification',
     'UserNotification',
     ]
@@ -38,6 +39,7 @@ import email.message
 import email.utils
 
 from email.header import Header
+from email.mime.multipart import MIMEMultipart
 
 from mailman.config import config
 
@@ -133,6 +135,11 @@ class Message(email.message.Message):
 
 
 
+class MultipartDigestMessage(MIMEMultipart, Message):
+    """Mix-in class for MIME digest messages."""
+
+
+
 class UserNotification(Message):
     """Class for internally crafted messages."""
 
@@ -159,7 +166,7 @@ class UserNotification(Message):
 
         :param mlist: The mailing list to send the message to.
         :type mlist: `IMailingList`
-        :param add_precedence: Flag indicating whether a `Precedence: bulk` 
+        :param add_precedence: Flag indicating whether a `Precedence: bulk`
             header should be added to the message or not.
         :type add_precedence: bool
 
diff --git a/src/mailman/handlers/tests/test_cook_headers.py b/src/mailman/handlers/tests/test_cook_headers.py
new file mode 100644
index 000000000..d83a44f20
--- /dev/null
+++ b/src/mailman/handlers/tests/test_cook_headers.py
@@ -0,0 +1,55 @@
+# Copyright (C) 2014 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Test the cook_headers handler."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    'TestCookHeaders',
+    ]
+
+
+import unittest
+
+from mailman.app.lifecycle import create_list
+from mailman.handlers import cook_headers
+from mailman.testing.helpers import get_queue_messages, make_digest_messages
+from mailman.testing.layers import ConfigLayer
+
+
+
+class TestCookHeaders(unittest.TestCase):
+    """Test the cook_headers handler."""
+
+    layer = ConfigLayer
+
+    def setUp(self):
+        self._mlist = create_list('test@example.com')
+
+    def test_process_digest(self):
+        # MIME digests messages are multiparts.
+        make_digest_messages(self._mlist)
+        messages = [bag.msg for bag in get_queue_messages('virgin')]
+        self.assertEqual(len(messages), 2)
+        for msg in messages:
+            try:
+                cook_headers.process(self._mlist, msg, {})
+            except AttributeError as e:
+                # LP: #1130696 would raise an AttributeError on .sender
+                self.fail(e)
diff --git a/src/mailman/runners/digest.py b/src/mailman/runners/digest.py
index 98fe82f39..e62c14abf 100644
--- a/src/mailman/runners/digest.py
+++ b/src/mailman/runners/digest.py
@@ -32,9 +32,7 @@ import logging
 from StringIO import StringIO
 from copy import deepcopy
 from email.header import Header
-from email.message import Message
 from email.mime.message import MIMEMessage
-from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 from email.utils import formatdate, getaddresses, make_msgid
 from urllib2 import URLError
@@ -42,6 +40,7 @@ from urllib2 import URLError
 from mailman.config import config
 from mailman.core.i18n import _
 from mailman.core.runner import Runner
+from mailman.email.message import Message, MultipartDigestMessage
 from mailman.handlers.decorate import decorate
 from mailman.interfaces.member import DeliveryMode, DeliveryStatus
 from mailman.utilities.i18n import make
@@ -172,7 +171,7 @@ class MIMEDigester(Digester):
         self._keepers = set(config.digests.mime_digest_keep_headers.split())
 
     def _make_message(self):
-        return MIMEMultipart('mixed')
+        return MultipartDigestMessage('mixed')
 
     def add_toc(self, count):
         """Add the table of contents."""
diff --git a/src/mailman/runners/tests/test_digest.py b/src/mailman/runners/tests/test_digest.py
index 80cf253bc..fb1bb7071 100644
--- a/src/mailman/runners/tests/test_digest.py
+++ b/src/mailman/runners/tests/test_digest.py
@@ -25,18 +25,20 @@ __all__ = [
     ]
 
 
-import os
 import unittest
 
+from StringIO import StringIO
+from email.iterators import _structure as structure
 from email.mime.text import MIMEText
 from mailman.app.lifecycle import create_list
 from mailman.config import config
 from mailman.email.message import Message
 from mailman.runners.digest import DigestRunner
 from mailman.testing.helpers import (
-    LogFileMark, digest_mbox, get_queue_messages, make_testable_runner,
-    specialized_message_from_string as mfs)
+    LogFileMark, digest_mbox, get_queue_messages, make_digest_messages,
+    make_testable_runner, message_from_string)
 from mailman.testing.layers import ConfigLayer
+from string import Template
 
 
 
@@ -44,6 +46,7 @@ class TestDigest(unittest.TestCase):
     """Test the digest runner."""
 
     layer = ConfigLayer
+    maxDiff = None
 
     def setUp(self):
         self._mlist = create_list('test@example.com')
@@ -54,23 +57,9 @@ class TestDigest(unittest.TestCase):
         self._runner = make_testable_runner(DigestRunner, 'digest')
         self._process = config.handlers['to-digest'].process
 
-    def test_simple_message(self):
-        msg = mfs("""\
-From: anne@example.org
-To: test@example.com
-
-message triggering a digest
-""")
-        mbox_path = os.path.join(self._mlist.data_path, 'digest.mmdf')
-        self._process(self._mlist, msg, {})
-        self._digestq.enqueue(
-            msg,
-            listname=self._mlist.fqdn_listname,
-            digest_path=mbox_path,
-            volume=1, digest_number=1)
-        self._runner.run()
-        # There are two messages in the virgin queue: the digest as plain-text
-        # and as multipart.
+    def _check_virgin_queue(self):
+        # There should be two messages in the virgin queue: the digest as
+        # plain-text and as multipart.
         messages = get_queue_messages('virgin')
         self.assertEqual(len(messages), 2)
         self.assertEqual(
@@ -80,6 +69,10 @@ message triggering a digest
             self.assertEqual(item.msg['subject'],
                              'Test Digest, Vol 1, Issue 1')
 
+    def test_simple_message(self):
+        make_digest_messages(self._mlist)
+        self._check_virgin_queue()
+
     def test_non_ascii_message(self):
         msg = Message()
         msg['From'] = 'anne@example.org'
@@ -88,25 +81,62 @@ message triggering a digest
         msg.attach(MIMEText('message with non-ascii chars: \xc3\xa9',
                             'plain', 'utf-8'))
         mbox = digest_mbox(self._mlist)
-        mbox_path = os.path.join(self._mlist.data_path, 'digest.mmdf')
         mbox.add(msg.as_string())
-        self._digestq.enqueue(
-            msg,
-            listname=self._mlist.fqdn_listname,
-            digest_path=mbox_path,
-            volume=1, digest_number=1)
         # Use any error logs as the error message if the test fails.
         error_log = LogFileMark('mailman.error')
-        self._runner.run()
+        make_digest_messages(self._mlist, msg)
         # The runner will send the file to the shunt queue on exception.
         self.assertEqual(len(self._shuntq.files), 0, error_log.read())
-        # There are two messages in the virgin queue: the digest as plain-text
-        # and as multipart.
-        messages = get_queue_messages('virgin')
-        self.assertEqual(len(messages), 2)
-        self.assertEqual(
-            sorted(item.msg.get_content_type() for item in messages),
-            ['multipart/mixed', 'text/plain'])
-        for item in messages:
-            self.assertEqual(item.msg['subject'],
-                             'Test Digest, Vol 1, Issue 1')
+        self._check_virgin_queue()
+
+    def test_mime_digest_format(self):
+        # Make sure that the format of the MIME digest is as expected.
+        self._mlist.digest_size_threshold = 0.6
+        self._mlist.volume = 1
+        self._mlist.next_digest_number = 1
+        self._mlist.send_welcome_message = False
+        # Fill the digest.
+        process = config.handlers['to-digest'].process
+        size = 0
+        for i in range(1, 5):
+            text = Template("""\
+From: aperson@example.com
+To: xtest@example.com
+Subject: Test message $i
+List-Post: <test@example.com>
+
+Here is message $i
+""").substitute(i=i)
+            msg = message_from_string(text)
+            process(self._mlist, msg, {})
+            size += len(text)
+            if size >= self._mlist.digest_size_threshold * 1024:
+                break
+        # Run the digest runner to create the MIME and RFC 1153 digests.
+        runner = make_testable_runner(DigestRunner)
+        runner.run()
+        items = get_queue_messages('virgin')
+        self.assertEqual(len(items), 2)
+        # Find the MIME one.
+        mime_digest = None
+        for item in items:
+            if item.msg.is_multipart():
+                assert mime_digest is None, 'We got two MIME digests'
+                mime_digest = item.msg
+        fp = StringIO()
+        # Verify the structure is what we expect.
+        structure(mime_digest, fp)
+        self.assertMultiLineEqual(fp.getvalue(), """\
+multipart/mixed
+    text/plain
+    text/plain
+    message/rfc822
+        text/plain
+    message/rfc822
+        text/plain
+    message/rfc822
+        text/plain
+    message/rfc822
+        text/plain
+    text/plain
+""")
diff --git a/src/mailman/testing/helpers.py b/src/mailman/testing/helpers.py
index b0fe14a0d..fd3af88ee 100644
--- a/src/mailman/testing/helpers.py
+++ b/src/mailman/testing/helpers.py
@@ -31,6 +31,7 @@ __all__ = [
     'get_lmtp_client',
     'get_nntp_server',
     'get_queue_messages',
+    'make_digest_messages',
     'make_testable_runner',
     'reset_the_world',
     'specialized_message_from_string',
@@ -72,6 +73,7 @@ from mailman.interfaces.member import MemberRole
 from mailman.interfaces.messages import IMessageStore
 from mailman.interfaces.styles import IStyleManager
 from mailman.interfaces.usermanager import IUserManager
+from mailman.runners.digest import DigestRunner
 from mailman.utilities.mailbox import Mailbox
 
 
@@ -529,3 +531,24 @@ class LogFileMark:
         with open(self._filename) as fp:
             fp.seek(self._filepos)
             return fp.read()
+
+
+
+def make_digest_messages(mlist, msg=None):
+    if msg is None:
+        msg = specialized_message_from_string("""\
+From: anne@example.org
+To: {listname}
+Message-ID: <testing>
+
+message triggering a digest
+""".format(listname=mlist.fqdn_listname))
+    mbox_path = os.path.join(mlist.data_path, 'digest.mmdf')
+    config.handlers['to-digest'].process(mlist, msg, {})
+    config.switchboards['digest'].enqueue(
+        msg,
+        listname=mlist.fqdn_listname,
+        digest_path=mbox_path,
+        volume=1, digest_number=1)
+    runner = make_testable_runner(DigestRunner, 'digest')
+    runner.run()