diff options
| -rw-r--r-- | src/mailman/core/tests/test_runner.py | 40 | ||||
| -rw-r--r-- | src/mailman/email/message.py | 9 | ||||
| -rw-r--r-- | src/mailman/handlers/tests/test_cook_headers.py | 55 | ||||
| -rw-r--r-- | src/mailman/runners/digest.py | 5 | ||||
| -rw-r--r-- | src/mailman/runners/tests/test_digest.py | 104 | ||||
| -rw-r--r-- | src/mailman/testing/helpers.py | 23 |
6 files changed, 193 insertions, 43 deletions
diff --git a/src/mailman/core/tests/test_runner.py b/src/mailman/core/tests/test_runner.py index 2875b3b10..1fb8f0b7b 100644 --- a/src/mailman/core/tests/test_runner.py +++ b/src/mailman/core/tests/test_runner.py @@ -31,9 +31,11 @@ from mailman.app.lifecycle import create_list from mailman.config import config from mailman.core.runner import Runner from mailman.interfaces.runner import RunnerCrashEvent +from mailman.runners.virgin import VirginRunner from mailman.testing.helpers import ( - configuration, event_subscribers, get_queue_messages, - make_testable_runner, specialized_message_from_string as mfs) + LogFileMark, configuration, event_subscribers, get_queue_messages, + make_digest_messages, make_testable_runner, + specialized_message_from_string as mfs) from mailman.testing.layers import ConfigLayer @@ -87,3 +89,37 @@ Message-ID: <ant> shunted = get_queue_messages('shunt') self.assertEqual(len(shunted), 1) self.assertEqual(shunted[0].msg['message-id'], '<ant>') + + def test_digest_messages(self): + # In LP: #1130697, the digest runner creates MIME digests using the + # stdlib MIMEMutlipart class, however this class does not have the + # extended attributes we require (e.g. .sender). The fix is to use a + # subclass of MIMEMultipart and our own Message subclass; this adds + # back the required attributes. (LP: #1130696) + # + # Start by creating the raw ingredients for the digests. This also + # runs the digest runner, thus producing the digest messages into the + # virgin queue. + make_digest_messages(self._mlist) + # Run the virgin queue processor, which runs the cook-headers and + # to-outgoing handlers. This should produce no error. + error_log = LogFileMark('mailman.error') + runner = make_testable_runner(VirginRunner, 'virgin') + runner.run() + error_text = error_log.read() + self.assertEqual(len(error_text), 0, error_text) + self.assertEqual(len(get_queue_messages('shunt')), 0) + messages = get_queue_messages('out') + self.assertEqual(len(messages), 2) + # Which one is the MIME digest? + mime_digest = None + for bag in messages: + if bag.msg.get_content_type() == 'multipart/mixed': + assert mime_digest is None, 'Found two MIME digests' + mime_digest = bag.msg + # The cook-headers handler ran. + self.assertIn('x-mailman-version', mime_digest) + self.assertEqual(mime_digest['precedence'], 'list') + # The list's -request address is the original sender. + self.assertEqual(bag.msgdata['original_sender'], + 'test-request@example.com') diff --git a/src/mailman/email/message.py b/src/mailman/email/message.py index 24c6ead9e..e653133ba 100644 --- a/src/mailman/email/message.py +++ b/src/mailman/email/message.py @@ -28,6 +28,7 @@ from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ 'Message', + 'MultipartDigestMessage', 'OwnerNotification', 'UserNotification', ] @@ -38,6 +39,7 @@ import email.message import email.utils from email.header import Header +from email.mime.multipart import MIMEMultipart from mailman.config import config @@ -133,6 +135,11 @@ class Message(email.message.Message): +class MultipartDigestMessage(MIMEMultipart, Message): + """Mix-in class for MIME digest messages.""" + + + class UserNotification(Message): """Class for internally crafted messages.""" @@ -159,7 +166,7 @@ class UserNotification(Message): :param mlist: The mailing list to send the message to. :type mlist: `IMailingList` - :param add_precedence: Flag indicating whether a `Precedence: bulk` + :param add_precedence: Flag indicating whether a `Precedence: bulk` header should be added to the message or not. :type add_precedence: bool diff --git a/src/mailman/handlers/tests/test_cook_headers.py b/src/mailman/handlers/tests/test_cook_headers.py new file mode 100644 index 000000000..d83a44f20 --- /dev/null +++ b/src/mailman/handlers/tests/test_cook_headers.py @@ -0,0 +1,55 @@ +# Copyright (C) 2014 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test the cook_headers handler.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestCookHeaders', + ] + + +import unittest + +from mailman.app.lifecycle import create_list +from mailman.handlers import cook_headers +from mailman.testing.helpers import get_queue_messages, make_digest_messages +from mailman.testing.layers import ConfigLayer + + + +class TestCookHeaders(unittest.TestCase): + """Test the cook_headers handler.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + + def test_process_digest(self): + # MIME digests messages are multiparts. + make_digest_messages(self._mlist) + messages = [bag.msg for bag in get_queue_messages('virgin')] + self.assertEqual(len(messages), 2) + for msg in messages: + try: + cook_headers.process(self._mlist, msg, {}) + except AttributeError as e: + # LP: #1130696 would raise an AttributeError on .sender + self.fail(e) diff --git a/src/mailman/runners/digest.py b/src/mailman/runners/digest.py index 98fe82f39..e62c14abf 100644 --- a/src/mailman/runners/digest.py +++ b/src/mailman/runners/digest.py @@ -32,9 +32,7 @@ import logging from StringIO import StringIO from copy import deepcopy from email.header import Header -from email.message import Message from email.mime.message import MIMEMessage -from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.utils import formatdate, getaddresses, make_msgid from urllib2 import URLError @@ -42,6 +40,7 @@ from urllib2 import URLError from mailman.config import config from mailman.core.i18n import _ from mailman.core.runner import Runner +from mailman.email.message import Message, MultipartDigestMessage from mailman.handlers.decorate import decorate from mailman.interfaces.member import DeliveryMode, DeliveryStatus from mailman.utilities.i18n import make @@ -172,7 +171,7 @@ class MIMEDigester(Digester): self._keepers = set(config.digests.mime_digest_keep_headers.split()) def _make_message(self): - return MIMEMultipart('mixed') + return MultipartDigestMessage('mixed') def add_toc(self, count): """Add the table of contents.""" diff --git a/src/mailman/runners/tests/test_digest.py b/src/mailman/runners/tests/test_digest.py index 80cf253bc..fb1bb7071 100644 --- a/src/mailman/runners/tests/test_digest.py +++ b/src/mailman/runners/tests/test_digest.py @@ -25,18 +25,20 @@ __all__ = [ ] -import os import unittest +from StringIO import StringIO +from email.iterators import _structure as structure from email.mime.text import MIMEText from mailman.app.lifecycle import create_list from mailman.config import config from mailman.email.message import Message from mailman.runners.digest import DigestRunner from mailman.testing.helpers import ( - LogFileMark, digest_mbox, get_queue_messages, make_testable_runner, - specialized_message_from_string as mfs) + LogFileMark, digest_mbox, get_queue_messages, make_digest_messages, + make_testable_runner, message_from_string) from mailman.testing.layers import ConfigLayer +from string import Template @@ -44,6 +46,7 @@ class TestDigest(unittest.TestCase): """Test the digest runner.""" layer = ConfigLayer + maxDiff = None def setUp(self): self._mlist = create_list('test@example.com') @@ -54,23 +57,9 @@ class TestDigest(unittest.TestCase): self._runner = make_testable_runner(DigestRunner, 'digest') self._process = config.handlers['to-digest'].process - def test_simple_message(self): - msg = mfs("""\ -From: anne@example.org -To: test@example.com - -message triggering a digest -""") - mbox_path = os.path.join(self._mlist.data_path, 'digest.mmdf') - self._process(self._mlist, msg, {}) - self._digestq.enqueue( - msg, - listname=self._mlist.fqdn_listname, - digest_path=mbox_path, - volume=1, digest_number=1) - self._runner.run() - # There are two messages in the virgin queue: the digest as plain-text - # and as multipart. + def _check_virgin_queue(self): + # There should be two messages in the virgin queue: the digest as + # plain-text and as multipart. messages = get_queue_messages('virgin') self.assertEqual(len(messages), 2) self.assertEqual( @@ -80,6 +69,10 @@ message triggering a digest self.assertEqual(item.msg['subject'], 'Test Digest, Vol 1, Issue 1') + def test_simple_message(self): + make_digest_messages(self._mlist) + self._check_virgin_queue() + def test_non_ascii_message(self): msg = Message() msg['From'] = 'anne@example.org' @@ -88,25 +81,62 @@ message triggering a digest msg.attach(MIMEText('message with non-ascii chars: \xc3\xa9', 'plain', 'utf-8')) mbox = digest_mbox(self._mlist) - mbox_path = os.path.join(self._mlist.data_path, 'digest.mmdf') mbox.add(msg.as_string()) - self._digestq.enqueue( - msg, - listname=self._mlist.fqdn_listname, - digest_path=mbox_path, - volume=1, digest_number=1) # Use any error logs as the error message if the test fails. error_log = LogFileMark('mailman.error') - self._runner.run() + make_digest_messages(self._mlist, msg) # The runner will send the file to the shunt queue on exception. self.assertEqual(len(self._shuntq.files), 0, error_log.read()) - # There are two messages in the virgin queue: the digest as plain-text - # and as multipart. - messages = get_queue_messages('virgin') - self.assertEqual(len(messages), 2) - self.assertEqual( - sorted(item.msg.get_content_type() for item in messages), - ['multipart/mixed', 'text/plain']) - for item in messages: - self.assertEqual(item.msg['subject'], - 'Test Digest, Vol 1, Issue 1') + self._check_virgin_queue() + + def test_mime_digest_format(self): + # Make sure that the format of the MIME digest is as expected. + self._mlist.digest_size_threshold = 0.6 + self._mlist.volume = 1 + self._mlist.next_digest_number = 1 + self._mlist.send_welcome_message = False + # Fill the digest. + process = config.handlers['to-digest'].process + size = 0 + for i in range(1, 5): + text = Template("""\ +From: aperson@example.com +To: xtest@example.com +Subject: Test message $i +List-Post: <test@example.com> + +Here is message $i +""").substitute(i=i) + msg = message_from_string(text) + process(self._mlist, msg, {}) + size += len(text) + if size >= self._mlist.digest_size_threshold * 1024: + break + # Run the digest runner to create the MIME and RFC 1153 digests. + runner = make_testable_runner(DigestRunner) + runner.run() + items = get_queue_messages('virgin') + self.assertEqual(len(items), 2) + # Find the MIME one. + mime_digest = None + for item in items: + if item.msg.is_multipart(): + assert mime_digest is None, 'We got two MIME digests' + mime_digest = item.msg + fp = StringIO() + # Verify the structure is what we expect. + structure(mime_digest, fp) + self.assertMultiLineEqual(fp.getvalue(), """\ +multipart/mixed + text/plain + text/plain + message/rfc822 + text/plain + message/rfc822 + text/plain + message/rfc822 + text/plain + message/rfc822 + text/plain + text/plain +""") diff --git a/src/mailman/testing/helpers.py b/src/mailman/testing/helpers.py index b0fe14a0d..fd3af88ee 100644 --- a/src/mailman/testing/helpers.py +++ b/src/mailman/testing/helpers.py @@ -31,6 +31,7 @@ __all__ = [ 'get_lmtp_client', 'get_nntp_server', 'get_queue_messages', + 'make_digest_messages', 'make_testable_runner', 'reset_the_world', 'specialized_message_from_string', @@ -72,6 +73,7 @@ from mailman.interfaces.member import MemberRole from mailman.interfaces.messages import IMessageStore from mailman.interfaces.styles import IStyleManager from mailman.interfaces.usermanager import IUserManager +from mailman.runners.digest import DigestRunner from mailman.utilities.mailbox import Mailbox @@ -529,3 +531,24 @@ class LogFileMark: with open(self._filename) as fp: fp.seek(self._filepos) return fp.read() + + + +def make_digest_messages(mlist, msg=None): + if msg is None: + msg = specialized_message_from_string("""\ +From: anne@example.org +To: {listname} +Message-ID: <testing> + +message triggering a digest +""".format(listname=mlist.fqdn_listname)) + mbox_path = os.path.join(mlist.data_path, 'digest.mmdf') + config.handlers['to-digest'].process(mlist, msg, {}) + config.switchboards['digest'].enqueue( + msg, + listname=mlist.fqdn_listname, + digest_path=mbox_path, + volume=1, digest_number=1) + runner = make_testable_runner(DigestRunner, 'digest') + runner.run() |