diff options
| -rw-r--r-- | src/mailman/runners/docs/digester.rst | 200 | ||||
| -rw-r--r-- | src/mailman/runners/tests/test_digest.py | 75 | ||||
| -rw-r--r-- | src/mailman/testing/helpers.py | 25 |
3 files changed, 101 insertions, 199 deletions
diff --git a/src/mailman/runners/docs/digester.rst b/src/mailman/runners/docs/digester.rst index 536cf06c2..fc59954bc 100644 --- a/src/mailman/runners/docs/digester.rst +++ b/src/mailman/runners/docs/digester.rst @@ -61,6 +61,7 @@ But the message metadata has a reference to the digest file. version : 3 volume : 1 +.. # Put the messages back in the queue for the runner to handle. >>> filebase = digestq.enqueue(entry.msg, entry.msgdata) @@ -281,205 +282,6 @@ The RFC 1153 contains the digest in a single plain text message. <BLANKLINE> -Internationalized digests -========================= - -When messages come in with a content-type character set different than that of -the list's preferred language, recipients will get an internationalized -digest. French is not enabled by default site-wide, so enable that now. -:: - - # Simulate the site administrator setting the default server language to - # French in the configuration file. Without this, the English template - # will be found and the masthead won't be translated. - >>> config.push('french', """ - ... [mailman] - ... default_language: fr - ... """) - - >>> mlist.preferred_language = 'fr' - >>> msg = message_from_string("""\ - ... From: aperson@example.org - ... To: test@example.com - ... Subject: =?iso-2022-jp?b?GyRCMGxIVhsoQg==?= - ... MIME-Version: 1.0 - ... Content-Type: text/plain; charset=iso-2022-jp - ... Content-Transfer-Encoding: 7bit - ... - ... \x1b$B0lHV\x1b(B - ... """) - -Set the digest threshold to zero so that the digests will be sent immediately. - - >>> mlist.digest_size_threshold = 0 - >>> process(mlist, msg, {}) - -The marker message is sitting in the digest queue. - - >>> len(digestq.files) - 1 - >>> entry = get_queue_messages('digest')[0] - >>> dump_msgdata(entry.msgdata) - _parsemsg : False - digest_number: 2 - digest_path : .../lists/test@example.com/digest.1.2.mmdf - listid : test.example.com - version : 3 - volume : 1 - -The digest runner runs a loop, placing the two digests into the virgin queue. - - # Put the messages back in the queue for the runner to handle. - >>> filebase = digestq.enqueue(entry.msg, entry.msgdata) - >>> runner.run() - >>> messages = get_queue_messages('virgin') - >>> len(messages) - 2 - -One of which is the MIME digest and the other of which is the RFC 1153 digest. - - >>> mime, rfc1153 = mime_rfc1153(messages) - -You can see that the digests contain a mix of French and Japanese. - - >>> print(mime.msg.as_string()) - Content-Type: multipart/mixed; boundary="===============...==" - MIME-Version: 1.0 - From: test-request@example.com - Subject: Groupe Test, Vol 1, Parution 2 - To: test@example.com - Reply-To: test@example.com - Date: ... - Message-ID: ... - <BLANKLINE> - --===============...== - Content-Type: text/plain; charset="iso-8859-1" - MIME-Version: 1.0 - Content-Transfer-Encoding: quoted-printable - Content-Description: Groupe Test, Vol 1, Parution 2 - <BLANKLINE> - Envoyez vos messages pour la liste Test =E0 - test@example.com - <BLANKLINE> - Pour vous (d=E9s)abonner par le web, consultez - http://lists.example.com/listinfo/test@example.com - <BLANKLINE> - ou, par courriel, envoyez un message avec =AB=A0help=A0=BB dans le corps ou - dans le sujet =E0 - test-request@example.com - <BLANKLINE> - Vous pouvez contacter l'administrateur de la liste =E0 l'adresse - test-owner@example.com - <BLANKLINE> - Si vous r=E9pondez, n'oubliez pas de changer l'objet du message afin - qu'il soit plus sp=E9cifique que =AB=A0Re: Contenu du groupe de Test...=A0= - =BB - --===============...== - Content-Type: text/plain; charset="utf-8" - MIME-Version: 1.0 - Content-Transfer-Encoding: base64 - Content-Description: Today's Topics (1 messages) - <BLANKLINE> - VGjDqG1lcyBkdSBqb3VyIDoKCiAgIDEuIOS4gOeVqiAoYXBlcnNvbkBleGFtcGxlLm9yZykK - <BLANKLINE> - --===============...== - Content-Type: message/rfc822 - MIME-Version: 1.0 - <BLANKLINE> - From: aperson@example.org - To: test@example.com - Subject: =?iso-2022-jp?b?GyRCMGxIVhsoQg==?= - MIME-Version: 1.0 - Content-Type: text/plain; charset=iso-2022-jp - Content-Transfer-Encoding: 7bit - <BLANKLINE> - $B0lHV(B - <BLANKLINE> - --===============...== - Content-Type: text/plain; charset="iso-8859-1" - MIME-Version: 1.0 - Content-Transfer-Encoding: quoted-printable - Content-Description: =?utf-8?q?Pied_de_page_des_remises_group=C3=A9es?= - <BLANKLINE> - _______________________________________________ - Test mailing list - test@example.com - http://lists.example.com/listinfo/test@example.com - <BLANKLINE> - --===============...==-- - -The RFC 1153 digest will be encoded in UTF-8 since it contains a mixture of -French and Japanese characters. - - >>> print(rfc1153.msg.as_string()) - From: test-request@example.com - Subject: Groupe Test, Vol 1, Parution 2 - To: test@example.com - Reply-To: test@example.com - Date: ... - Message-ID: ... - MIME-Version: 1.0 - Content-Type: text/plain; charset="utf-8" - Content-Transfer-Encoding: base64 - <BLANKLINE> - RW52b... - <BLANKLINE> - -The content can be decoded to see the actual digest text. -:: - - # We must display the repr of the decoded value because doctests cannot - # handle the non-ascii characters. - >>> [repr(line) - ... for line in rfc1153.msg.get_payload(decode=True).splitlines()] - ["'Envoyez vos messages pour la liste Test \\xc3\\xa0'", - "'\\ttest@example.com'", - "''", - "'Pour vous (d\\xc3\\xa9s)abonner par le web, consultez'", - "'\\thttp://lists.example.com/listinfo/test@example.com'", - "''", - "'ou, par courriel, envoyez un message avec \\xc2\\xab\\xc2\\xa0... - "'dans le sujet \\xc3\\xa0'", - "'\\ttest-request@example.com'", - "''", - '"Vous pouvez contacter l\'administrateur de la liste \\xc3\\xa0 ... - "'\\ttest-owner@example.com'", - "''", - '"Si vous r\\xc3\\xa9pondez, n\'oubliez pas de changer l\'objet du ... - '"qu\'il soit plus sp\\xc3\\xa9cifique que \\xc2\\xab\\xc2\\xa0Re: ... - "''", - "'Th\\xc3\\xa8mes du jour :'", - "''", - "' 1. \\xe4\\xb8\\x80\\xe7\\x95\\xaa (aperson@example.org)'", - "''", - "''", - "'---------------------------------------------------------------------... - "''", - "'From: aperson@example.org'", - "'Subject: \\xe4\\xb8\\x80\\xe7\\x95\\xaa'", - "'To: test@example.com'", - "'Content-Type: text/plain; charset=iso-2022-jp'", - "''", - "'\\xe4\\xb8\\x80\\xe7\\x95\\xaa'", - "''", - "'------------------------------'", - "''", - "'Subject: Pied de page des remises group\\xc3\\xa9es'", - "''", - "'_______________________________________________'", - "'Test mailing list'", - "'test@example.com'", - "'http://lists.example.com/listinfo/test@example.com'", - "''", - "''", - "'------------------------------'", - "''", - "'Fin de Groupe Test, Vol 1, Parution 2'", - "'*************************************'"] - - >>> config.pop('french') - - Digest delivery =============== diff --git a/src/mailman/runners/tests/test_digest.py b/src/mailman/runners/tests/test_digest.py index 6cd3c9a01..bd32050fb 100644 --- a/src/mailman/runners/tests/test_digest.py +++ b/src/mailman/runners/tests/test_digest.py @@ -22,6 +22,7 @@ from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ 'TestDigest', + 'TestI18nDigest', ] @@ -110,3 +111,77 @@ message triggering a digest for item in messages: self.assertEqual(item.msg['subject'], 'Test Digest, Vol 1, Issue 1') + + + +class TestI18nDigest(unittest.TestCase): + layer = ConfigLayer + maxDiff = None + + def setUp(self): + config.push('french', """ + [mailman] + default_language: fr + """) + self.addCleanup(config.pop, 'french') + self._mlist = create_list('test@example.com') + self._mlist.preferred_language = 'fr' + self._mlist.digest_size_threshold = 0 + self._process = config.handlers['to-digest'].process + self._runner = make_testable_runner(DigestRunner) + + def test_multilingual_digest(self): + # When messages come in with a content-type character set different + # than that of the list's preferred language, recipients will get an + # internationalized digest. + msg = mfs("""\ +From: aperson@example.org +To: test@example.com +Subject: =?iso-2022-jp?b?GyRCMGxIVhsoQg==?= +MIME-Version: 1.0 +Content-Type: text/plain; charset=iso-2022-jp +Content-Transfer-Encoding: 7bit + +\x1b$B0lHV\x1b(B +""") + self._process(self._mlist, msg, {}) + self._runner.run() + # There are two digests in the virgin queue; one is the MIME digest + # and the other is the RFC 1153 digest. + messages = get_queue_messages('virgin') + self.assertEqual(len(messages), 2) + if messages[0].msg.is_multipart(): + mime, rfc1153 = messages[0].msg, messages[1].msg + else: + rfc1153, mime = messages[0].msg, messages[1].msg + # The MIME version contains a mix of French and Japanese. The digest + # chrome added by Mailman is in French. + self.assertEqual(mime['subject'].encode(), + '=?iso-8859-1?q?Groupe_Test=2C_Vol_1=2C_Parution_1?=') + self.assertEqual(str(mime['subject']), + 'Groupe Test, Vol 1, Parution 1') + # The first subpart contains the iso-8859-1 masthead. + masthead = mime.get_payload(0).get_payload(decode=True).decode( + 'iso-8859-1') + self.assertMultiLineEqual(masthead.splitlines()[0], + 'Envoyez vos messages pour la liste Test à') + # The second subpart contains the utf-8 table of contents. + self.assertEqual(mime.get_payload(1)['content-description'], + "Today's Topics (1 messages)") + toc = mime.get_payload(1).get_payload(decode=True).decode('utf-8') + self.assertMultiLineEqual(toc.splitlines()[0], 'Thèmes du jour :') + # The third subpart contains the posted message in Japanese. + self.assertEqual(mime.get_payload(2).get_content_type(), + 'message/rfc822') + post = mime.get_payload(2).get_payload(0) + self.assertEqual(post['subject'], '=?iso-2022-jp?b?GyRCMGxIVhsoQg==?=') + # Compare the bytes so that this module doesn't contain string + # literals in multiple incompatible character sets. + self.assertEqual(post.get_payload(decode=True), b'\x1b$B0lHV\x1b(B\n') + # The RFC 1153 digest will have the same subject, but its payload will + # be recast into utf-8. + self.assertEqual(str(rfc1153['subject']), + 'Groupe Test, Vol 1, Parution 1') + self.assertEqual(rfc1153.get_charset(), 'utf-8') + lines = rfc1153.get_payload(decode=True).decode('utf-8').splitlines() + self.assertEqual(lines[0], 'Envoyez vos messages pour la liste Test à') diff --git a/src/mailman/testing/helpers.py b/src/mailman/testing/helpers.py index 5bfac20ee..1f68e6975 100644 --- a/src/mailman/testing/helpers.py +++ b/src/mailman/testing/helpers.py @@ -22,6 +22,7 @@ from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ 'LogFileMark', + 'PrettyEmailPolicy', 'TestableMaster', 'call_api', 'chdir', @@ -61,9 +62,11 @@ from httplib2 import Http from lazr.config import as_timedelta from six.moves.urllib_error import HTTPError from six.moves.urllib_parse import urlencode +from unittest.mock import patch from zope import event from zope.component import getUtility +from email.policy import Compat32 from mailman.bin.master import Loop as Master from mailman.config import config from mailman.database.transaction import transaction @@ -532,3 +535,25 @@ class LogFileMark: with open(self._filename) as fp: fp.seek(self._filepos) return fp.read() + + + +def _pretty(self, *args, **kws): + return str(self) + + +class PrettyEmailPolicy(Compat32): + """Horrible hack to make mailman/runners/docs/digester.rst work. + + Back in Python 2 days, the i18n'd headers printed in digester.rst used the + full unicode string version, instead of the RFC 2047 encoded headers. + It's more correct to use the RFC 2047 headers, but it's also uglier in a + doctest, so to port the doctest to Python 3, we use this email policy hack + to get the headers printed as (unicode) strings instead of RFC 2047 + encoded headers. + """ + # This will hurt your eyeballs. It relies on the specific implementation + # of Compat32 and it *will* break if that class is refactored. + @patch('email.header.Header.encode', _pretty) + def _fold(self, name, value, sanitize): + return super()._fold(name, value, sanitize) |