diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/mailman/app/docs/pipelines.rst | 8 | ||||
| -rw-r--r-- | src/mailman/core/pipelines.py | 1 | ||||
| -rw-r--r-- | src/mailman/handlers/cook_headers.py | 118 | ||||
| -rw-r--r-- | src/mailman/handlers/docs/subject-munging.rst | 81 | ||||
| -rw-r--r-- | src/mailman/handlers/subject_prefix.py | 187 | ||||
| -rw-r--r-- | src/mailman/handlers/tests/test_subject_prefix.py | 132 | ||||
| -rw-r--r-- | src/mailman/runners/nntp.py | 6 |
7 files changed, 356 insertions, 177 deletions
diff --git a/src/mailman/app/docs/pipelines.rst b/src/mailman/app/docs/pipelines.rst index daedab50a..dfdc6d70c 100644 --- a/src/mailman/app/docs/pipelines.rst +++ b/src/mailman/app/docs/pipelines.rst @@ -45,9 +45,9 @@ etc. To: test@example.com Message-ID: <first> X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB - Subject: [Test] My first post X-Mailman-Version: ... Precedence: list + Subject: [Test] My first post List-Id: <test.example.com> Archived-At: http://lists.example.com/.../4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB List-Archive: <http://lists.example.com/archives/test@example.com> @@ -84,9 +84,9 @@ processing queues. To: test@example.com Message-ID: <first> X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB - Subject: [Test] My first post X-Mailman-Version: ... Precedence: list + Subject: [Test] My first post List-Id: <test.example.com> ... <BLANKLINE> @@ -121,9 +121,9 @@ delivered to end recipients. To: test@example.com Message-ID: <first> X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB - Subject: [Test] My first post X-Mailman-Version: ... Precedence: list + Subject: [Test] My first post List-Id: <test.example.com> ... <BLANKLINE> @@ -152,9 +152,9 @@ There's now one message in the digest mailbox, getting ready to be sent. To: test@example.com Message-ID: <first> X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB - Subject: [Test] My first post X-Mailman-Version: ... Precedence: list + Subject: [Test] My first post List-Id: <test.example.com> ... <BLANKLINE> diff --git a/src/mailman/core/pipelines.py b/src/mailman/core/pipelines.py index e164169a4..5fdba8358 100644 --- a/src/mailman/core/pipelines.py +++ b/src/mailman/core/pipelines.py @@ -120,6 +120,7 @@ class PostingPipeline(BasePipeline): 'cleanse', 'cleanse-dkim', 'cook-headers', + 'subject-prefix', 'rfc-2369', 'to-archive', 'to-digest', diff --git a/src/mailman/handlers/cook_headers.py b/src/mailman/handlers/cook_headers.py index 1ab527bb4..f37e8f0e2 100644 --- a/src/mailman/handlers/cook_headers.py +++ b/src/mailman/handlers/cook_headers.py @@ -27,8 +27,7 @@ __all__ = [ import re -from email.errors import HeaderParseError -from email.header import Header, decode_header, make_header +from email.header import Header from email.utils import parseaddr, formataddr, getaddresses from zope.interface import implementer @@ -78,13 +77,6 @@ def process(mlist, msg, msgdata): msgdata['original_sender'] = msg.sender # VirginRunner sets _fasttrack for internally crafted messages. fasttrack = msgdata.get('_fasttrack') - if not msgdata.get('isdigest') and not fasttrack: - try: - prefix_subject(mlist, msg, msgdata) - except (UnicodeError, ValueError): - # TK: Sometimes subject header is not MIME encoded for 8bit - # simply abort prefixing. - pass # Add Precedence: and other useful headers. None of these are standard # and finding information on some of them are fairly difficult. Some are # just common practice, and we'll add more here as they become necessary. @@ -171,114 +163,6 @@ def process(mlist, msg, msgdata): -def prefix_subject(mlist, msg, msgdata): - """Maybe add a subject prefix. - - Add the subject prefix unless the message is a digest or is being fast - tracked (e.g. internally crafted, delivered to a single user such as the - list admin). - """ - if not mlist.subject_prefix.strip(): - return - prefix = mlist.subject_prefix - subject = msg.get('subject', '') - # Try to figure out what the continuation_ws is for the header - if isinstance(subject, Header): - lines = str(subject).splitlines() - else: - lines = subject.splitlines() - ws = '\t' - if len(lines) > 1 and lines[1] and lines[1][0] in ' \t': - ws = lines[1][0] - msgdata['original_subject'] = subject - # The subject may be multilingual but we take the first charset as major - # one and try to decode. If it is decodable, returned subject is in one - # line and cset is properly set. If fail, subject is mime-encoded and - # cset is set as us-ascii. See detail for ch_oneline() (CookHeaders one - # line function). - subject, cset = ch_oneline(subject) - # TK: Python interpreter has evolved to be strict on ascii charset code - # range. It is safe to use unicode string when manupilating header - # contents with re module. It would be best to return unicode in - # ch_oneline() but here is temporary solution. - subject = subject.decode(cset) - # If the subject_prefix contains '%d', it is replaced with the - # mailing list sequential number. Sequential number format allows - # '%d' or '%05d' like pattern. - prefix_pattern = re.escape(prefix) - # unescape '%' :-< - prefix_pattern = '%'.join(prefix_pattern.split(r'\%')) - p = re.compile('%\d*d') - if p.search(prefix, 1): - # prefix have number, so we should search prefix w/number in subject. - # Also, force new style. - prefix_pattern = p.sub(r'\s*\d+\s*', prefix_pattern) - subject = re.sub(prefix_pattern, '', subject) - rematch = re.match('((RE|AW|SV|VS)(\[\d+\])?:\s*)+', subject, re.I) - if rematch: - subject = subject[rematch.end():] - recolon = 'Re:' - else: - recolon = '' - # At this point, subject may become null if someone post mail with - # subject: [subject prefix] - if subject.strip() == '': - subject = _('(no subject)') - cset = mlist.preferred_language.charset - # and substitute %d in prefix with post_id - try: - prefix = prefix % mlist.post_id - except TypeError: - pass - # Get the header as a Header instance, with proper unicode conversion - if not recolon: - h = uheader(mlist, prefix, 'Subject', continuation_ws=ws) - else: - h = uheader(mlist, prefix, 'Subject', continuation_ws=ws) - h.append(recolon) - # TK: Subject is concatenated and unicode string. - subject = subject.encode(cset, 'replace') - h.append(subject, cset) - del msg['subject'] - msg['Subject'] = h - ss = uheader(mlist, recolon, 'Subject', continuation_ws=ws) - ss.append(subject, cset) - msgdata['stripped_subject'] = ss - - - -def ch_oneline(headerstr): - # Decode header string in one line and convert into single charset. - # Return (string, cset) tuple as check for failure. - try: - d = decode_header(headerstr) - # At this point, we should rstrip() every string because some - # MUA deliberately add trailing spaces when composing return - # message. - d = [(s.rstrip(), c) for (s, c) in d] - # Find all charsets in the original header. We use 'utf-8' rather - # than using the first charset (in mailman 2.1.x) if multiple - # charsets are used. - csets = [] - for (s, c) in d: - if c and c not in csets: - csets.append(c) - if len(csets) == 0: - cset = 'us-ascii' - elif len(csets) == 1: - cset = csets[0] - else: - cset = 'utf-8' - h = make_header(d) - ustr = str(h) - oneline = ''.join(ustr.splitlines()) - return oneline.encode(cset, 'replace'), cset - except (LookupError, UnicodeError, ValueError, HeaderParseError): - # possibly charset problem. return with undecoded string in one line. - return ''.join(headerstr.splitlines()), 'us-ascii' - - - @implementer(IHandler) class CookHeaders: """Modify message headers.""" diff --git a/src/mailman/handlers/docs/subject-munging.rst b/src/mailman/handlers/docs/subject-munging.rst index b51fedebd..de22a928c 100644 --- a/src/mailman/handlers/docs/subject-munging.rst +++ b/src/mailman/handlers/docs/subject-munging.rst @@ -1,44 +1,42 @@ -=============== -Subject munging -=============== +================ +Subject prefixes +================ -Messages that flow through the global pipeline get their headers *cooked*, -which basically means that their headers go through several mostly unrelated -transformations. Some headers get added, others get changed. Some of these -changes depend on mailing list settings and others depend on how the message -is getting sent through the system. We'll take things one-by-one. +Mailing lists can define a *subject prefix* which gets added to the front of +any ``Subject`` text. This can be used to quickly identify which mailing list +the message was posted to. >>> mlist = create_list('test@example.com') +The default list style gives the mailing list a default prefix. -Inserting a prefix -================== + >>> print(mlist.subject_prefix) + [Test] -Another thing header cooking does is *munge* the ``Subject`` header by -inserting the subject prefix for the list at the front. If there's no subject -header in the original message, Mailman uses a canned default. In order to do -subject munging, a mailing list must have a preferred language. -:: +This can be changed to anything, but typically ends with a trailing space. >>> mlist.subject_prefix = '[XTest] ' - >>> mlist.preferred_language = 'en' + >>> process = config.handlers['subject-prefix'].process + + +No Subject +========== + +If the original message has no ``Subject``, then a canned one is used. + >>> msg = message_from_string("""\ ... From: aperson@example.com ... ... A message of great import. ... """) - >>> msgdata = {} - - >>> from mailman.handlers.cook_headers import process - >>> process(mlist, msg, msgdata) - -The original subject header is stored in the message metadata. - - >>> msgdata['original_subject'] - '' + >>> process(mlist, msg, {}) >>> print(msg['subject']) [XTest] (no subject) + +Inserting a prefix +================== + If the original message had a ``Subject`` header, then the prefix is inserted at the beginning of the header's value. @@ -50,34 +48,12 @@ at the beginning of the header's value. ... """) >>> msgdata = {} >>> process(mlist, msg, msgdata) - >>> print(msgdata['original_subject']) - Something important >>> print(msg['subject']) [XTest] Something important -``Subject`` headers are not munged for digest messages. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Subject: Something important - ... - ... A message of great import. - ... """) - >>> process(mlist, msg, dict(isdigest=True)) - >>> print(msg['subject']) - Something important +The original ``Subject`` is available in the metadata. -Nor are they munged for *fast tracked* messages, which are generally defined -as messages that Mailman crafts internally. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Subject: Something important - ... - ... A message of great import. - ... """) - >>> process(mlist, msg, dict(_fasttrack=True)) - >>> print(msg['subject']) + >>> print(msgdata['original_subject']) Something important If a ``Subject`` header already has a prefix, usually following a ``Re:`` @@ -95,8 +71,7 @@ front of the header text. [XTest] Re: Something important If the ``Subject`` header has a prefix at the front of the header text, that's -where it will stay. This is called *new style* prefixing and is the only -option available in Mailman 3. +where it will stay. >>> msg = message_from_string("""\ ... From: aperson@example.com @@ -124,7 +99,7 @@ set than the encoded header. >>> process(mlist, msg, {}) >>> print(msg['subject'].encode()) [XTest] =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= - >>> print(msg['subject']) + >>> print(str(msg['subject'])) [XTest] メールマン @@ -194,7 +169,7 @@ prefix, possibly with a different posting number. >>> print(msg['subject'].encode()) [XTest 456] Re: =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= >>> print(msg['subject']) - [XTest 456] Re: メールマン + [XTest 456] Re: メールマン As before, old style subject prefixes are re-ordered. diff --git a/src/mailman/handlers/subject_prefix.py b/src/mailman/handlers/subject_prefix.py new file mode 100644 index 000000000..ee1921ac2 --- /dev/null +++ b/src/mailman/handlers/subject_prefix.py @@ -0,0 +1,187 @@ +# Copyright (C) 2014 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Subject header prefix munging.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'SubjectPrefix', + ] + + +import re + +from email.header import Header, make_header, decode_header +from mailman.core.i18n import _ +from mailman.interfaces.handler import IHandler +from zope.interface import implementer + + +RE_PATTERN = '((RE|AW|SV|VS)(\[\d+\])?:\s*)+' +ASCII_CHARSETS = (None, 'ascii', 'us-ascii') +EMPTYSTRING = '' + + + +def ascii_header(mlist, msgdata, subject, prefix, prefix_pattern, ws): + if mlist.preferred_language.charset not in ASCII_CHARSETS: + return None + for chunk, charset in decode_header(subject.encode()): + if charset not in ASCII_CHARSETS: + return None + subject_text = EMPTYSTRING.join(str(subject).splitlines()) + rematch = re.match(RE_PATTERN, subject_text, re.I) + if rematch: + subject_text = subject_text[rematch.end():] + recolon = 'Re: ' + else: + recolon = '' + # At this point, the subject may become null if someone posted mail + # with "Subject: [subject prefix]". + if subject_text.strip() == '': + with _.using(mlist.preferred_language.code): + subject_text = _('(no subject)') + else: + subject_text = re.sub(prefix_pattern, '', subject_text) + msgdata['stripped_subject'] = subject_text + lines = subject_text.splitlines() + first_line = [lines[0]] + if recolon: + first_line.insert(0, recolon) + if prefix: + first_line.insert(0, prefix) + subject_text = EMPTYSTRING.join(first_line) + return Header(subject_text, continuation_ws=ws) + + +def all_same_charset(mlist, msgdata, subject, prefix, prefix_pattern, ws): + list_charset = mlist.preferred_language.charset + chunks = [] + for chunk, charset in decode_header(subject.encode()): + if charset is None: + charset = 'us-ascii' + chunks.append(chunk.decode(charset)) + if charset != list_charset: + return None + subject_text = EMPTYSTRING.join(chunks) + rematch = re.match(RE_PATTERN, subject_text, re.I) + if rematch: + subject_text = subject_text[rematch.end():] + recolon = 'Re: ' + else: + recolon = '' + # At this point, the subject may become null if someone posted mail + # with "Subject: [subject prefix]". + if subject_text.strip() == '': + with _.push(mlist.preferred_language.code): + subject_text = _('(no subject)') + else: + subject_text = re.sub(prefix_pattern, '', subject_text) + msgdata['stripped_subject'] = subject_text + lines = subject_text.splitlines() + first_line = [lines[0]] + if recolon: + first_line.insert(0, recolon) + if prefix: + first_line.insert(0, prefix) + subject_text = EMPTYSTRING.join(first_line) + return Header(subject_text, charset=list_charset, continuation_ws=ws) + + +def mixed_charsets(mlist, msgdata, subject, prefix, prefix_pattern, ws): + list_charset = mlist.preferred_language.charset + chunks = decode_header(subject.encode()) + if len(chunks) == 0: + with _.push(mlist.preferred_language.code): + subject_text = _('(no subject)') + chunks = [(prefix, list_charset), + (subject_text, list_charset), + ] + return make_header(chunks, continuation_ws=ws) + # Only search the first chunk for Re and existing prefix. + chunk_text, chunk_charset = chunks[0] + if chunk_charset is None: + chunk_charset = 'us-ascii' + first_text = chunk_text.decode(chunk_charset) + first_text = re.sub(prefix_pattern, '', first_text).lstrip() + rematch = re.match(RE_PATTERN, first_text, re.I) + if rematch: + first_text = 'Re: ' + first_text[rematch.end():] + chunks[0] = (first_text, chunk_charset) + # The subject text stripped of the prefix, for use in the NNTP gateway. + msgdata['stripped_subject'] = str(make_header(chunks, continuation_ws=ws)) + chunks.insert(0, (prefix, list_charset)) + return make_header(chunks, continuation_ws=ws) + + + +@implementer(IHandler) +class SubjectPrefix: + """Add a list-specific prefix to the Subject header value.""" + + name = 'subject-prefix' + description = _('Add a list-specific prefix to the Subject header value.') + + def process(self, mlist, msg, msgdata): + """See `IHandler`.""" + if msgdata.get('isdigest') or msgdata.get('_fasttrack'): + return + prefix = mlist.subject_prefix + if not prefix.strip(): + return + subject = msg.get('subject', '') + # Turn the value into a Header instance and try to figure out what + # continuation whitespace is being used. + # Save the original Subject. + msgdata['original_subject'] = subject + if isinstance(subject, Header): + subject_text = str(subject) + else: + subject = make_header(decode_header(subject)) + subject_text = str(subject) + lines = subject_text.splitlines() + ws = '\t' + if len(lines) > 1 and lines[1] and lines[1][0] in ' \t': + ws = lines[1][0] + # If the subject_prefix contains '%d', it is replaced with the mailing + # list's sequence number. The sequential number format allows '%d' or + # '%05d' like pattern. + prefix_pattern = re.escape(prefix) + # Unescape '%'. + prefix_pattern = '%'.join(prefix_pattern.split(r'\%')) + p = re.compile('%\d*d') + if p.search(prefix, 1): + # The prefix has number, so we should search prefix w/number in + # subject. Also, force new style. + prefix_pattern = p.sub(r'\s*\d+\s*', prefix_pattern) + # Substitute %d in prefix with post_id + try: + prefix = prefix % mlist.post_id + except TypeError: + pass + for handler in (ascii_header, + all_same_charset, + mixed_charsets, + ): + new_subject = handler( + mlist, msgdata, subject, prefix, prefix_pattern, ws) + if new_subject is not None: + del msg['subject'] + msg['Subject'] = new_subject + return diff --git a/src/mailman/handlers/tests/test_subject_prefix.py b/src/mailman/handlers/tests/test_subject_prefix.py new file mode 100644 index 000000000..1125f3811 --- /dev/null +++ b/src/mailman/handlers/tests/test_subject_prefix.py @@ -0,0 +1,132 @@ +# Copyright (C) 2014 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test the Subject header prefix munging..""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestSubjectPrefix', + ] + + +import unittest + +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.email.message import Message +from mailman.testing.layers import ConfigLayer + + + +class TestSubjectPrefix(unittest.TestCase): + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._process = config.handlers['subject-prefix'].process + + def test_isdigest(self): + # If the message is destined for the digest, the Subject header does + # not get touched. + msg = Message() + msg['Subject'] = 'A test message' + self._process(self._mlist, msg, dict(isdigest=True)) + self.assertEqual(str(msg['subject']), 'A test message') + + def test_fasttrack(self): + # Messages internally crafted are 'fast tracked' and don't get their + # Subjects prefixed either. + msg = Message() + msg['Subject'] = 'A test message' + self._process(self._mlist, msg, dict(_fasttrack=True)) + self.assertEqual(str(msg['subject']), 'A test message') + + def test_whitespace_only_prefix(self): + # If the Subject prefix only contains whitespace, ignore it. + self._mlist.subject_prefix = ' ' + msg = Message() + msg['Subject'] = 'A test message' + self._process(self._mlist, msg, dict(_fasttrack=True)) + self.assertEqual(str(msg['subject']), 'A test message') + + def test_save_original_subject(self): + # When the Subject gets prefixed, the original is saved in the message + # metadata. + msgdata = {} + msg = Message() + msg['Subject'] = 'A test message' + self._process(self._mlist, msg, msgdata) + self.assertEqual(msgdata['original_subject'], 'A test message') + + def test_prefix(self): + # The Subject gets prefixed. The prefix gets automatically set by the + # list style when the list gets created. + msg = Message() + msg['Subject'] = 'A test message' + self._process(self._mlist, msg, {}) + self.assertEqual(str(msg['subject']), '[Test] A test message') + + def test_no_double_prefix(self): + # Don't add a prefix if the subject already contains one. + msg = Message() + msg['Subject'] = '[Test] A test message' + self._process(self._mlist, msg, {}) + self.assertEqual(str(msg['subject']), '[Test] A test message') + + def test_re_prefix(self): + # The subject has a Re: prefix. Make sure that gets preserved, but + # after the list prefix. + msg = Message() + msg['Subject'] = 'Re: [Test] A test message' + self._process(self._mlist, msg, {}) + self.assertEqual(str(msg['subject']), '[Test] Re: A test message') + + def test_multiline_subject(self): + # The subject appears on multiple lines. + msg = Message() + msg['Subject'] = '\n A test message' + self._process(self._mlist, msg, {}) + self.assertEqual(str(msg['subject']), '[Test] A test message') + + def test_i18n_prefix(self): + # The Subject header is encoded, but the prefix is still added. + msg = Message() + msg['Subject'] = '=?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?=' + self._process(self._mlist, msg, {}) + subject = msg['subject'] + self.assertEqual(subject.encode(), + '[Test] =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?=') + self.assertEqual(str(subject), '[Test] メールマン') + + def test_i18n_subject_with_sequential_prefix_and_re(self): + # The mailing list defines a sequential prefix, and the original + # Subject has a prefix with a different sequence number, *and* it also + # contains a Re: prefix. Make sure the sequence gets updated and all + # the bits get put back together in the right order. + self._mlist.subject_prefix = '[Test %d]' + self._mlist.post_id = 456 + msg = Message() + msg['Subject'] = \ + '[Test 123] Re: =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?=' + self._process(self._mlist, msg, {}) + subject = msg['subject'] + self.assertEqual( + subject.encode(), + '[Test 456] Re: =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?=') + self.assertEqual(str(subject), '[Test 456] Re: メールマン') diff --git a/src/mailman/runners/nntp.py b/src/mailman/runners/nntp.py index 5d0013055..d26001a57 100644 --- a/src/mailman/runners/nntp.py +++ b/src/mailman/runners/nntp.py @@ -111,9 +111,9 @@ def prepare_message(mlist, msg, msgdata): del msg['approved'] msg['Approved'] = mlist.posting_address # Should we restore the original, non-prefixed subject for gatewayed - # messages? TK: We use stripped_subject (prefix stripped) which was - # crafted in CookHeaders.py to ensure prefix was stripped from the subject - # came from mailing list user. + # messages? TK: We use stripped_subject (prefix stripped) which was crafted + # in the subject-prefix handler to ensure prefix was stripped from the + # subject came from mailing list user. stripped_subject = msgdata.get('stripped_subject', msgdata.get('original_subject')) if not mlist.nntp_prefix_subject_too and stripped_subject is not None: |