diff options
Diffstat (limited to 'src/mailman/handlers')
| -rw-r--r-- | src/mailman/handlers/dmarc.py | 203 | ||||
| -rw-r--r-- | src/mailman/handlers/docs/dmarc-mitigations.rst | 145 | ||||
| -rw-r--r-- | src/mailman/handlers/tests/test_dmarc.py | 244 |
3 files changed, 592 insertions, 0 deletions
diff --git a/src/mailman/handlers/dmarc.py b/src/mailman/handlers/dmarc.py new file mode 100644 index 000000000..8bdce040f --- /dev/null +++ b/src/mailman/handlers/dmarc.py @@ -0,0 +1,203 @@ +# Copyright (C) 2016 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Do DMARC Munge From and Wrap Message actions.""" + +import re +import copy +import logging + +from email.header import Header, decode_header +from email.mime.message import MIMEMessage +from email.mime.text import MIMEText +from email.utils import formataddr, getaddresses, make_msgid +from mailman import public +from mailman.core.i18n import _ +from mailman.interfaces.handler import IHandler +from mailman.interfaces.mailinglist import ( + DMARCModerationAction, FromIsList, ReplyToMunging) +from mailman.utilities.string import wrap +from zope.interface import implementer + + +log = logging.getLogger('mailman.error') + +COMMASPACE = ', ' +MAXLINELEN = 78 +NONASCII = re.compile('[^\s!-~]') +# Headers from the original that we want to keep in the wrapper. These are +# actually regexps matched with re.match so they match anything that starts +# with the given string unless they end with '$'. +KEEPERS = ('archived-at', + 'date', + 'in-reply-to', + 'list-', + 'precedence', + 'references', + 'subject', + 'to', + 'x-mailman-', + ) + + +def munged_headers(mlist, msg, msgdata): + # Be as robust as possible here. + faddrs = getaddresses(msg.get_all('from', [])) + # Strip the nulls and bad emails. + faddrs = [x for x in faddrs if x[1].find('@') > 0] + if len(faddrs) == 1: + realname, email = o_from = faddrs[0] + else: + # No From: or multiple addresses. Just punt and take + # the get_sender result. + realname = '' + email = msgdata['original_sender'] + o_from = (realname, email) + if not realname: + member = mlist.members.get_member(email) + if member: + realname = member.display_name or email + else: + realname = email + # Remove domain from realname if it looks like an email address + realname = re.sub(r'@([^ .]+\.)+[^ .]+$', '---', realname) + # Make a display name and RFC 2047 encode it if necessary. This is + # difficult and kludgy. If the realname came from From: it should be + # ascii or RFC 2047 encoded. If it came from the list, it should be + # a string. If it's from the email address, it should be an ascii string. + # In any case, ensure it's an unencoded string. + srn = '' + for frag, cs in decode_header(realname): + if not cs: + # Character set should be ascii, but use iso-8859-1 anyway. + cs = 'iso-8859-1' + if not isinstance(frag, str): + srn += str(frag, cs, errors='replace') + else: + srn += frag + # The list's real_name is a string. + slrn = mlist.display_name + # get translated 'via' with dummy replacements + realname = '$realname' + lrn = '$lrn' # noqa F841 + # Ensure the i18n context is the list's preferred_language. + with _.using(mlist.preferred_language.code): + via = _('$realname via $lrn') + # Replace the dummy replacements. + via = re.sub('\$lrn', slrn, re.sub('\$realname', srn, via)) + # And get an RFC 2047 encoded header string. + dn = str(Header(via, mlist.preferred_language.charset)) + retn = [('From', formataddr((dn, mlist.posting_address)))] + # We've made the munged From:. Now put the original in Reply-To: or Cc: + if mlist.reply_goes_to_list == ReplyToMunging.no_munging: + # Add original from to Reply-To: + add_to = 'Reply-To' + else: + # Add original from to Cc: + add_to = 'Cc' + orig = getaddresses(msg.get_all(add_to, [])) + if o_from[1] not in [x[1] for x in orig]: + orig.append(o_from) + retn.append((add_to, COMMASPACE.join([formataddr(x) for x in orig]))) + return retn + + +def munge_from(mlist, msg, msgdata): + for k, v in munged_headers(mlist, msg, msgdata): + del msg[k] + msg[k] = v + return + + +def wrap_message(mlist, msg, msgdata, dmarc_wrap=False): + # Create a wrapper message around the original. + # There are various headers in msg that we don't want, so we basically + # make a copy of the msg, then delete almost everything and set/copy + # what we want. + omsg = copy.deepcopy(msg) + for key in msg.keys(): + keep = False + for keeper in KEEPERS: + if re.match(keeper, key, re.I): + keep = True + break + if not keep: + del msg[key] + msg['MIME-Version'] = '1.0' + msg['Message-ID'] = make_msgid() + for k, v in munged_headers(mlist, omsg, msgdata): + msg[k] = v + # Are we including dmarc_wrapped_message_text? I.e., do we have text and + # are we wrapping because of dmarc_moderation_action? + if mlist.dmarc_wrapped_message_text and dmarc_wrap: + part1 = MIMEText(wrap(mlist.dmarc_wrapped_message_text), + 'plain', + mlist.preferred_language.charset) + part1['Content-Disposition'] = 'inline' + part2 = MIMEMessage(omsg) + part2['Content-Disposition'] = 'inline' + msg['Content-Type'] = 'multipart/mixed' + msg.set_payload([part1, part2]) + else: + msg['Content-Type'] = 'message/rfc822' + msg['Content-Disposition'] = 'inline' + msg.set_payload([omsg]) + return + + +def process(mlist, msg, msgdata): + """Process DMARC actions.""" + if ((not msgdata.get('dmarc') or + mlist.dmarc_moderation_action == DMARCModerationAction.none) and + mlist.from_is_list == FromIsList.none): + return + if mlist.anonymous_list: + # DMARC mitigation is not required for anonymous lists. + return + if (mlist.dmarc_moderation_action != DMARCModerationAction.none and + msgdata.get('dmarc')): + if mlist.dmarc_moderation_action == DMARCModerationAction.munge_from: + munge_from(mlist, msg, msgdata) + elif (mlist.dmarc_moderation_action == + DMARCModerationAction.wrap_message): + wrap_message(mlist, msg, msgdata, dmarc_wrap=True) + else: + assert False, ( + 'handlers/dmarc.py: dmarc_moderation_action = {0}'.format( + mlist.dmarc_moderation_action)) + else: + if mlist.from_is_list == FromIsList.munge_from: + munge_from(mlist, msg, msgdata) + elif mlist.from_is_list == FromIsList.wrap_message: + wrap_message(mlist, msg, msgdata) + else: + assert False, ( + 'handlers/dmarc.py: from_is_list = {0}'.format( + mlist.from_is_list)) + + +@public +@implementer(IHandler) +class DMARC: + """Apply DMARC mitigations.""" + + name = 'dmarc' + description = _('Apply DMARC mitigations.') + + def process(self, mlist, msg, msgdata): + """See `IHandler`.""" + process(mlist, msg, msgdata) diff --git a/src/mailman/handlers/docs/dmarc-mitigations.rst b/src/mailman/handlers/docs/dmarc-mitigations.rst new file mode 100644 index 000000000..ee7dd98e0 --- /dev/null +++ b/src/mailman/handlers/docs/dmarc-mitigations.rst @@ -0,0 +1,145 @@ +================= +DMARC Mitigations +================= + +In order to mitigate the effects of DMARC on mailing list traffic, list +administrators have the ability to apply transformations to messages delivered +to list members. These transformations are applied only to individual messages +sent to list members and not to messages in digests, archives or gated to +usenet. + +The messages can be transformed by either munging the From: header and putting +original From: in Cc: or Reply-To: or by wrapping the original message in an +outer message From: the list. + +Exactly what transformations are applied depends on a number of list settings. + +The settings and their effects are: + + * anonymous_list: If True, no mitigations are ever applied because the message + is already From: the list. + * dmarc_moderation_action: The action to apply to messages From: a domain + publishing a DMARC policy of reject and possibly quarantine or none. + * dmarc_quarantine_moderation_action: A flag to apply dmarc_moderation_action + to messages From: a domain publishing a DMARC policy of quarantine. + * dmarc_none_moderation_action: A flag to apply dmarc_moderation_action to + messages From: a domain publishing a DMARC policy of none, but only when + dmarc_quarantine_moderation_action is also true. + * dmarc_moderation_notice: Text to include in any rejection notice to be sent + when dmarc_moderation_action of reject applies. + * dmarc_wrapped_message_text: Text to be added as a separate text/plain MIME + part preceding the original message part in the wrapped message when + dmarc_moderation_action of wrap_message applies. + * from_is_list: The action to be applied to all messages for which + dmarc_moderation_action is none or not applicable. + * reply_goes_to_list: If this is set to other than no_munging of Reply-To, + the original From: goes in Cc: rather than Reply-To:. This is intended to + make MUA functions of reply and reply-all have the same effect with + messages to which mitigations have been applied as they do with other + messages. + +The possible actions for both dmarc_moderation_action and from_is_list are: + + * none: Make no transformation to the message. + * munge_from: Change the From: header and put the original From: in Reply-To: + or in some cases Cc: + * wrap_message: Wrap the message in an outer message with headers as in + munge_from. + +In addition, there are two more possible actions (actually processed by the +dmarc-moderation rule) for dmarc_moderation_action only: + + * reject: Bounce the message back to the sender with a default reason or one + supplied in dmarc_moderation_notice. + * discard: Silently discard the message. + +Here's what happens when we munge the From. + + >>> from mailman.interfaces.mailinglist import (DMARCModerationAction, + ... ReplyToMunging) + >>> mlist = create_list('test@example.com') + >>> mlist.dmarc_moderation_action = DMARCModerationAction.munge_from + >>> mlist.reply_goes_to_list = ReplyToMunging.no_munging + >>> msg = message_from_string("""\ + ... From: A Person <aperson@example.com> + ... To: test@example.com + ... + ... A message of great import. + ... """) + >>> msgdata = dict(dmarc=True, original_sender='aperson@example.com') + >>> from mailman.handlers.dmarc import process + >>> process(mlist, msg, msgdata) + >>> print(msg.as_string()) + To: test@example.com + From: A Person via Test <test@example.com> + Reply-To: A Person <aperson@example.com> + <BLANKLINE> + A message of great import. + <BLANKLINE> + +Here we wrap the message without adding a text part. + + >>> mlist.dmarc_moderation_action = DMARCModerationAction.wrap_message + >>> mlist.dmarc_wrapped_message_text = '' + >>> msg = message_from_string("""\ + ... From: A Person <aperson@example.com> + ... To: test@example.com + ... + ... A message of great import. + ... """) + >>> msgdata = dict(dmarc=True, original_sender='aperson@example.com') + >>> process(mlist, msg, msgdata) + >>> print(msg.as_string()) + To: test@example.com + MIME-Version: 1.0 + Message-ID: <...> + From: A Person via Test <test@example.com> + Reply-To: A Person <aperson@example.com> + Content-Type: message/rfc822 + Content-Disposition: inline + <BLANKLINE> + From: A Person <aperson@example.com> + To: test@example.com + <BLANKLINE> + A message of great import. + <BLANKLINE> + +And here's a wrapped message with an added text part. + + >>> mlist.dmarc_wrapped_message_text = 'The original message is attached.' + >>> msg = message_from_string("""\ + ... From: A Person <aperson@example.com> + ... To: test@example.com + ... + ... A message of great import. + ... """) + >>> msgdata = dict(dmarc=True, original_sender='aperson@example.com') + >>> process(mlist, msg, msgdata) + >>> print(msg.as_string()) + To: test@example.com + MIME-Version: 1.0 + Message-ID: <...> + From: A Person via Test <test@example.com> + Reply-To: A Person <aperson@example.com> + Content-Type: multipart/mixed; boundary="..." + <BLANKLINE> + --... + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: inline + <BLANKLINE> + The original message is attached. + --... + Content-Type: message/rfc822 + MIME-Version: 1.0 + Content-Disposition: inline + <BLANKLINE> + From: A Person <aperson@example.com> + To: test@example.com + <BLANKLINE> + A message of great import. + <BLANKLINE> + --...-- + <BLANKLINE> + diff --git a/src/mailman/handlers/tests/test_dmarc.py b/src/mailman/handlers/tests/test_dmarc.py new file mode 100644 index 000000000..648fbab0e --- /dev/null +++ b/src/mailman/handlers/tests/test_dmarc.py @@ -0,0 +1,244 @@ +# Copyright (C) 2016 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test the dmarc handler.""" + +import unittest + +from mailman.app.lifecycle import create_list +from mailman.handlers import dmarc +from mailman.interfaces.mailinglist import ( + DMARCModerationAction, FromIsList, ReplyToMunging) +from mailman.testing.helpers import specialized_message_from_string as mfs +from mailman.testing.layers import ConfigLayer + + +class TestDMARCMitigations(unittest.TestCase): + """Test the dmarc handler.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('ant@example.com') + self._mlist.anonymous_list = False + self._mlist.dmarc_moderation_action = DMARCModerationAction.none + self._mlist.dmarc_wrapped_message_text = '' + self._mlist.from_is_list = FromIsList.none + self._mlist.reply_goes_to_list = ReplyToMunging.no_munging + # We can use the same message text for all tests. + self._text = """\ +From: anne@example.com +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Message-ID: <some-id@example.com> +Date: Fri, 1 Jan 2016 00:00:01 +0000 +Another-Header: To test removal in wrapper +MIME-Version: 1.0 +Content-Type: multipart/alternative; boundary="=====abc==" + +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +Some things to say. +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +<html><head></head><body>Some things to say.</body></html> +--=====abc==-- +""" + + def test_default_no_change(self): + msg = mfs(self._text) + dmarc.process(self._mlist, msg, {}) + self.assertMultiLineEqual(msg.as_string(), self._text) + + def test_anonymous_no_change(self): + self._mlist.anonymous_list = True + self._mlist.dmarc_moderation_action = DMARCModerationAction.munge_from + msgdata = {'dmarc': True} + msg = mfs(self._text) + dmarc.process(self._mlist, msg, msgdata) + self.assertMultiLineEqual(msg.as_string(), self._text) + + def test_action_munge_from(self): + self._mlist.dmarc_moderation_action = DMARCModerationAction.munge_from + msgdata = {'dmarc': True} + msg = mfs(self._text) + dmarc.process(self._mlist, msg, msgdata) + self.assertMultiLineEqual(msg.as_string(), """\ +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Message-ID: <some-id@example.com> +Date: Fri, 1 Jan 2016 00:00:01 +0000 +Another-Header: To test removal in wrapper +MIME-Version: 1.0 +Content-Type: multipart/alternative; boundary="=====abc==" +From: anne--- via Ant <ant@example.com> +Reply-To: anne@example.com + +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +Some things to say. +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +<html><head></head><body>Some things to say.</body></html> +--=====abc==-- +""") + + def test_no_action_without_msgdata(self): + self._mlist.dmarc_moderation_action = DMARCModerationAction.munge_from + msg = mfs(self._text) + dmarc.process(self._mlist, msg, {}) + self.assertMultiLineEqual(msg.as_string(), self._text) + + def test_from_is_list_no_msgdata(self): + self._mlist.from_is_list = FromIsList.munge_from + msg = mfs(self._text) + dmarc.process(self._mlist, msg, {}) + self.assertMultiLineEqual(msg.as_string(), """\ +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Message-ID: <some-id@example.com> +Date: Fri, 1 Jan 2016 00:00:01 +0000 +Another-Header: To test removal in wrapper +MIME-Version: 1.0 +Content-Type: multipart/alternative; boundary="=====abc==" +From: anne--- via Ant <ant@example.com> +Reply-To: anne@example.com + +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +Some things to say. +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +<html><head></head><body>Some things to say.</body></html> +--=====abc==-- +""") + + def test_from_in_cc(self): + self._mlist.dmarc_moderation_action = DMARCModerationAction.munge_from + self._mlist.reply_goes_to_list = ReplyToMunging.point_to_list + msgdata = {'dmarc': True} + msg = mfs(self._text) + dmarc.process(self._mlist, msg, msgdata) + self.assertMultiLineEqual(msg.as_string(), """\ +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Message-ID: <some-id@example.com> +Date: Fri, 1 Jan 2016 00:00:01 +0000 +Another-Header: To test removal in wrapper +MIME-Version: 1.0 +Content-Type: multipart/alternative; boundary="=====abc==" +From: anne--- via Ant <ant@example.com> +Cc: anne@example.com + +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +Some things to say. +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +<html><head></head><body>Some things to say.</body></html> +--=====abc==-- +""") + + def test_wrap_message_1(self): + self._mlist.from_is_list = FromIsList.wrap_message + msg = mfs(self._text) + dmarc.process(self._mlist, msg, {}) + # We can't predict the Message-ID in the wrapper so delete it, but + # ensure we have one. + self.assertIsNotNone(msg.get('message-id')) + del msg['message-id'] + self.assertMultiLineEqual(msg.as_string(), """\ +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Date: Fri, 1 Jan 2016 00:00:01 +0000 +MIME-Version: 1.0 +From: anne--- via Ant <ant@example.com> +Reply-To: anne@example.com +Content-Type: message/rfc822 +Content-Disposition: inline + +""" + self._text) + + def test_wrap_message_2(self): + self._mlist.dmarc_moderation_action = ( + DMARCModerationAction.wrap_message) + msgdata = {'dmarc': True} + msg = mfs(self._text) + dmarc.process(self._mlist, msg, msgdata) + # We can't predict the Message-ID in the wrapper so delete it, but + # ensure we have one. + self.assertIsNotNone(msg.get('message-id')) + del msg['message-id'] + self.maxDiff = None + self.assertMultiLineEqual(msg.as_string(), """\ +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Date: Fri, 1 Jan 2016 00:00:01 +0000 +MIME-Version: 1.0 +From: anne--- via Ant <ant@example.com> +Reply-To: anne@example.com +Content-Type: message/rfc822 +Content-Disposition: inline + +""" + self._text) + + def test_wrap_message_cc(self): + self._mlist.dmarc_moderation_action = ( + DMARCModerationAction.wrap_message) + self._mlist.reply_goes_to_list = ReplyToMunging.point_to_list + msgdata = {'dmarc': True} + msg = mfs(self._text) + dmarc.process(self._mlist, msg, msgdata) + # We can't predict the Message-ID in the wrapper so delete it, but + # ensure we have one. + self.assertIsNotNone(msg.get('message-id')) + del msg['message-id'] + self.maxDiff = None + self.assertMultiLineEqual(msg.as_string(), """\ +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Date: Fri, 1 Jan 2016 00:00:01 +0000 +MIME-Version: 1.0 +From: anne--- via Ant <ant@example.com> +Cc: anne@example.com +Content-Type: message/rfc822 +Content-Disposition: inline + +""" + self._text) |