3 files changed, 596 insertions, 0 deletions

diff --git a/src/mailman/handlers/dmarc.py b/src/mailman/handlers/dmarc.py
new file mode 100644
index 000000000..9b8f1bd30
--- /dev/null
+++ b/src/mailman/handlers/dmarc.py
@@ -0,0 +1,207 @@
+# Copyright (C) 2016 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Do DMARC Munge From and Wrap Message actions."""
+
+import re
+import copy
+import logging
+
+from email.header import Header
+from email.mime.message import MIMEMessage
+from email.mime.text import MIMEText
+from email.utils import formataddr, getaddresses, make_msgid
+from mailman import public
+from mailman.core.i18n import _
+from mailman.interfaces.handler import IHandler
+from mailman.interfaces.mailinglist import (
+    DMARCModerationAction, FromIsList, ReplyToMunging)
+from mailman.utilities.string import wrap
+from zope.interface import implementer
+
+
+log = logging.getLogger('mailman.error')
+
+COMMASPACE = ', '
+MAXLINELEN = 78
+NONASCII = re.compile('[^\s!-~]')
+# Headers from the original that we want to keep in the wrapper.  These are
+# actually regexps matched with re.match so they match anything that starts
+# with the given string unless they end with '$'.
+KEEPERS = ('archived-at',
+           'date',
+           'in-reply-to',
+           'list-',
+           'precedence',
+           'references',
+           'subject',
+           'to',
+           'x-mailman-',
+           )
+
+
+@public
+def uheader(mlist, s, header_name=None, continuation_ws='\t', maxlinelen=None):
+    """Get the charset to encode the string in.
+
+    Then search if there is any non-ascii character is in the string.  If
+    there is and the charset is us-ascii then we use iso-8859-1 instead.  If
+    the string is ascii only we use 'us-ascii' if another charset is
+    specified.
+
+    If the header contains a newline, truncate it (see GL#273).
+    """
+    charset = mlist.preferred_language.charset
+    if NONASCII.search(s):
+        # use list charset but ...
+        if charset == 'us-ascii':
+            charset = 'iso-8859-1'
+    else:
+        # there is no non-ascii so ...
+        charset = 'us-ascii'
+    if '\n' in s:
+        s = '{} [...]'.format(s.split('\n')[0])
+        log.warning('Header {} contains a newline, truncating it.'.format(
+            header_name, s))
+    return Header(s, charset, maxlinelen, header_name, continuation_ws)
+
+
+def munged_headers(mlist, msg, msgdata):
+    # Be as robust as possible here.
+    faddrs = getaddresses(msg.get_all('from', []))
+    # Strip the nulls and bad emails.
+    faddrs = [x for x in faddrs if x[1].find('@') > 0]
+    if len(faddrs) == 1:
+        realname, email = o_from = faddrs[0]
+    else:
+        # No From: or multiple addresses.  Just punt and take
+        # the get_sender result.
+        realname = ''
+        email = msgdata['original_sender']
+        o_from = (realname, email)
+    if not realname:
+        member = mlist.members.get_member(email)
+        if member:
+            realname = member.display_name or email
+        else:
+            realname = email
+    # Remove domain from realname if it looks like an email address
+    realname = re.sub(r'@([^ .]+\.)+[^ .]+$', '---', realname)
+    # RFC 2047 encode realname if necessary.
+    realname = str(uheader(mlist, realname))
+    lrn = mlist.display_name                         # noqa F841
+    retn = [('From', formataddr((_('$realname via $lrn'),
+                                mlist.posting_address)))]
+    # We've made the munged From:.  Now put the original in Reply-To: or Cc:
+    if mlist.reply_goes_to_list == ReplyToMunging.no_munging:
+        # Add original from to Reply-To:
+        add_to = 'Reply-To'
+    else:
+        # Add original from to Cc:
+        add_to = 'Cc'
+    orig = getaddresses(msg.get_all(add_to, []))
+    if o_from[1] not in [x[1] for x in orig]:
+        orig.append(o_from)
+    retn.append((add_to, COMMASPACE.join([formataddr(x) for x in orig])))
+    return retn
+
+
+def munge_from(mlist, msg, msgdata):
+    for k, v in munged_headers(mlist, msg, msgdata):
+        del msg[k]
+        msg[k] = v
+    return
+
+
+def wrap_message(mlist, msg, msgdata, dmarc_wrap=False):
+    # Create a wrapper message around the original.
+    # There are various headers in msg that we don't want, so we basically
+    # make a copy of the msg, then delete almost everything and set/copy
+    # what we want.
+    omsg = copy.deepcopy(msg)
+    for key in msg.keys():
+        keep = False
+        for keeper in KEEPERS:
+            if re.match(keeper, key, re.I):
+                keep = True
+                break
+        if not keep:
+            del msg[key]
+    msg['MIME-Version'] = '1.0'
+    msg['Message-ID'] = make_msgid()
+    for k, v in munged_headers(mlist, omsg, msgdata):
+        msg[k] = v
+    # Are we including dmarc_wrapped_message_text?  I.e., do we have text and
+    # are we wrapping because of dmarc_moderation_action?
+    if mlist.dmarc_wrapped_message_text and dmarc_wrap:
+        part1 = MIMEText(wrap(mlist.dmarc_wrapped_message_text),
+                         'plain',
+                         mlist.preferred_language.charset)
+        part1['Content-Disposition'] = 'inline'
+        part2 = MIMEMessage(omsg)
+        part2['Content-Disposition'] = 'inline'
+        msg['Content-Type'] = 'multipart/mixed'
+        msg.set_payload([part1, part2])
+    else:
+        msg['Content-Type'] = 'message/rfc822'
+        msg['Content-Disposition'] = 'inline'
+        msg.set_payload([omsg])
+    return
+
+
+def process(mlist, msg, msgdata):
+    """Process DMARC actions."""
+    if ((not msgdata.get('dmarc') or
+            mlist.dmarc_moderation_action == DMARCModerationAction.none) and
+            mlist.from_is_list == FromIsList.none):
+        return
+    if mlist.anonymous_list:
+        # DMARC mitigation is not required for anonymous lists.
+        return
+    if (mlist.dmarc_moderation_action != DMARCModerationAction.none and
+            msgdata.get('dmarc')):
+        if mlist.dmarc_moderation_action == DMARCModerationAction.munge_from:
+            munge_from(mlist, msg, msgdata)
+        elif (mlist.dmarc_moderation_action ==
+                DMARCModerationAction.wrap_message):
+            wrap_message(mlist, msg, msgdata, dmarc_wrap=True)
+        else:
+            assert False, (
+                'handlers/dmarc.py: dmarc_moderation_action = {0}'.format(
+                    mlist.dmarc_moderation_action))
+    else:
+        if mlist.from_is_list == FromIsList.munge_from:
+            munge_from(mlist, msg, msgdata)
+        elif mlist.from_is_list == FromIsList.wrap_message:
+            wrap_message(mlist, msg, msgdata)
+        else:
+            assert False, (
+                'handlers/dmarc.py: from_is_list = {0}'.format(
+                    mlist.from_is_list))
+
+
+@public
+@implementer(IHandler)
+class DMARC:
+    """Apply DMARC mitigations."""
+
+    name = 'dmarc'
+    description = _('Apply DMARC mitigations.')
+
+    def process(self, mlist, msg, msgdata):
+        """See `IHandler`."""
+        process(mlist, msg, msgdata)
diff --git a/src/mailman/handlers/docs/dmarc-mitigations.rst b/src/mailman/handlers/docs/dmarc-mitigations.rst
new file mode 100644
index 000000000..ee7dd98e0
--- /dev/null
+++ b/src/mailman/handlers/docs/dmarc-mitigations.rst
@@ -0,0 +1,145 @@
+=================
+DMARC Mitigations
+=================
+
+In order to mitigate the effects of DMARC on mailing list traffic, list
+administrators have the ability to apply transformations to messages delivered
+to list members.  These transformations are applied only to individual messages
+sent to list members and not to messages in digests, archives or gated to
+usenet.
+
+The messages can be transformed by either munging the From: header and putting
+original From: in Cc: or Reply-To: or by wrapping the original message in an
+outer message From: the list.
+
+Exactly what transformations are applied depends on a number of list settings.
+
+The settings and their effects are:
+
+ * anonymous_list: If True, no mitigations are ever applied because the message
+   is already From: the list.
+ * dmarc_moderation_action: The action to apply to messages From: a domain
+   publishing a DMARC policy of reject and possibly quarantine or none.
+ * dmarc_quarantine_moderation_action: A flag to apply dmarc_moderation_action
+   to messages From: a domain publishing a DMARC policy of quarantine.
+ * dmarc_none_moderation_action: A flag to apply dmarc_moderation_action to
+   messages From: a domain publishing a DMARC policy of none, but only when
+   dmarc_quarantine_moderation_action is also true.
+ * dmarc_moderation_notice: Text to include in any rejection notice to be sent
+   when dmarc_moderation_action of reject applies.
+ * dmarc_wrapped_message_text: Text to be added as a separate text/plain MIME
+   part preceding the original message part in the wrapped message when
+   dmarc_moderation_action of wrap_message applies.
+ * from_is_list: The action to be applied to all messages for which
+   dmarc_moderation_action is none or not applicable.
+ * reply_goes_to_list: If this is set to other than no_munging of Reply-To,
+   the original From: goes in Cc: rather than Reply-To:.  This is intended to
+   make MUA functions of reply and reply-all have the same effect with
+   messages to which mitigations have been applied as they do with other
+   messages.
+
+The possible actions for both dmarc_moderation_action and from_is_list are:
+
+ * none: Make no transformation to the message.
+ * munge_from: Change the From: header and put the original From: in Reply-To:
+   or in some cases Cc:
+ * wrap_message: Wrap the message in an outer message with headers as in
+   munge_from.
+
+In addition, there are two more possible actions (actually processed by the
+dmarc-moderation rule) for dmarc_moderation_action only:
+
+ * reject: Bounce the message back to the sender with a default reason or one
+   supplied in dmarc_moderation_notice.
+ * discard: Silently discard the message.
+
+Here's what happens when we munge the From.
+
+    >>> from mailman.interfaces.mailinglist import (DMARCModerationAction,
+    ... ReplyToMunging)
+    >>> mlist = create_list('test@example.com')
+    >>> mlist.dmarc_moderation_action = DMARCModerationAction.munge_from
+    >>> mlist.reply_goes_to_list = ReplyToMunging.no_munging
+    >>> msg = message_from_string("""\
+    ... From: A Person <aperson@example.com>
+    ... To: test@example.com
+    ...
+    ... A message of great import.
+    ... """)
+    >>> msgdata = dict(dmarc=True, original_sender='aperson@example.com')
+    >>> from mailman.handlers.dmarc import process
+    >>> process(mlist, msg, msgdata)
+    >>> print(msg.as_string())
+    To: test@example.com
+    From: A Person via Test <test@example.com>
+    Reply-To: A Person <aperson@example.com>
+    <BLANKLINE>
+    A message of great import.
+    <BLANKLINE>
+    
+Here we wrap the message without adding a text part.
+
+    >>> mlist.dmarc_moderation_action = DMARCModerationAction.wrap_message
+    >>> mlist.dmarc_wrapped_message_text = ''
+    >>> msg = message_from_string("""\
+    ... From: A Person <aperson@example.com>
+    ... To: test@example.com
+    ...
+    ... A message of great import.
+    ... """)
+    >>> msgdata = dict(dmarc=True, original_sender='aperson@example.com')
+    >>> process(mlist, msg, msgdata)
+    >>> print(msg.as_string())
+    To: test@example.com
+    MIME-Version: 1.0
+    Message-ID: <...>
+    From: A Person via Test <test@example.com>
+    Reply-To: A Person <aperson@example.com>
+    Content-Type: message/rfc822
+    Content-Disposition: inline
+    <BLANKLINE>
+    From: A Person <aperson@example.com>
+    To: test@example.com
+    <BLANKLINE>
+    A message of great import.
+    <BLANKLINE>
+
+And here's a wrapped message with an added text part.
+
+    >>> mlist.dmarc_wrapped_message_text = 'The original message is attached.'
+    >>> msg = message_from_string("""\
+    ... From: A Person <aperson@example.com>
+    ... To: test@example.com
+    ...
+    ... A message of great import.
+    ... """)
+    >>> msgdata = dict(dmarc=True, original_sender='aperson@example.com')
+    >>> process(mlist, msg, msgdata)
+    >>> print(msg.as_string())
+    To: test@example.com
+    MIME-Version: 1.0
+    Message-ID: <...>
+    From: A Person via Test <test@example.com>
+    Reply-To: A Person <aperson@example.com>
+    Content-Type: multipart/mixed; boundary="..."
+    <BLANKLINE>
+    --...
+    Content-Type: text/plain; charset="us-ascii"
+    MIME-Version: 1.0
+    Content-Transfer-Encoding: 7bit
+    Content-Disposition: inline
+    <BLANKLINE>
+    The original message is attached.
+    --...
+    Content-Type: message/rfc822
+    MIME-Version: 1.0
+    Content-Disposition: inline
+    <BLANKLINE>
+    From: A Person <aperson@example.com>
+    To: test@example.com
+    <BLANKLINE>
+    A message of great import.
+    <BLANKLINE>
+    --...--
+    <BLANKLINE>
+
diff --git a/src/mailman/handlers/tests/test_dmarc.py b/src/mailman/handlers/tests/test_dmarc.py
new file mode 100644
index 000000000..648fbab0e
--- /dev/null
+++ b/src/mailman/handlers/tests/test_dmarc.py
@@ -0,0 +1,244 @@
+# Copyright (C) 2016 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Test the dmarc handler."""
+
+import unittest
+
+from mailman.app.lifecycle import create_list
+from mailman.handlers import dmarc
+from mailman.interfaces.mailinglist import (
+    DMARCModerationAction, FromIsList, ReplyToMunging)
+from mailman.testing.helpers import specialized_message_from_string as mfs
+from mailman.testing.layers import ConfigLayer
+
+
+class TestDMARCMitigations(unittest.TestCase):
+    """Test the dmarc handler."""
+
+    layer = ConfigLayer
+
+    def setUp(self):
+        self._mlist = create_list('ant@example.com')
+        self._mlist.anonymous_list = False
+        self._mlist.dmarc_moderation_action = DMARCModerationAction.none
+        self._mlist.dmarc_wrapped_message_text = ''
+        self._mlist.from_is_list = FromIsList.none
+        self._mlist.reply_goes_to_list = ReplyToMunging.no_munging
+        # We can use the same message text for all tests.
+        self._text = """\
+From: anne@example.com
+To: ant@example.com
+Subject: A subject
+X-Mailman-Version: X.Y
+Message-ID: <some-id@example.com>
+Date: Fri, 1 Jan 2016 00:00:01 +0000
+Another-Header: To test removal in wrapper
+MIME-Version: 1.0
+Content-Type: multipart/alternative; boundary="=====abc=="
+
+--=====abc==
+Content-Type: text/plain; charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+
+Some things to say.
+--=====abc==
+Content-Type: text/plain; charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+
+<html><head></head><body>Some things to say.</body></html>
+--=====abc==--
+"""
+
+    def test_default_no_change(self):
+        msg = mfs(self._text)
+        dmarc.process(self._mlist, msg, {})
+        self.assertMultiLineEqual(msg.as_string(), self._text)
+
+    def test_anonymous_no_change(self):
+        self._mlist.anonymous_list = True
+        self._mlist.dmarc_moderation_action = DMARCModerationAction.munge_from
+        msgdata = {'dmarc': True}
+        msg = mfs(self._text)
+        dmarc.process(self._mlist, msg, msgdata)
+        self.assertMultiLineEqual(msg.as_string(), self._text)
+
+    def test_action_munge_from(self):
+        self._mlist.dmarc_moderation_action = DMARCModerationAction.munge_from
+        msgdata = {'dmarc': True}
+        msg = mfs(self._text)
+        dmarc.process(self._mlist, msg, msgdata)
+        self.assertMultiLineEqual(msg.as_string(), """\
+To: ant@example.com
+Subject: A subject
+X-Mailman-Version: X.Y
+Message-ID: <some-id@example.com>
+Date: Fri, 1 Jan 2016 00:00:01 +0000
+Another-Header: To test removal in wrapper
+MIME-Version: 1.0
+Content-Type: multipart/alternative; boundary="=====abc=="
+From: anne--- via Ant <ant@example.com>
+Reply-To: anne@example.com
+
+--=====abc==
+Content-Type: text/plain; charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+
+Some things to say.
+--=====abc==
+Content-Type: text/plain; charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+
+<html><head></head><body>Some things to say.</body></html>
+--=====abc==--
+""")
+
+    def test_no_action_without_msgdata(self):
+        self._mlist.dmarc_moderation_action = DMARCModerationAction.munge_from
+        msg = mfs(self._text)
+        dmarc.process(self._mlist, msg, {})
+        self.assertMultiLineEqual(msg.as_string(), self._text)
+
+    def test_from_is_list_no_msgdata(self):
+        self._mlist.from_is_list = FromIsList.munge_from
+        msg = mfs(self._text)
+        dmarc.process(self._mlist, msg, {})
+        self.assertMultiLineEqual(msg.as_string(), """\
+To: ant@example.com
+Subject: A subject
+X-Mailman-Version: X.Y
+Message-ID: <some-id@example.com>
+Date: Fri, 1 Jan 2016 00:00:01 +0000
+Another-Header: To test removal in wrapper
+MIME-Version: 1.0
+Content-Type: multipart/alternative; boundary="=====abc=="
+From: anne--- via Ant <ant@example.com>
+Reply-To: anne@example.com
+
+--=====abc==
+Content-Type: text/plain; charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+
+Some things to say.
+--=====abc==
+Content-Type: text/plain; charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+
+<html><head></head><body>Some things to say.</body></html>
+--=====abc==--
+""")
+
+    def test_from_in_cc(self):
+        self._mlist.dmarc_moderation_action = DMARCModerationAction.munge_from
+        self._mlist.reply_goes_to_list = ReplyToMunging.point_to_list
+        msgdata = {'dmarc': True}
+        msg = mfs(self._text)
+        dmarc.process(self._mlist, msg, msgdata)
+        self.assertMultiLineEqual(msg.as_string(), """\
+To: ant@example.com
+Subject: A subject
+X-Mailman-Version: X.Y
+Message-ID: <some-id@example.com>
+Date: Fri, 1 Jan 2016 00:00:01 +0000
+Another-Header: To test removal in wrapper
+MIME-Version: 1.0
+Content-Type: multipart/alternative; boundary="=====abc=="
+From: anne--- via Ant <ant@example.com>
+Cc: anne@example.com
+
+--=====abc==
+Content-Type: text/plain; charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+
+Some things to say.
+--=====abc==
+Content-Type: text/plain; charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+
+<html><head></head><body>Some things to say.</body></html>
+--=====abc==--
+""")
+
+    def test_wrap_message_1(self):
+        self._mlist.from_is_list = FromIsList.wrap_message
+        msg = mfs(self._text)
+        dmarc.process(self._mlist, msg, {})
+        # We can't predict the Message-ID in the wrapper so delete it, but
+        # ensure we have one.
+        self.assertIsNotNone(msg.get('message-id'))
+        del msg['message-id']
+        self.assertMultiLineEqual(msg.as_string(), """\
+To: ant@example.com
+Subject: A subject
+X-Mailman-Version: X.Y
+Date: Fri, 1 Jan 2016 00:00:01 +0000
+MIME-Version: 1.0
+From: anne--- via Ant <ant@example.com>
+Reply-To: anne@example.com
+Content-Type: message/rfc822
+Content-Disposition: inline
+
+""" + self._text)
+
+    def test_wrap_message_2(self):
+        self._mlist.dmarc_moderation_action = (
+            DMARCModerationAction.wrap_message)
+        msgdata = {'dmarc': True}
+        msg = mfs(self._text)
+        dmarc.process(self._mlist, msg, msgdata)
+        # We can't predict the Message-ID in the wrapper so delete it, but
+        # ensure we have one.
+        self.assertIsNotNone(msg.get('message-id'))
+        del msg['message-id']
+        self.maxDiff = None
+        self.assertMultiLineEqual(msg.as_string(), """\
+To: ant@example.com
+Subject: A subject
+X-Mailman-Version: X.Y
+Date: Fri, 1 Jan 2016 00:00:01 +0000
+MIME-Version: 1.0
+From: anne--- via Ant <ant@example.com>
+Reply-To: anne@example.com
+Content-Type: message/rfc822
+Content-Disposition: inline
+
+""" + self._text)
+
+    def test_wrap_message_cc(self):
+        self._mlist.dmarc_moderation_action = (
+            DMARCModerationAction.wrap_message)
+        self._mlist.reply_goes_to_list = ReplyToMunging.point_to_list
+        msgdata = {'dmarc': True}
+        msg = mfs(self._text)
+        dmarc.process(self._mlist, msg, msgdata)
+        # We can't predict the Message-ID in the wrapper so delete it, but
+        # ensure we have one.
+        self.assertIsNotNone(msg.get('message-id'))
+        del msg['message-id']
+        self.maxDiff = None
+        self.assertMultiLineEqual(msg.as_string(), """\
+To: ant@example.com
+Subject: A subject
+X-Mailman-Version: X.Y
+Date: Fri, 1 Jan 2016 00:00:01 +0000
+MIME-Version: 1.0
+From: anne--- via Ant <ant@example.com>
+Cc: anne@example.com
+Content-Type: message/rfc822
+Content-Disposition: inline
+
+""" + self._text)