diff options
24 files changed, 1545 insertions, 17 deletions
@@ -106,6 +106,7 @@ case second `m'. Any other spelling is incorrect.""", 'aiosmtpd', 'alembic', 'atpublic', + 'dnspython>=1.14.0', 'falcon>=1.0.0rc1', 'flufl.bounce', 'flufl.i18n', diff --git a/src/mailman/chains/builtin.py b/src/mailman/chains/builtin.py index 16de1043a..e26c38470 100644 --- a/src/mailman/chains/builtin.py +++ b/src/mailman/chains/builtin.py @@ -38,6 +38,10 @@ class BuiltInChain: description = _('The built-in moderation chain.') _link_descriptions = ( + # First check DMARC. For a reject or discard, the rule hits and we + # jump to the moderation chain to do the action. Otherwise, the rule + # misses buts sets msgdata['dmarc'] for the handler. + ('dmarc-mitigation', LinkAction.jump, 'moderation'), ('approved', LinkAction.jump, 'accept'), ('emergency', LinkAction.jump, 'hold'), ('loop', LinkAction.jump, 'discard'), diff --git a/src/mailman/chains/docs/moderation.rst b/src/mailman/chains/docs/moderation.rst index 3b40c7293..e3c9e12f5 100644 --- a/src/mailman/chains/docs/moderation.rst +++ b/src/mailman/chains/docs/moderation.rst @@ -87,6 +87,7 @@ built-in chain. No rules hit and so the message is accepted. Subject: aardvark Hits: Misses: + dmarc-mitigation approved emergency loop @@ -124,6 +125,7 @@ moderator approval. Hits: member-moderation Misses: + dmarc-mitigation approved emergency loop @@ -150,6 +152,7 @@ Anne's moderation action can also be set to `discard`... Hits: member-moderation Misses: + dmarc-mitigation approved emergency loop @@ -175,6 +178,7 @@ Anne's moderation action can also be set to `discard`... Hits: member-moderation Misses: + dmarc-mitigation approved emergency loop @@ -215,6 +219,7 @@ moderator approval. Hits: nonmember-moderation Misses: + dmarc-mitigation approved emergency loop diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg index e3ddb6f8e..fdf4c2b6f 100644 --- a/src/mailman/config/schema.cfg +++ b/src/mailman/config/schema.cfg @@ -74,6 +74,22 @@ filtered_messages_are_preservable: no # The command should print the converted text to stdout. html_to_plain_text_command: /usr/bin/lynx -dump $filename +# Parameters for DMARC DNS lookups. If you are seeing 'DNSException: Unable +# to query DMARC policy ...' entries in your error log, you may need to adjust +# these. +# +# The time to wait for a response from a name server before timeout. +dmarc_resolver_timeout: 3s +# The total time to spend trying to get an answer to the DNS question. +dmarc_resolver_lifetime: 5s + +# A URL from which to retrieve the data for the algorithm that computes +# Organizational Domains for DMARC policy lookup purposes. This can be +# anything handled by the Python urllib.request.urlopen function. See +# https://publicsuffix.org/list/ for info. +dmarc_org_domain_data: https://publicsuffix.org/list/public_suffix_list.dat + + [shell] # `mailman shell` (also `withlist`) gives you an interactive prompt that you diff --git a/src/mailman/core/docs/chains.rst b/src/mailman/core/docs/chains.rst index 2b30a25e4..e3190e364 100644 --- a/src/mailman/core/docs/chains.rst +++ b/src/mailman/core/docs/chains.rst @@ -268,9 +268,9 @@ This message will end up in the `pipeline` queue. Message-ID: <first> Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB - X-Mailman-Rule-Misses: approved; emergency; loop; banned-address; - member-moderation; nonmember-moderation; administrivia; implicit-dest; - max-recipients; max-size; news-moderation; no-subject; + X-Mailman-Rule-Misses: dmarc-mitigation; approved; emergency; loop; + banned-address; member-moderation; nonmember-moderation; administrivia; + implicit-dest; max-recipients; max-size; news-moderation; no-subject; suspicious-header <BLANKLINE> An important message. @@ -285,6 +285,7 @@ hit and all rules that have missed. administrivia approved banned-address + dmarc-mitigation emergency implicit-dest loop diff --git a/src/mailman/core/pipelines.py b/src/mailman/core/pipelines.py index fb663cee9..7b4c6feb4 100644 --- a/src/mailman/core/pipelines.py +++ b/src/mailman/core/pipelines.py @@ -115,6 +115,7 @@ class PostingPipeline(BasePipeline): 'after-delivery', 'acknowledge', 'decorate', + 'dmarc', 'to-outgoing', ) diff --git a/src/mailman/database/alembic/versions/3002bac0c25a_dmarc_attributes.py b/src/mailman/database/alembic/versions/3002bac0c25a_dmarc_attributes.py new file mode 100644 index 000000000..aa2d8610d --- /dev/null +++ b/src/mailman/database/alembic/versions/3002bac0c25a_dmarc_attributes.py @@ -0,0 +1,70 @@ +"""dmarc_attributes + +Revision ID: 3002bac0c25a +Revises: a46993b05703 +Create Date: 2016-10-30 22:05:17.881880 + +""" + +import sqlalchemy as sa + +from alembic import op +from mailman.database.helpers import exists_in_db +from mailman.database.types import Enum, SAUnicodeLarge +from mailman.interfaces.mailinglist import DMARCMitigateAction + + +# revision identifiers, used by Alembic. +revision = '3002bac0c25a' +down_revision = 'a46993b05703' + + +def upgrade(): + if not exists_in_db(op.get_bind(), + 'mailinglist', + 'dmarc_mitigate_action' + ): + # SQLite may not have removed it when downgrading. It should be OK + # to just test one. + op.add_column('mailinglist', sa.Column( + 'dmarc_mitigate_action', + Enum(DMARCMitigateAction), + nullable=True)) + op.add_column('mailinglist', sa.Column( + 'dmarc_mitigate_unconditionally', + sa.Boolean, + nullable=True)) + op.add_column('mailinglist', sa.Column( + 'dmarc_moderation_notice', + SAUnicodeLarge(), + nullable=True)) + op.add_column('mailinglist', sa.Column( + 'dmarc_wrapped_message_text', + SAUnicodeLarge(), + nullable=True)) + # Now migrate the data. Don't import the table definition from the + # models, it may break this migration when the model is updated in the + # future (see the Alembic doc). + mlist = sa.sql.table( + 'mailinglist', + sa.sql.column('dmarc_mitigate_action', Enum(DMARCMitigateAction)), + sa.sql.column('dmarc_mitigate_unconditionally', sa.Boolean), + sa.sql.column('dmarc_moderation_notice', SAUnicodeLarge()), + sa.sql.column('dmarc_wrapped_message_text', SAUnicodeLarge()), + ) + # These are all new attributes so just set defaults. + op.execute(mlist.update().values(dict( + dmarc_mitigate_action=op.inline_literal( + DMARCMitigateAction.no_mitigation), + dmarc_mitigate_unconditionally=op.inline_literal(False), + dmarc_moderation_notice=op.inline_literal(''), + dmarc_wrapped_message_text=op.inline_literal(''), + ))) + + +def downgrade(): + with op.batch_alter_table('mailinglist') as batch_op: + batch_op.drop_column('dmarc_mitigate_action') + batch_op.drop_column('dmarc_mitigate_unconditionally') + batch_op.drop_column('dmarc_moderation_notice') + batch_op.drop_column('dmarc_wrapped_message_text') diff --git a/src/mailman/handlers/cook_headers.py b/src/mailman/handlers/cook_headers.py index 672738bf9..413304ea2 100644 --- a/src/mailman/handlers/cook_headers.py +++ b/src/mailman/handlers/cook_headers.py @@ -114,7 +114,7 @@ def process(mlist, msg, msgdata): d[lcaddr] = pair new.append(pair) # List admin wants an explicit Reply-To: added - if mlist.reply_goes_to_list == ReplyToMunging.explicit_header: + if mlist.reply_goes_to_list is ReplyToMunging.explicit_header: add(parseaddr(mlist.reply_to_address)) # If we're not first stripping existing Reply-To: then we need to add # the original Reply-To:'s to the list we're building up. In both diff --git a/src/mailman/handlers/dmarc.py b/src/mailman/handlers/dmarc.py new file mode 100644 index 000000000..99d8e08df --- /dev/null +++ b/src/mailman/handlers/dmarc.py @@ -0,0 +1,221 @@ +# Copyright (C) 2016 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Do DMARC Munge From and Wrap Message actions. + +This does the work of modifying the messages From: and Cc: or Reply-To: or +wrapping the message in an outer message with From: and Cc: or Reply-To: +as appropriate to avoid issues because of the original From: domain's DMARC +policy. It does this either to selected messages flagged by the DMARC +moderation rule based on list settings and the original From: domain's DMARC +policy or to all messages based on list settings.""" + +import re +import copy +import logging + +from email.header import Header, decode_header +from email.mime.message import MIMEMessage +from email.mime.text import MIMEText +from email.utils import formataddr, getaddresses, make_msgid +from mailman.core.i18n import _ +from mailman.interfaces.handler import IHandler +from mailman.interfaces.mailinglist import DMARCMitigateAction, ReplyToMunging +from mailman.utilities.string import wrap +from public import public +from zope.interface import implementer + + +log = logging.getLogger('mailman.error') + +COMMASPACE = ', ' +MAXLINELEN = 78 +NONASCII = re.compile('[^\s!-~]') +# Headers from the original that we want to keep in the wrapper. These are +# actually regexps matched with re.match so they match anything that starts +# with the given string unless they end with '$'. +KEEPERS = ( + 'archived-at', + 'date', + 'in-reply-to', + 'list-', + 'precedence', + 'references', + 'subject', + 'to', + 'x-mailman-', + ) + + +def munged_headers(mlist, msg, msgdata): + # This returns a list of tuples (header, content) where header is the + # name of a header to be added to or replaced in the wrapper or message + # for DMARC mitigation. It sets From: to the string + # 'original From: display name' via 'list name' <list posting address> + # and adds the original From: to Reply-To: or Cc: per the following. + # Our goals for this process are not completely compatible, so we do + # the best we can. Our goals are: + # 1) as long as the list is not anonymous, the original From: address + # should be obviously exposed, i.e. not just in a header that MUAs + # don't display. + # 2) the original From: address should not be in a comment or display + # name in the new From: because it is claimed that multiple domains + # in any fields in From: are indicative of spamminess. This means + # it should be in Reply-To: or Cc:. + # 3) the behavior of an MUA doing a 'reply' or 'reply all' should be + # consistent regardless of whether or not the From: is munged. + # Goal 3) implies sometimes the original From: should be in Reply-To: + # and sometimes in Cc:, and even so, this goal won't be achieved in + # all cases with all MUAs. In cases of conflict, the above ordering of + # goals is priority order. + # + # Be as robust as possible here. + faddrs = getaddresses(msg.get_all('from', [])) + # Strip the nulls and bad emails. + faddrs = [x for x in faddrs if x[1].find('@') > 0] + if len(faddrs) == 1: + realname, email = o_from = faddrs[0] + else: + # No From: or multiple addresses. Just punt and take + # the get_sender result. + realname = '' + email = msgdata['original_sender'] + o_from = (realname, email) + if len(realname) == 0: + member = mlist.members.get_member(email) + if member: + realname = member.display_name or email + else: + realname = email + # Remove domain from realname if it looks like an email address. + realname = re.sub(r'@([^ .]+\.)+[^ .]+$', '---', realname) + # Make a display name and RFC 2047 encode it if necessary. This is + # difficult and kludgy. If the realname came from From: it should be + # ascii or RFC 2047 encoded. If it came from the list, it should be + # a string. If it's from the email address, it should be an ascii string. + # In any case, ensure it's an unencoded string. + srn = '' + for frag, cs in decode_header(realname): + if not cs: + # Character set should be ascii, but use iso-8859-1 anyway. + cs = 'iso-8859-1' + if not isinstance(frag, str): + srn += str(frag, cs, errors='replace') + else: + srn += frag + # The list's real_name is a string. + lrn = mlist.display_name # noqa F841 + realname = srn + # Ensure the i18n context is the list's preferred_language. + with _.using(mlist.preferred_language.code): + via = _('$realname via $lrn') + # Get an RFC 2047 encoded header string. + dn = str(Header(via, mlist.preferred_language.charset)) + retn = [('From', formataddr((dn, mlist.posting_address)))] + # We've made the munged From:. Now put the original in Reply-To: or Cc: + if mlist.reply_goes_to_list is ReplyToMunging.no_munging: + # Add original from to Reply-To: + add_to = 'Reply-To' + else: + # Add original from to Cc: + add_to = 'Cc' + orig = getaddresses(msg.get_all(add_to, [])) + if o_from[1] not in [x[1] for x in orig]: + orig.append(o_from) + retn.append((add_to, COMMASPACE.join(formataddr(x) for x in orig))) + return retn + + +def munge_from(mlist, msg, msgdata): + for k, v in munged_headers(mlist, msg, msgdata): + del msg[k] + msg[k] = v + return + + +def wrap_message(mlist, msg, msgdata): + # Create a wrapper message around the original. + # There are various headers in msg that we don't want, so we basically + # make a copy of the msg, then delete almost everything and set/copy + # what we want. + omsg = copy.deepcopy(msg) + for key in msg: + keep = False + for keeper in KEEPERS: + if re.match(keeper, key, re.I): + keep = True + break + if not keep: + del msg[key] + msg['MIME-Version'] = '1.0' + msg['Message-ID'] = make_msgid() + for k, v in munged_headers(mlist, omsg, msgdata): + msg[k] = v + # Are we including dmarc_wrapped_message_text? + if len(mlist.dmarc_wrapped_message_text) > 0: + part1 = MIMEText(wrap(mlist.dmarc_wrapped_message_text), + 'plain', + mlist.preferred_language.charset) + part1['Content-Disposition'] = 'inline' + part2 = MIMEMessage(omsg) + part2['Content-Disposition'] = 'inline' + msg['Content-Type'] = 'multipart/mixed' + msg.set_payload([part1, part2]) + else: + msg['Content-Type'] = 'message/rfc822' + msg['Content-Disposition'] = 'inline' + msg.set_payload([omsg]) + return + + +def process(mlist, msg, msgdata): + # If we're mitigating on policy and we have no hit, return. + if not msgdata.get('dmarc') and not mlist.dmarc_mitigate_unconditionally: + return + # if we're not mitigating, return. + if mlist.dmarc_mitigate_action is DMARCMitigateAction.no_mitigation: + return + if mlist.anonymous_list: + # DMARC mitigation is not required for anonymous lists. + return + if msgdata.get('dmarc') or mlist.dmarc_mitigate_unconditionally: + if mlist.dmarc_mitigate_action is DMARCMitigateAction.munge_from: + munge_from(mlist, msg, msgdata) + elif mlist.dmarc_mitigate_action is DMARCMitigateAction.wrap_message: + wrap_message(mlist, msg, msgdata) + else: + # We can get here if DMARCMitigateAction is reject or discard + # but the From: domain has no reject or quarantine policy and + # mlist.dmarc_mitigate_unconditionally is True. We just ignore + # this. + return + else: + raise AssertionError( + 'handlers/dmarc.py: no hit and unconditional is False') + + +@public +@implementer(IHandler) +class DMARC: + """Apply DMARC mitigations.""" + + name = 'dmarc' + description = _('Apply DMARC mitigations.') + + def process(self, mlist, msg, msgdata): + """See `IHandler`.""" + process(mlist, msg, msgdata) diff --git a/src/mailman/handlers/docs/dmarc-mitigations.rst b/src/mailman/handlers/docs/dmarc-mitigations.rst new file mode 100644 index 000000000..d26fc6bdd --- /dev/null +++ b/src/mailman/handlers/docs/dmarc-mitigations.rst @@ -0,0 +1,151 @@ +================= +DMARC Mitigations +================= + +In order to mitigate the effects of DMARC_ on mailing list traffic, list +administrators have the ability to apply transformations to messages delivered +to list members. These transformations are applied only to individual messages +sent to list members and not to messages in digests, archives, or gated to +USENET. + +The messages can be transformed by either munging the From: header and putting +original From: in Cc: or Reply-To: or by wrapping the original message in an +outer message From: the list. + +Exactly what transformations are applied depends on a number of list settings. + +The settings and their effects are: + +``anonymous_list`` + If True, no mitigations are ever applied because the message + is already From: the list. +``dmarc_mitigate_action`` + The action to apply to messages From: a domain + publishing a DMARC policy of reject or quarantine or to all messages + depending on the next setting. +``dmarc_mitigate_unconditionally`` + A Flag to apply dmarc_mitigate_action to all messages, but only if + dmarc_mitigate_action is neither reject or discard. +``dmarc_moderation_notice`` + Text to include in any rejection notice to be sent + when dmarc_policy_mitigation of reject applies. This overrides the bult-in + default text. +``dmarc_wrapped_message_text`` + Text to be added as a separate text/plain MIME + part preceding the original message part in the wrapped message when a + wrap_message mitigation applies. If this is not provided the separate + text/plain MIME part is not added. +``reply_goes_to_list`` + If this is set to other than no_munging of Reply-To, + the original From: goes in Cc: rather than Reply-To:. This is intended to + make MUA functions of reply and reply-all have the same effect with + messages to which mitigations have been applied as they do with other + messages. + +The possible actions for dmarc_mitigate_action are: + +``no_mitigation`` + Make no transformation to the message. +``munge_from`` + Change the From: header and put the original From: in Reply-To: + or in some cases Cc: +``wrap_message`` + Wrap the message in an outer message with headers as in + munge_from. +``reject`` + Bounce the message back to the sender with a default reason or one + supplied in dmarc_moderation_notice. +``discard`` + Silently discard the message. + +Here's what happens when we munge the From. + + >>> from mailman.interfaces.mailinglist import (DMARCMitigateAction, + ... ReplyToMunging) + >>> mlist = create_list('test@example.com') + >>> mlist.dmarc_mitigate_action = DMARCMitigateAction.munge_from + >>> mlist.reply_goes_to_list = ReplyToMunging.no_munging + >>> msg = message_from_string("""\ + ... From: A Person <aperson@example.com> + ... To: test@example.com + ... + ... A message of great import. + ... """) + >>> msgdata = dict(dmarc=True, original_sender='aperson@example.com') + >>> from mailman.handlers.dmarc import process + >>> process(mlist, msg, msgdata) + >>> print(msg.as_string()) + To: test@example.com + From: A Person via Test <test@example.com> + Reply-To: A Person <aperson@example.com> + <BLANKLINE> + A message of great import. + <BLANKLINE> + +Here we wrap the message without adding a text part. + + >>> mlist.dmarc_mitigate_action = DMARCMitigateAction.wrap_message + >>> mlist.dmarc_wrapped_message_text = '' + >>> msg = message_from_string("""\ + ... From: A Person <aperson@example.com> + ... To: test@example.com + ... + ... A message of great import. + ... """) + >>> msgdata = dict(dmarc=True, original_sender='aperson@example.com') + >>> process(mlist, msg, msgdata) + >>> print(msg.as_string()) + To: test@example.com + MIME-Version: 1.0 + Message-ID: <...> + From: A Person via Test <test@example.com> + Reply-To: A Person <aperson@example.com> + Content-Type: message/rfc822 + Content-Disposition: inline + <BLANKLINE> + From: A Person <aperson@example.com> + To: test@example.com + <BLANKLINE> + A message of great import. + <BLANKLINE> + +And here's a wrapped message with an added text part. + + >>> mlist.dmarc_wrapped_message_text = 'The original message is attached.' + >>> msg = message_from_string("""\ + ... From: A Person <aperson@example.com> + ... To: test@example.com + ... + ... A message of great import. + ... """) + >>> msgdata = dict(dmarc=True, original_sender='aperson@example.com') + >>> process(mlist, msg, msgdata) + >>> print(msg.as_string()) + To: test@example.com + MIME-Version: 1.0 + Message-ID: <...> + From: A Person via Test <test@example.com> + Reply-To: A Person <aperson@example.com> + Content-Type: multipart/mixed; boundary="..." + <BLANKLINE> + --... + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: inline + <BLANKLINE> + The original message is attached. + --... + Content-Type: message/rfc822 + MIME-Version: 1.0 + Content-Disposition: inline + <BLANKLINE> + From: A Person <aperson@example.com> + To: test@example.com + <BLANKLINE> + A message of great import. + <BLANKLINE> + --...-- + <BLANKLINE> + +.. _DMARC: https://wikipedia.org/wiki/DMARC diff --git a/src/mailman/handlers/tests/test_dmarc.py b/src/mailman/handlers/tests/test_dmarc.py new file mode 100644 index 000000000..6fe15bd18 --- /dev/null +++ b/src/mailman/handlers/tests/test_dmarc.py @@ -0,0 +1,441 @@ +# Copyright (C) 2016 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test the dmarc handler.""" + +import unittest + +from mailman.app.lifecycle import create_list +from mailman.app.membership import add_member +from mailman.handlers import dmarc +from mailman.interfaces.mailinglist import DMARCMitigateAction, ReplyToMunging +from mailman.interfaces.subscriptions import RequestRecord +from mailman.testing.helpers import specialized_message_from_string as mfs +from mailman.testing.layers import ConfigLayer + + +class TestDMARCMitigations(unittest.TestCase): + """Test the dmarc handler.""" + + layer = ConfigLayer + maxDiff = None + + def setUp(self): + self._mlist = create_list('ant@example.com') + self._mlist.anonymous_list = False + self._mlist.dmarc_policy_mitigate_action = ( + DMARCMitigateAction.no_mitigation) + self._mlist.dmarc_wrapped_message_text = '' + self._mlist.dmarc_mitigate_unconditionally = False + self._mlist.reply_goes_to_list = ReplyToMunging.no_munging + # We can use the same message text for most tests. + self._text = """\ +From: anne@example.com +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Message-ID: <alpha@example.com> +Date: Fri, 1 Jan 2016 00:00:01 +0000 +Another-Header: To test removal in wrapper +MIME-Version: 1.0 +Content-Type: multipart/alternative; boundary="=====abc==" + +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +Some things to say. +--=====abc== +Content-Type: text/html; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +<html><head></head><body>Some things to say.</body></html> +--=====abc==-- +""" + + def test_default_no_change(self): + msg = mfs(self._text) + dmarc.process(self._mlist, msg, {}) + self.assertMultiLineEqual(msg.as_string(), self._text) + + def test_anonymous_no_change(self): + self._mlist.anonymous_list = True + self._mlist.dmarc_mitigate_action = DMARCMitigateAction.munge_from + msgdata = {'dmarc': True} + msg = mfs(self._text) + dmarc.process(self._mlist, msg, msgdata) + self.assertMultiLineEqual(msg.as_string(), self._text) + + def test_no_mitigation_no_change_1(self): + msg = mfs(self._text) + self._mlist.dmarc_mitigate_unconditionally = True + dmarc.process(self._mlist, msg, {}) + self.assertMultiLineEqual(msg.as_string(), self._text) + + def test_no_mitigation_no_change_2(self): + msg = mfs(self._text) + msgdata = {'dmarc': True} + dmarc.process(self._mlist, msg, msgdata) + self.assertMultiLineEqual(msg.as_string(), self._text) + + def test_action_reject_mitigate_unconditionally(self): + msg = mfs(self._text) + self._mlist.dmarc_mitigate_unconditionally = True + self._mlist.dmarc_mitigate_action = DMARCMitigateAction.reject + dmarc.process(self._mlist, msg, {}) + self.assertMultiLineEqual(msg.as_string(), self._text) + + def test_action_discard_mitigate_unconditionally(self): + msg = mfs(self._text) + self._mlist.dmarc_mitigate_unconditionally = True + self._mlist.dmarc_mitigate_action = DMARCMitigateAction.discard + dmarc.process(self._mlist, msg, {}) + self.assertMultiLineEqual(msg.as_string(), self._text) + + def test_action_munge_from(self): + self._mlist.dmarc_mitigate_action = DMARCMitigateAction.munge_from + msgdata = {'dmarc': True} + msg = mfs(self._text) + dmarc.process(self._mlist, msg, msgdata) + self.assertMultiLineEqual(msg.as_string(), """\ +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Message-ID: <alpha@example.com> +Date: Fri, 1 Jan 2016 00:00:01 +0000 +Another-Header: To test removal in wrapper +MIME-Version: 1.0 +Content-Type: multipart/alternative; boundary="=====abc==" +From: anne--- via Ant <ant@example.com> +Reply-To: anne@example.com + +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +Some things to say. +--=====abc== +Content-Type: text/html; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +<html><head></head><body>Some things to say.</body></html> +--=====abc==-- +""") + + def test_action_munge_from_no_from(self): + self._mlist.dmarc_mitigate_action = DMARCMitigateAction.munge_from + msgdata = dict( + dmarc=True, + original_sender='anne@example.com', + ) + msg = mfs(self._text) + del msg['from'] + dmarc.process(self._mlist, msg, msgdata) + self.assertMultiLineEqual(msg.as_string(), """\ +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Message-ID: <alpha@example.com> +Date: Fri, 1 Jan 2016 00:00:01 +0000 +Another-Header: To test removal in wrapper +MIME-Version: 1.0 +Content-Type: multipart/alternative; boundary="=====abc==" +From: anne--- via Ant <ant@example.com> +Reply-To: anne@example.com + +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +Some things to say. +--=====abc== +Content-Type: text/html; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +<html><head></head><body>Some things to say.</body></html> +--=====abc==-- +""") + + def test_action_munge_multiple_froms(self): + self._mlist.dmarc_mitigate_action = DMARCMitigateAction.munge_from + msgdata = dict( + dmarc=True, + original_sender='cate@example.com', + ) + msg = mfs(self._text) + # Put multiple addresses in the From: header. The msgdata must + # contain a key naming the "original sender" as determined by the + # Message.sender attribute. + del msg['from'] + msg['From'] = 'anne@example.com, bart@example.com' + dmarc.process(self._mlist, msg, msgdata) + self.assertMultiLineEqual(msg.as_string(), """\ +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Message-ID: <alpha@example.com> +Date: Fri, 1 Jan 2016 00:00:01 +0000 +Another-Header: To test removal in wrapper +MIME-Version: 1.0 +Content-Type: multipart/alternative; boundary="=====abc==" +From: cate--- via Ant <ant@example.com> +Reply-To: cate@example.com + +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +Some things to say. +--=====abc== +Content-Type: text/html; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +<html><head></head><body>Some things to say.</body></html> +--=====abc==-- +""") + + def test_action_munge_from_display_name_in_from(self): + self._mlist.dmarc_mitigate_action = DMARCMitigateAction.munge_from + msgdata = {'dmarc': True} + msg = mfs(self._text) + del msg['from'] + msg['From'] = 'Anne Person <anne@example.com>' + dmarc.process(self._mlist, msg, msgdata) + self.assertMultiLineEqual(msg.as_string(), """\ +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Message-ID: <alpha@example.com> +Date: Fri, 1 Jan 2016 00:00:01 +0000 +Another-Header: To test removal in wrapper +MIME-Version: 1.0 +Content-Type: multipart/alternative; boundary="=====abc==" +From: Anne Person via Ant <ant@example.com> +Reply-To: Anne Person <anne@example.com> + +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +Some things to say. +--=====abc== +Content-Type: text/html; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +<html><head></head><body>Some things to say.</body></html> +--=====abc==-- +""") + + def test_action_munge_from_display_name_in_list(self): + self._mlist.dmarc_mitigate_action = DMARCMitigateAction.munge_from + add_member( + self._mlist, + RequestRecord('anne@example.com', 'Anna Banana') + ) + msgdata = {'dmarc': True} + msg = mfs(self._text) + dmarc.process(self._mlist, msg, msgdata) + self.assertMultiLineEqual(msg.as_string(), """\ +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Message-ID: <alpha@example.com> +Date: Fri, 1 Jan 2016 00:00:01 +0000 +Another-Header: To test removal in wrapper +MIME-Version: 1.0 +Content-Type: multipart/alternative; boundary="=====abc==" +From: Anna Banana via Ant <ant@example.com> +Reply-To: anne@example.com + +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +Some things to say. +--=====abc== +Content-Type: text/html; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +<html><head></head><body>Some things to say.</body></html> +--=====abc==-- +""") + + def test_no_action_without_msgdata(self): + self._mlist.dmarc_mitigate_action = DMARCMitigateAction.munge_from + msg = mfs(self._text) + dmarc.process(self._mlist, msg, {}) + self.assertMultiLineEqual(msg.as_string(), self._text) + + def test_unconditional_no_msgdata(self): + self._mlist.dmarc_mitigate_action = DMARCMitigateAction.munge_from + self._mlist.dmarc_mitigate_unconditionally = True + msg = mfs(self._text) + dmarc.process(self._mlist, msg, {}) + self.assertMultiLineEqual(msg.as_string(), """\ +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Message-ID: <alpha@example.com> +Date: Fri, 1 Jan 2016 00:00:01 +0000 +Another-Header: To test removal in wrapper +MIME-Version: 1.0 +Content-Type: multipart/alternative; boundary="=====abc==" +From: anne--- via Ant <ant@example.com> +Reply-To: anne@example.com + +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +Some things to say. +--=====abc== +Content-Type: text/html; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +<html><head></head><body>Some things to say.</body></html> +--=====abc==-- +""") + + def test_from_in_cc(self): + self._mlist.dmarc_mitigate_action = DMARCMitigateAction.munge_from + self._mlist.reply_goes_to_list = ReplyToMunging.point_to_list + msgdata = {'dmarc': True} + msg = mfs(self._text) + dmarc.process(self._mlist, msg, msgdata) + self.assertMultiLineEqual(msg.as_string(), """\ +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Message-ID: <alpha@example.com> +Date: Fri, 1 Jan 2016 00:00:01 +0000 +Another-Header: To test removal in wrapper +MIME-Version: 1.0 +Content-Type: multipart/alternative; boundary="=====abc==" +From: anne--- via Ant <ant@example.com> +Cc: anne@example.com + +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +Some things to say. +--=====abc== +Content-Type: text/html; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +<html><head></head><body>Some things to say.</body></html> +--=====abc==-- +""") + + def test_wrap_message_unconditionally(self): + self._mlist.dmarc_mitigate_action = DMARCMitigateAction.wrap_message + self._mlist.dmarc_mitigate_unconditionally = True + msg = mfs(self._text) + dmarc.process(self._mlist, msg, {}) + # We can't predict the Message-ID in the wrapper so delete it, but + # ensure we have one. + self.assertIsNotNone(msg.get('message-id')) + del msg['message-id'] + self.assertMultiLineEqual(msg.as_string(), """\ +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Date: Fri, 1 Jan 2016 00:00:01 +0000 +MIME-Version: 1.0 +From: anne--- via Ant <ant@example.com> +Reply-To: anne@example.com +Content-Type: message/rfc822 +Content-Disposition: inline + +""" + self._text) + + def test_wrap_message(self): + self._mlist.dmarc_mitigate_action = DMARCMitigateAction.wrap_message + msgdata = {'dmarc': True} + msg = mfs(self._text) + dmarc.process(self._mlist, msg, msgdata) + # We can't predict the Message-ID in the wrapper so delete it, but + # ensure we have one. + self.assertIsNotNone(msg.get('message-id')) + del msg['message-id'] + self.assertMultiLineEqual(msg.as_string(), """\ +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Date: Fri, 1 Jan 2016 00:00:01 +0000 +MIME-Version: 1.0 +From: anne--- via Ant <ant@example.com> +Reply-To: anne@example.com +Content-Type: message/rfc822 +Content-Disposition: inline + +""" + self._text) + + def test_wrap_message_cc(self): + self._mlist.dmarc_mitigate_action = DMARCMitigateAction.wrap_message + self._mlist.reply_goes_to_list = ReplyToMunging.point_to_list + msgdata = {'dmarc': True} + msg = mfs(self._text) + dmarc.process(self._mlist, msg, msgdata) + # We can't predict the Message-ID in the wrapper so delete it, but + # ensure we have one. + self.assertIsNotNone(msg.get('message-id')) + del msg['message-id'] + self.assertMultiLineEqual(msg.as_string(), """\ +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Date: Fri, 1 Jan 2016 00:00:01 +0000 +MIME-Version: 1.0 +From: anne--- via Ant <ant@example.com> +Cc: anne@example.com +Content-Type: message/rfc822 +Content-Disposition: inline + +""" + self._text) + + def test_rfc2047_encoded_from(self): + self._mlist.dmarc_mitigate_action = DMARCMitigateAction.munge_from + msgdata = {'dmarc': True} + msg = mfs(self._text) + del msg['from'] + msg['From'] = '=?iso-8859-1?Q?A_Pers=F3n?= <anne@example.com>' + dmarc.process(self._mlist, msg, msgdata) + self.assertMultiLineEqual(msg.as_string(), """\ +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Message-ID: <alpha@example.com> +Date: Fri, 1 Jan 2016 00:00:01 +0000 +Another-Header: To test removal in wrapper +MIME-Version: 1.0 +Content-Type: multipart/alternative; boundary="=====abc==" +From: =?utf-8?q?A_Pers=C3=B3n_via_Ant?= <ant@example.com> +Reply-To: =?iso-8859-1?Q?A_Pers=F3n?= <anne@example.com> + +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +Some things to say. +--=====abc== +Content-Type: text/html; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +<html><head></head><body>Some things to say.</body></html> +--=====abc==-- +""") diff --git a/src/mailman/interfaces/mailinglist.py b/src/mailman/interfaces/mailinglist.py index 359cd5069..0f1fbace1 100644 --- a/src/mailman/interfaces/mailinglist.py +++ b/src/mailman/interfaces/mailinglist.py @@ -24,6 +24,25 @@ from zope.interface import Attribute, Interface @public +class DMARCMitigateAction(Enum): + # Mitigation to apply to messages From: domains publishing an applicable + # DMARC policy or unconditionally depending on settings. + no_mitigation = 0 + # No DMARC mitigations. + munge_from = 1 + # Messages From: domains with DMARC policy will have From: replaced by + # the list posting address and the original From: added to Reply-To: + # or Cc:. + wrap_message = 2 + # Messages From: domains with DMARC policy will be wrapped in an outer + # message From: the list posting address. + reject = 3 + # Messages From: domains with DMARC policy will be rejected. + discard = 4 + # Messages From: domains with DMARC policy will be discarded. + + +@public class Personalization(Enum): none = 0 # Everyone gets a unique copy of the message, and there are a few more @@ -216,6 +235,29 @@ class IMailingList(Interface): def confirm_address(cookie=''): """The address used for various forms of email confirmation.""" + # DMARC attributes. + + dmarc_mitigate_action = Attribute( + """The DMARCMitigateAction to be applied to messages From: a domain + publishing DMARC p=reject or quarantine and possibly unconditionally. + """) + + dmarc_mitigate_unconditionally = Attribute( + """A Flag to apply dmarc_mitigate_action to all messages but only if + dmarc_mitigate_action is other than reject or discard. + """) + + dmarc_moderation_notice = Attribute( + """Text to include in any rejection notice to be sent when + DMARCMitigateAction of reject applies. + """) + + dmarc_wrapped_message_text = Attribute( + """Text to be added as a separate text/plain MIME part preceding the + original message part in the wrapped message when + DMARCMitigateAction of wrap_message applies. + """) + # Rosters and subscriptions. owners = Attribute( diff --git a/src/mailman/model/mailinglist.py b/src/mailman/model/mailinglist.py index bd6538aaf..fe35f6127 100644 --- a/src/mailman/model/mailinglist.py +++ b/src/mailman/model/mailinglist.py @@ -22,7 +22,7 @@ import os from mailman.config import config from mailman.database.model import Model from mailman.database.transaction import dbconnection -from mailman.database.types import Enum, SAUnicode +from mailman.database.types import Enum, SAUnicode, SAUnicodeLarge from mailman.interfaces.action import Action, FilterAction from mailman.interfaces.address import IAddress from mailman.interfaces.archiver import ArchivePolicy @@ -32,9 +32,9 @@ from mailman.interfaces.digests import DigestFrequency from mailman.interfaces.domain import IDomainManager from mailman.interfaces.languages import ILanguageManager from mailman.interfaces.mailinglist import ( - IAcceptableAlias, IAcceptableAliasSet, IHeaderMatch, IHeaderMatchList, - IListArchiver, IListArchiverSet, IMailingList, Personalization, - ReplyToMunging, SubscriptionPolicy) + DMARCMitigateAction, IAcceptableAlias, IAcceptableAliasSet, + IHeaderMatch, IHeaderMatchList, IListArchiver, IListArchiverSet, + IMailingList, Personalization, ReplyToMunging, SubscriptionPolicy) from mailman.interfaces.member import ( AlreadySubscribedError, MemberRole, MissingPreferredAddressError, SubscriptionEvent) @@ -127,6 +127,11 @@ class MailingList(Model): forward_unrecognized_bounces_to = Column( Enum(UnrecognizedBounceDisposition)) process_bounces = Column(Boolean) + # DMARC + dmarc_mitigate_action = Column(Enum(DMARCMitigateAction)) + dmarc_mitigate_unconditionally = Column(Boolean) + dmarc_moderation_notice = Column(SAUnicodeLarge) + dmarc_wrapped_message_text = Column(SAUnicodeLarge) # Miscellaneous default_member_action = Column(Enum(Action)) default_nonmember_action = Column(Enum(Action)) diff --git a/src/mailman/rest/docs/listconf.rst b/src/mailman/rest/docs/listconf.rst index 3434f2a3d..343e50715 100644 --- a/src/mailman/rest/docs/listconf.rst +++ b/src/mailman/rest/docs/listconf.rst @@ -44,6 +44,10 @@ All readable attributes for a list are available on a sub-resource. digest_volume_frequency: monthly digests_enabled: True display_name: Ant + dmarc_mitigate_action: no_mitigation + dmarc_mitigate_unconditionally: False + dmarc_moderation_notice: + dmarc_wrapped_message_text: filter_content: False first_strip_reply_to: False footer_uri: @@ -110,6 +114,10 @@ When using ``PUT``, all writable attributes must be included. ... digest_size_threshold=10.5, ... digest_volume_frequency='yearly', ... digests_enabled=False, + ... dmarc_mitigate_action='munge_from', + ... dmarc_mitigate_unconditionally=False, + ... dmarc_moderation_notice='Some moderation notice', + ... dmarc_wrapped_message_text='some message text', ... posting_pipeline='virgin', ... filter_content=True, ... first_strip_reply_to=True, @@ -162,6 +170,10 @@ These values are changed permanently. digest_volume_frequency: yearly digests_enabled: False display_name: Fnords + dmarc_mitigate_action: munge_from + dmarc_mitigate_unconditionally: False + dmarc_moderation_notice: Some moderation notice + dmarc_wrapped_message_text: some message text filter_content: True first_strip_reply_to: True footer_uri: diff --git a/src/mailman/rest/docs/systemconf.rst b/src/mailman/rest/docs/systemconf.rst index 385588077..24c6bef4a 100644 --- a/src/mailman/rest/docs/systemconf.rst +++ b/src/mailman/rest/docs/systemconf.rst @@ -14,6 +14,9 @@ You can also get all the values for a particular section. >>> dump_json('http://localhost:9001/3.0/system/configuration/mailman') cache_life: 7d default_language: en + dmarc_org_domain_data: https://publicsuffix.org/list/public_suffix_list.dat + dmarc_resolver_lifetime: 5s + dmarc_resolver_timeout: 3s email_commands_max_lines: 10 filtered_messages_are_preservable: no html_to_plain_text_command: /usr/bin/lynx -dump $filename diff --git a/src/mailman/rest/listconf.py b/src/mailman/rest/listconf.py index 94598b0ab..d5a1ca0c1 100644 --- a/src/mailman/rest/listconf.py +++ b/src/mailman/rest/listconf.py @@ -24,7 +24,8 @@ from mailman.interfaces.archiver import ArchivePolicy from mailman.interfaces.autorespond import ResponseAction from mailman.interfaces.digests import DigestFrequency from mailman.interfaces.mailinglist import ( - IAcceptableAliasSet, IMailingList, ReplyToMunging, SubscriptionPolicy) + DMARCMitigateAction, IAcceptableAliasSet, IMailingList, ReplyToMunging, + SubscriptionPolicy) from mailman.interfaces.template import ITemplateManager from mailman.rest.helpers import ( GetterSetter, bad_request, etag, no_content, not_found, okay) @@ -128,7 +129,9 @@ ATTRIBUTES = dict( admin_notify_mchanges=GetterSetter(as_boolean), administrivia=GetterSetter(as_boolean), advertised=GetterSetter(as_boolean), + allow_list_posts=GetterSetter(as_boolean), anonymous_list=GetterSetter(as_boolean), + archive_policy=GetterSetter(enum_validator(ArchivePolicy)), autorespond_owner=GetterSetter(enum_validator(ResponseAction)), autorespond_postings=GetterSetter(enum_validator(ResponseAction)), autorespond_requests=GetterSetter(enum_validator(ResponseAction)), @@ -136,7 +139,6 @@ ATTRIBUTES = dict( autoresponse_owner_text=GetterSetter(str), autoresponse_postings_text=GetterSetter(str), autoresponse_request_text=GetterSetter(str), - archive_policy=GetterSetter(enum_validator(ArchivePolicy)), bounces_address=GetterSetter(None), collapse_alternatives=GetterSetter(as_boolean), convert_html_to_plaintext=GetterSetter(as_boolean), @@ -144,22 +146,27 @@ ATTRIBUTES = dict( default_member_action=GetterSetter(enum_validator(Action)), default_nonmember_action=GetterSetter(enum_validator(Action)), description=GetterSetter(no_newlines_validator), + display_name=GetterSetter(str), digest_last_sent_at=GetterSetter(None), digest_send_periodic=GetterSetter(as_boolean), digest_size_threshold=GetterSetter(float), digest_volume_frequency=GetterSetter(enum_validator(DigestFrequency)), digests_enabled=GetterSetter(as_boolean), + dmarc_mitigate_action=GetterSetter( + enum_validator(DMARCMitigateAction)), + dmarc_mitigate_unconditionally=GetterSetter(as_boolean), + dmarc_moderation_notice=GetterSetter(str), + dmarc_wrapped_message_text=GetterSetter(str), filter_content=GetterSetter(as_boolean), first_strip_reply_to=GetterSetter(as_boolean), fqdn_listname=GetterSetter(None), - mail_host=GetterSetter(None), - allow_list_posts=GetterSetter(as_boolean), include_rfc2369_headers=GetterSetter(as_boolean), info=GetterSetter(str), join_address=GetterSetter(None), last_post_at=GetterSetter(None), leave_address=GetterSetter(None), list_name=GetterSetter(None), + mail_host=GetterSetter(None), moderator_password=GetterSetter(password_bytes_validator), next_digest_number=GetterSetter(None), no_reply_address=GetterSetter(None), @@ -167,7 +174,6 @@ ATTRIBUTES = dict( post_id=GetterSetter(None), posting_address=GetterSetter(None), posting_pipeline=GetterSetter(pipeline_validator), - display_name=GetterSetter(str), reply_goes_to_list=GetterSetter(enum_validator(ReplyToMunging)), reply_to_address=GetterSetter(str), request_address=GetterSetter(None), diff --git a/src/mailman/rest/tests/test_listconf.py b/src/mailman/rest/tests/test_listconf.py index 3260d6afb..b9b64464c 100644 --- a/src/mailman/rest/tests/test_listconf.py +++ b/src/mailman/rest/tests/test_listconf.py @@ -62,6 +62,10 @@ RESOURCE = dict( digest_volume_frequency='monthly', digests_enabled=True, display_name='Fnords', + dmarc_mitigate_action='munge_from', + dmarc_mitigate_unconditionally=False, + dmarc_moderation_notice='Some moderation notice', + dmarc_wrapped_message_text='some message text', filter_content=True, first_strip_reply_to=True, goodbye_message_uri='mailman:///goodbye.txt', diff --git a/src/mailman/rest/tests/test_systemconf.py b/src/mailman/rest/tests/test_systemconf.py index e76c082df..28fa315a9 100644 --- a/src/mailman/rest/tests/test_systemconf.py +++ b/src/mailman/rest/tests/test_systemconf.py @@ -39,6 +39,10 @@ class TestSystemConfiguration(unittest.TestCase): self.assertEqual(json, dict( cache_life='7d', default_language='en', + dmarc_org_domain_data= # noqa E251 + 'https://publicsuffix.org/list/public_suffix_list.dat', + dmarc_resolver_lifetime='5s', + dmarc_resolver_timeout='3s', email_commands_max_lines='10', filtered_messages_are_preservable='no', html_to_plain_text_command='/usr/bin/lynx -dump $filename', diff --git a/src/mailman/rules/dmarc.py b/src/mailman/rules/dmarc.py new file mode 100644 index 000000000..890fe9ff2 --- /dev/null +++ b/src/mailman/rules/dmarc.py @@ -0,0 +1,253 @@ +# Copyright (C) 2016 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""DMARC mitigation rule.""" + +import re +import logging +import dns.resolver + +from dns.exception import DNSException +from email.utils import parseaddr +from lazr.config import as_timedelta +from mailman.config import config +from mailman.core.i18n import _ +from mailman.interfaces.mailinglist import DMARCMitigateAction +from mailman.interfaces.rules import IRule +from mailman.utilities.string import wrap +from public import public +from urllib import error, request +from zope.interface import implementer + + +elog = logging.getLogger('mailman.error') +vlog = logging.getLogger('mailman.vette') +s_dict = dict() + + +def _get_suffixes(url): + # This loads and parses the data from the url argument into s_dict for + # use by _get_org_dom. + global s_dict + if not url: + return + try: + d = request.urlopen(url) + except error.URLError as e: + elog.error('Unable to retrieve data from %s: %s', url, e.reason) + return + for line in d.readlines(): + line = str(line, encoding='utf-8') + if not line.strip() or line.startswith(' ') or line.startswith('//'): + continue + line = re.sub(' .*', '', line.strip()) + if not line: + continue + parts = line.lower().split('.') + if parts[0].startswith('!'): + exc = True + parts = [parts[0][1:]] + parts[1:] + else: + exc = False + parts.reverse() + k = '.'.join(parts) + s_dict[k] = exc + + +def _get_dom(d, l): + # A helper to get a domain name consisting of the first l+1 labels + # in d. + dom = d[:min(l+1, len(d))] + dom.reverse() + return '.'.join(dom) + + +def _get_org_dom(domain): + # Given a domain name, this returns the corresponding Organizational + # Domain which may be the same as the input. + global s_dict + if not s_dict: + _get_suffixes(config.mailman.dmarc_org_domain_data) + hits = [] + d = domain.lower().split('.') + d.reverse() + for k in s_dict.keys(): + ks = k.split('.') + if len(d) >= len(ks): + for i in range(len(ks)-1): + if d[i] != ks[i] and ks[i] != '*': + break + else: + if d[len(ks)-1] == ks[-1] or ks[-1] == '*': + hits.append(k) + if not hits: + return _get_dom(d, 1) + l = 0 + for k in hits: + if s_dict[k]: + # It's an exception + return _get_dom(d, len(k.split('.'))-1) + if len(k.split('.')) > l: + l = len(k.split('.')) + return _get_dom(d, l) + + +def _IsDMARCProhibited(mlist, email): + # This takes an email address, and returns True if DMARC policy is + # p=reject or quarantine. + email = email.lower() + # Scan from the right in case quoted local part has an '@'. + at_sign = email.rfind('@') + if at_sign < 1: + return False + f_dom = email[at_sign+1:] + x = _DMARCProhibited(mlist, email, '_dmarc.' + f_dom) + if x != 'continue': + return x + o_dom = _get_org_dom(f_dom) + if o_dom != f_dom: + x = _DMARCProhibited(mlist, email, '_dmarc.' + o_dom, org=True) + if x != 'continue': + return x + return False + + +def _DMARCProhibited(mlist, email, dmarc_domain, org=False): + + try: + resolver = dns.resolver.Resolver() + resolver.timeout = as_timedelta( + config.mailman.dmarc_resolver_timeout).total_seconds() + resolver.lifetime = as_timedelta( + config.mailman.dmarc_resolver_lifetime).total_seconds() + txt_recs = resolver.query(dmarc_domain, dns.rdatatype.TXT) + except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): + return 'continue' + except DNSException as e: + elog.error( + 'DNSException: Unable to query DMARC policy for %s (%s). %s', + email, dmarc_domain, e.__doc__) + return 'continue' + else: + # Be as robust as possible in parsing the result. + results_by_name = {} + cnames = {} + want_names = set([dmarc_domain + '.']) + for txt_rec in txt_recs.response.answer: + if txt_rec.rdtype == dns.rdatatype.CNAME: + cnames[txt_rec.name.to_text()] = ( + txt_rec.items[0].target.to_text()) + if txt_rec.rdtype != dns.rdatatype.TXT: + continue + results_by_name.setdefault( + txt_rec.name.to_text(), []).append( + "".join( + [str(x, encoding='utf-8') + for x in txt_rec.items[0].strings])) + expands = list(want_names) + seen = set(expands) + while expands: + item = expands.pop(0) + if item in cnames: + if cnames[item] in seen: + continue # cname loop + expands.append(cnames[item]) + seen.add(cnames[item]) + want_names.add(cnames[item]) + want_names.discard(item) + + if len(want_names) != 1: + elog.error( + """multiple DMARC entries in results for %s, + processing each to be strict""", + dmarc_domain) + for name in want_names: + if name not in results_by_name: + continue + dmarcs = [x for x in results_by_name[name] + if x.startswith('v=DMARC1;')] + if len(dmarcs) == 0: + return 'continue' + if len(dmarcs) > 1: + elog.error( + """RRset of TXT records for %s has %d v=DMARC1 entries; + testing them all""", + dmarc_domain, len(dmarcs)) + for entry in dmarcs: + mo = re.search(r'\bsp=(\w*)\b', entry, re.IGNORECASE) + if org and mo: + policy = mo.group(1).lower() + else: + mo = re.search(r'\bp=(\w*)\b', entry, re.IGNORECASE) + if mo: + policy = mo.group(1).lower() + else: + continue + if policy in ('reject', 'quarantine'): + vlog.info( + """%s: DMARC lookup for %s (%s) + found p=%s in %s = %s""", + mlist.list_name, + email, + dmarc_domain, + policy, + name, + entry) + return True + return False + + +@public +@implementer(IRule) +class DMARCMitigation: + """The DMARC mitigation rule.""" + + name = 'dmarc-mitigation' + description = _('Find DMARC policy of From: domain.') + record = True + + def check(self, mlist, msg, msgdata): + """See `IRule`.""" + if mlist.dmarc_mitigate_action is DMARCMitigateAction.no_mitigation: + # Don't bother to check if we're not going to do anything. + return False + dn, addr = parseaddr(msg.get('from')) + if _IsDMARCProhibited(mlist, addr): + # If dmarc_mitigate_action is discard or reject, this rule fires + # and jumps to the 'moderation' chain to do the actual discard. + # Otherwise, the rule misses but sets a flag for the dmarc handler + # to do the appropriate action. + msgdata['dmarc'] = True + if mlist.dmarc_mitigate_action is DMARCMitigateAction.discard: + msgdata['moderation_action'] = 'discard' + msgdata['moderation_reasons'] = [_('DMARC moderation')] + elif mlist.dmarc_mitigate_action is DMARCMitigateAction.reject: + listowner = mlist.owner_address # noqa F841 + reason = (mlist.dmarc_moderation_notice or + _('You are not allowed to post to this mailing ' + 'list From: a domain which publishes a DMARC ' + 'policy of reject or quarantine, and your message' + ' has been automatically rejected. If you think ' + 'that your messages are being rejected in error, ' + 'contact the mailing list owner at ${listowner}.')) + msgdata['moderation_reasons'] = [wrap(reason)] + msgdata['moderation_action'] = 'reject' + else: + return False + msgdata['moderation_sender'] = addr + return True + return False diff --git a/src/mailman/rules/docs/dmarc-mitigation.rst b/src/mailman/rules/docs/dmarc-mitigation.rst new file mode 100644 index 000000000..7118a5be4 --- /dev/null +++ b/src/mailman/rules/docs/dmarc-mitigation.rst @@ -0,0 +1,148 @@ +================ +DMARC mitigation +================ + +This rule only matches in order to jump to the moderation chain to reject +or discard the message. The rule looks at the list's dmarc_mitigate_action +and if it is other than 'no_mitigation', it checks the domain of the From: +address for a DMARC policy and depending on settings may reject or discard +the message or just flag it for the dmarc handler to apply DMARC mitigations +to the message. + + >>> mlist = create_list('_xtest@example.com') + >>> rule = config.rules['dmarc-mitigation'] + >>> print(rule.name) + dmarc-mitigation + +First we set up a mock patcher to return predictable responses to DNS lookups. +This returns p=reject for the example.biz domain and not for any others. + + >>> from mailman.rules.tests.test_dmarc import get_dns_resolver + >>> patcher = get_dns_resolver() + +A message From: a domain without a DMARC policy does not set any flags. + + >>> from mailman.interfaces.mailinglist import DMARCMitigateAction + >>> mlist.dmarc_mitigate_action = DMARCMitigateAction.munge_from + >>> msg = message_from_string("""\ + ... From: aperson@example.org + ... To: _xtest@example.com + ... Subject: A posted message + ... + ... """) + >>> msgdata = {} + >>> with patcher as Resolver: + ... rule.check(mlist, msg, msgdata) + False + >>> msgdata == {} + True + +Even if the From: domain publishes p=reject, no flags are set if the list's +action is no_mitigation. + + >>> mlist.dmarc_mitigate_action = DMARCMitigateAction.no_mitigation + >>> msg = message_from_string("""\ + ... From: aperson@example.biz + ... To: _xtest@example.com + ... Subject: A posted message + ... + ... """) + >>> msgdata = {} + >>> with patcher as Resolver: + ... rule.check(mlist, msg, msgdata) + False + >>> msgdata == {} + True + +But with a different list setting, the message is flagged. + + >>> mlist.dmarc_mitigate_action = DMARCMitigateAction.munge_from + >>> msg = message_from_string("""\ + ... From: aperson@example.biz + ... To: _xtest@example.com + ... Subject: A posted message + ... + ... """) + >>> msgdata = {} + >>> with patcher as Resolver: + ... rule.check(mlist, msg, msgdata) + False + >>> msgdata['dmarc'] + True + +Subdomains which don't have a policy will check the organizational domain. + + >>> msg = message_from_string("""\ + ... From: aperson@sub.domain.example.biz + ... To: _xtest@example.com + ... Subject: A posted message + ... + ... """) + >>> msgdata = {} + >>> with patcher as Resolver: + ... rule.check(mlist, msg, msgdata) + False + >>> msgdata['dmarc'] + True + +The list's action can also be set to immediately discard or reject the +message. + + >>> mlist.dmarc_mitigate_action = DMARCMitigateAction.discard + >>> msg = message_from_string("""\ + ... From: aperson@example.biz + ... To: _xtest@example.com + ... Subject: A posted message + ... Message-ID: <xxx_message_id@example.biz> + ... + ... """) + >>> msgdata = {} + >>> with patcher as Resolver: + ... rule.check(mlist, msg, msgdata) + True + >>> msgdata['dmarc'] + True + >>> msgdata['moderation_action'] + 'discard' + +We can reject the message with a default reason. + + >>> mlist.dmarc_mitigate_action = DMARCMitigateAction.reject + >>> msg = message_from_string("""\ + ... From: aperson@example.biz + ... To: _xtest@example.com + ... Subject: A posted message + ... Message-ID: <xxx_message_id@example.biz> + ... + ... """) + >>> msgdata = {} + >>> with patcher as Resolver: + ... rule.check(mlist, msg, msgdata) + True + >>> msgdata['dmarc'] + True + >>> msgdata['moderation_action'] + 'reject' + >>> msgdata['moderation_reasons'] + ['You are not allowed to post to this mailing list From: a domain ... + +And, we can reject with a custom message. + + >>> mlist.dmarc_moderation_notice = 'A silly reason' + >>> msg = message_from_string("""\ + ... From: aperson@example.biz + ... To: _xtest@example.com + ... Subject: A posted message + ... Message-ID: <xxx_message_id@example.biz> + ... + ... """) + >>> msgdata = {} + >>> with patcher as Resolver: + ... rule.check(mlist, msg, msgdata) + True + >>> msgdata['dmarc'] + True + >>> msgdata['moderation_action'] + 'reject' + >>> msgdata['moderation_reasons'] + ['A silly reason'] diff --git a/src/mailman/rules/docs/rules.rst b/src/mailman/rules/docs/rules.rst index 812486b45..8b351650c 100644 --- a/src/mailman/rules/docs/rules.rst +++ b/src/mailman/rules/docs/rules.rst @@ -22,6 +22,7 @@ names to rule objects. any True approved True banned-address True + dmarc-mitigation True emergency True implicit-dest True loop True diff --git a/src/mailman/rules/tests/test_dmarc.py b/src/mailman/rules/tests/test_dmarc.py new file mode 100644 index 000000000..a486ad57a --- /dev/null +++ b/src/mailman/rules/tests/test_dmarc.py @@ -0,0 +1,134 @@ +# Copyright (C) 2016 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Provides support for mocking dnspython calls from dmarc rules and some +organizational domain tests.""" + +from contextlib import ExitStack +from dns.rdatatype import TXT +from dns.resolver import NXDOMAIN, NoAnswer +from mailman.rules import dmarc +from mailman.testing.helpers import LogFileMark +from mailman.testing.layers import ConfigLayer +from public import public +from unittest import TestCase +from unittest.mock import patch +from urllib.error import URLError + + +@public +def get_dns_resolver(): + """Create a dns.resolver.Resolver mock. + + This is used to return a predictable response to a _dmarc query. It + returns p=reject for the example.biz domain and raises either NXDOMAIN + or NoAnswer for any other. + + It only implements those classes and attributes used by the dmarc rule. + """ + class Name: + # mock answer.name + def __init__(self): + pass + + def to_text(self): + return '_dmarc.example.biz.' + + class Item: + # mock answer.items + def __init__(self): + self.strings = [b'v=DMARC1; p=reject;'] + + class Ans_e: + # mock answer element + def __init__(self): + self.rdtype = TXT + self.items = [Item()] + self.name = Name() + + class Answer: + # mock answer + def __init__(self): + self.answer = [Ans_e()] + + class Resolver: + # mock dns.resolver.Resolver class. + def __init__(self): + pass + + def query(self, domain, data_type): + if data_type != TXT: + raise NoAnswer + dparts = domain.split('.') + if len(dparts) < 3: + raise NXDOMAIN + if len(dparts) > 3: + raise NoAnswer + if dparts[0] != '_dmarc': + raise NoAnswer + if dparts[1] != 'example' or dparts[2] != 'biz': + raise NXDOMAIN + self.response = Answer() + return self + patcher = patch('dns.resolver.Resolver', Resolver) + return patcher + + +class TestDMARCRules(TestCase): + """Test organizational domain determination.""" + + layer = ConfigLayer + + def setUp(self): + self.resources = ExitStack() + self.addCleanup(self.resources.close) + # Make sure every test has a clean cache. + self.cache = {} + self.resources.enter_context( + patch('mailman.rules.dmarc.s_dict', self.cache)) + + def test_no_url(self): + dmarc._get_suffixes(None) + self.assertEqual(len(self.cache), 0) + + def test_no_data_for_domain(self): + self.assertEqual( + dmarc._get_org_dom('sub.dom.example.nxtld'), + 'example.nxtld') + + def test_domain_with_wild_card(self): + self.assertEqual( + dmarc._get_org_dom('ssub.sub.foo.kobe.jp'), + 'sub.foo.kobe.jp') + + def test_exception_to_wild_card(self): + self.assertEqual( + dmarc._get_org_dom('ssub.sub.city.kobe.jp'), + 'city.kobe.jp') + + def test_no_publicsuffix_dot_org(self): + mark = LogFileMark('mailman.error') + with patch('mailman.rules.dmarc.request.urlopen', + side_effect=URLError('no internet')): + domain = dmarc._get_org_dom('ssub.sub.city.kobe.jp') + line = mark.readline() + self.assertEqual( + line[-95:], + 'Unable to retrieve data from ' + 'https://publicsuffix.org/list/public_suffix_list.dat: ' + 'no internet\n') + self.assertEqual(domain, 'kobe.jp') diff --git a/src/mailman/runners/docs/incoming.rst b/src/mailman/runners/docs/incoming.rst index 70b870d18..33810d104 100644 --- a/src/mailman/runners/docs/incoming.rst +++ b/src/mailman/runners/docs/incoming.rst @@ -128,9 +128,9 @@ Now the message is in the pipeline queue. Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB Date: ... - X-Mailman-Rule-Misses: approved; emergency; loop; banned-address; - member-moderation; nonmember-moderation; administrivia; implicit-dest; - max-recipients; max-size; news-moderation; no-subject; + X-Mailman-Rule-Misses: dmarc-mitigation; approved; emergency; loop; + banned-address; member-moderation; nonmember-moderation; administrivia; + implicit-dest; max-recipients; max-size; news-moderation; no-subject; suspicious-header <BLANKLINE> First post! diff --git a/src/mailman/styles/base.py b/src/mailman/styles/base.py index 02a17d54d..693a2ab14 100644 --- a/src/mailman/styles/base.py +++ b/src/mailman/styles/base.py @@ -31,7 +31,7 @@ from mailman.interfaces.autorespond import ResponseAction from mailman.interfaces.bounce import UnrecognizedBounceDisposition from mailman.interfaces.digests import DigestFrequency from mailman.interfaces.mailinglist import ( - Personalization, ReplyToMunging, SubscriptionPolicy) + DMARCMitigateAction, Personalization, ReplyToMunging, SubscriptionPolicy) from mailman.interfaces.nntp import NewsgroupModeration from public import public @@ -85,6 +85,11 @@ class BasicOperation: mlist.digest_send_periodic = True mlist.digest_volume_frequency = DigestFrequency.monthly mlist.next_digest_number = 1 + # DMARC + mlist.dmarc_mitigate_action = DMARCMitigateAction.no_mitigation + mlist.dmarc_mitigate_unconditionally = False + mlist.dmarc_moderation_notice = '' + mlist.dmarc_wrapped_message_text = '' # NNTP gateway mlist.nntp_host = '' mlist.linked_newsgroup = '' |