diff options
Diffstat (limited to 'src')
23 files changed, 1340 insertions, 17 deletions
diff --git a/src/mailman/chains/builtin.py b/src/mailman/chains/builtin.py index 16de1043a..3bf0fd5b5 100644 --- a/src/mailman/chains/builtin.py +++ b/src/mailman/chains/builtin.py @@ -38,6 +38,10 @@ class BuiltInChain: description = _('The built-in moderation chain.') _link_descriptions = ( + # First check DMARC. For a reject or discard, the rule hits and we + # jump to the moderation chain to do the action. Otherwise, the rule + # misses buts sets msgdata['dmarc'] for the handler. + ('dmarc-moderation', LinkAction.jump, 'moderation'), ('approved', LinkAction.jump, 'accept'), ('emergency', LinkAction.jump, 'hold'), ('loop', LinkAction.jump, 'discard'), diff --git a/src/mailman/chains/docs/moderation.rst b/src/mailman/chains/docs/moderation.rst index 3b40c7293..53c32bc33 100644 --- a/src/mailman/chains/docs/moderation.rst +++ b/src/mailman/chains/docs/moderation.rst @@ -87,6 +87,7 @@ built-in chain. No rules hit and so the message is accepted. Subject: aardvark Hits: Misses: + dmarc-moderation approved emergency loop @@ -124,6 +125,7 @@ moderator approval. Hits: member-moderation Misses: + dmarc-moderation approved emergency loop @@ -150,6 +152,7 @@ Anne's moderation action can also be set to `discard`... Hits: member-moderation Misses: + dmarc-moderation approved emergency loop @@ -175,6 +178,7 @@ Anne's moderation action can also be set to `discard`... Hits: member-moderation Misses: + dmarc-moderation approved emergency loop @@ -215,6 +219,7 @@ moderator approval. Hits: nonmember-moderation Misses: + dmarc-moderation approved emergency loop diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg index e3ddb6f8e..91be0742d 100644 --- a/src/mailman/config/schema.cfg +++ b/src/mailman/config/schema.cfg @@ -74,6 +74,21 @@ filtered_messages_are_preservable: no # The command should print the converted text to stdout. html_to_plain_text_command: /usr/bin/lynx -dump $filename +# Parameters for DMARC DNS lookups. If you are seeing 'DNSException: +# Unable to query DMARC policy ...' entries in your error log, you may need +# to adjust these. +# The time to wait for a response from a name server before timeout. +dmarc_resolver_timeout: 3s +# The total time to spend trying to get an answer to the question. +dmarc_resolver_lifetime: 5s + +# A URL from which to retrieve the data for the algorithm that computes +# Organizational Domains for DMARC policy lookup purposes. This can be +# anything handled by the Python urllib.request.urlopen function. See +# https://publicsuffix.org/list/ for info. +dmarc_org_domain_data: https://publicsuffix.org/list/public_suffix_list.dat + + [shell] # `mailman shell` (also `withlist`) gives you an interactive prompt that you diff --git a/src/mailman/core/docs/chains.rst b/src/mailman/core/docs/chains.rst index 2b30a25e4..43b41bcf5 100644 --- a/src/mailman/core/docs/chains.rst +++ b/src/mailman/core/docs/chains.rst @@ -268,9 +268,9 @@ This message will end up in the `pipeline` queue. Message-ID: <first> Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB - X-Mailman-Rule-Misses: approved; emergency; loop; banned-address; - member-moderation; nonmember-moderation; administrivia; implicit-dest; - max-recipients; max-size; news-moderation; no-subject; + X-Mailman-Rule-Misses: dmarc-moderation; approved; emergency; loop; + banned-address; member-moderation; nonmember-moderation; administrivia; + implicit-dest; max-recipients; max-size; news-moderation; no-subject; suspicious-header <BLANKLINE> An important message. @@ -285,6 +285,7 @@ hit and all rules that have missed. administrivia approved banned-address + dmarc-moderation emergency implicit-dest loop diff --git a/src/mailman/core/pipelines.py b/src/mailman/core/pipelines.py index fb663cee9..7b4c6feb4 100644 --- a/src/mailman/core/pipelines.py +++ b/src/mailman/core/pipelines.py @@ -115,6 +115,7 @@ class PostingPipeline(BasePipeline): 'after-delivery', 'acknowledge', 'decorate', + 'dmarc', 'to-outgoing', ) diff --git a/src/mailman/database/alembic/versions/3002bac0c25a_dmarc_attributes.py b/src/mailman/database/alembic/versions/3002bac0c25a_dmarc_attributes.py new file mode 100644 index 000000000..b0c2297a0 --- /dev/null +++ b/src/mailman/database/alembic/versions/3002bac0c25a_dmarc_attributes.py @@ -0,0 +1,89 @@ +"""dmarc_attributes + +Revision ID: 3002bac0c25a +Revises: 448a93984c35 +Create Date: 2016-10-30 22:05:17.881880 + +""" + +import sqlalchemy as sa + +from alembic import op +from mailman.database.helpers import exists_in_db +from mailman.database.types import Enum, SAUnicode, SAUnicodeLarge +from mailman.interfaces.mailinglist import DMARCModerationAction, FromIsList + + +# revision identifiers, used by Alembic. +revision = '3002bac0c25a' +down_revision = '448a93984c35' + + +def upgrade(): + if not exists_in_db(op.get_bind(), + 'mailinglist', + 'dmarc_moderation_action' + ): + # SQLite may not have removed it when downgrading. It should be OK + # to just test one. + op.add_column('mailinglist', sa.Column( + 'dmarc_moderation_action', + Enum(DMARCModerationAction), + nullable=True)) + op.add_column('mailinglist', sa.Column( + 'dmarc_quarantine_moderation_action', + sa.Boolean(), + nullable=True)) + op.add_column('mailinglist', sa.Column( + 'dmarc_none_moderation_action', + sa.Boolean(), + nullable=True)) + op.add_column('mailinglist', sa.Column( + 'dmarc_moderation_notice', + SAUnicode(), + nullable=True)) + op.add_column('mailinglist', sa.Column( + 'dmarc_wrapped_message_text', + SAUnicode(), + nullable=True)) + op.add_column('mailinglist', sa.Column( + 'from_is_list', + Enum(FromIsList), + nullable=True)) + # Now migrate the data. Don't import the table definition from the + # models, it may break this migration when the model is updated in the + # future (see the Alembic doc). + mlist = sa.sql.table( + 'mailinglist', + sa.sql.column('dmarc_moderation_action', Enum(DMARCModerationAction)), + sa.sql.column('dmarc_quarantine_moderation_action', sa.Boolean()), + sa.sql.column('dmarc_none_moderation_action', sa.Boolean()), + sa.sql.column('dmarc_moderation_notice', SAUnicode()), + sa.sql.column('dmarc_wrapped_message_text', SAUnicode()), + sa.sql.column('from_is_list', Enum(FromIsList)) + ) + # These are all new attributes so just set defaults. + op.execute(mlist.update().values(dict( + dmarc_moderation_action=op.inline_literal(DMARCModerationAction.none), + dmarc_quarantine_moderation_action=op.inline_literal(True), + dmarc_none_moderation_action=op.inline_literal(False), + dmarc_moderation_notice=op.inline_literal(''), + dmarc_wrapped_message_text=op.inline_literal(''), + from_is_list=op.inline_literal(FromIsList.none), + ))) + # Adding another rule can make the rule Hits/Misses too long for MySQL + # SaUnicode. + with op.batch_alter_table('pendedkeyvalue') as batch_op: + batch_op.alter_column('value', type_=SAUnicodeLarge) + + +def downgrade(): + with op.batch_alter_table('mailinglist') as batch_op: + batch_op.drop_column('dmarc_moderation_action') + batch_op.drop_column('dmarc_quarantine_moderation_action') + batch_op.drop_column('dmarc_none_moderation_action') + batch_op.drop_column('dmarc_moderation_notice') + batch_op.drop_column('dmarc_wrapped_message_text') + batch_op.drop_column('from_is_list') + with op.batch_alter_table('pendedkeyvalue') as batch_op: + batch_op.alter_column('value', type_=SAUnicode) diff --git a/src/mailman/handlers/dmarc.py b/src/mailman/handlers/dmarc.py new file mode 100644 index 000000000..8bdce040f --- /dev/null +++ b/src/mailman/handlers/dmarc.py @@ -0,0 +1,203 @@ +# Copyright (C) 2016 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Do DMARC Munge From and Wrap Message actions.""" + +import re +import copy +import logging + +from email.header import Header, decode_header +from email.mime.message import MIMEMessage +from email.mime.text import MIMEText +from email.utils import formataddr, getaddresses, make_msgid +from mailman import public +from mailman.core.i18n import _ +from mailman.interfaces.handler import IHandler +from mailman.interfaces.mailinglist import ( + DMARCModerationAction, FromIsList, ReplyToMunging) +from mailman.utilities.string import wrap +from zope.interface import implementer + + +log = logging.getLogger('mailman.error') + +COMMASPACE = ', ' +MAXLINELEN = 78 +NONASCII = re.compile('[^\s!-~]') +# Headers from the original that we want to keep in the wrapper. These are +# actually regexps matched with re.match so they match anything that starts +# with the given string unless they end with '$'. +KEEPERS = ('archived-at', + 'date', + 'in-reply-to', + 'list-', + 'precedence', + 'references', + 'subject', + 'to', + 'x-mailman-', + ) + + +def munged_headers(mlist, msg, msgdata): + # Be as robust as possible here. + faddrs = getaddresses(msg.get_all('from', [])) + # Strip the nulls and bad emails. + faddrs = [x for x in faddrs if x[1].find('@') > 0] + if len(faddrs) == 1: + realname, email = o_from = faddrs[0] + else: + # No From: or multiple addresses. Just punt and take + # the get_sender result. + realname = '' + email = msgdata['original_sender'] + o_from = (realname, email) + if not realname: + member = mlist.members.get_member(email) + if member: + realname = member.display_name or email + else: + realname = email + # Remove domain from realname if it looks like an email address + realname = re.sub(r'@([^ .]+\.)+[^ .]+$', '---', realname) + # Make a display name and RFC 2047 encode it if necessary. This is + # difficult and kludgy. If the realname came from From: it should be + # ascii or RFC 2047 encoded. If it came from the list, it should be + # a string. If it's from the email address, it should be an ascii string. + # In any case, ensure it's an unencoded string. + srn = '' + for frag, cs in decode_header(realname): + if not cs: + # Character set should be ascii, but use iso-8859-1 anyway. + cs = 'iso-8859-1' + if not isinstance(frag, str): + srn += str(frag, cs, errors='replace') + else: + srn += frag + # The list's real_name is a string. + slrn = mlist.display_name + # get translated 'via' with dummy replacements + realname = '$realname' + lrn = '$lrn' # noqa F841 + # Ensure the i18n context is the list's preferred_language. + with _.using(mlist.preferred_language.code): + via = _('$realname via $lrn') + # Replace the dummy replacements. + via = re.sub('\$lrn', slrn, re.sub('\$realname', srn, via)) + # And get an RFC 2047 encoded header string. + dn = str(Header(via, mlist.preferred_language.charset)) + retn = [('From', formataddr((dn, mlist.posting_address)))] + # We've made the munged From:. Now put the original in Reply-To: or Cc: + if mlist.reply_goes_to_list == ReplyToMunging.no_munging: + # Add original from to Reply-To: + add_to = 'Reply-To' + else: + # Add original from to Cc: + add_to = 'Cc' + orig = getaddresses(msg.get_all(add_to, [])) + if o_from[1] not in [x[1] for x in orig]: + orig.append(o_from) + retn.append((add_to, COMMASPACE.join([formataddr(x) for x in orig]))) + return retn + + +def munge_from(mlist, msg, msgdata): + for k, v in munged_headers(mlist, msg, msgdata): + del msg[k] + msg[k] = v + return + + +def wrap_message(mlist, msg, msgdata, dmarc_wrap=False): + # Create a wrapper message around the original. + # There are various headers in msg that we don't want, so we basically + # make a copy of the msg, then delete almost everything and set/copy + # what we want. + omsg = copy.deepcopy(msg) + for key in msg.keys(): + keep = False + for keeper in KEEPERS: + if re.match(keeper, key, re.I): + keep = True + break + if not keep: + del msg[key] + msg['MIME-Version'] = '1.0' + msg['Message-ID'] = make_msgid() + for k, v in munged_headers(mlist, omsg, msgdata): + msg[k] = v + # Are we including dmarc_wrapped_message_text? I.e., do we have text and + # are we wrapping because of dmarc_moderation_action? + if mlist.dmarc_wrapped_message_text and dmarc_wrap: + part1 = MIMEText(wrap(mlist.dmarc_wrapped_message_text), + 'plain', + mlist.preferred_language.charset) + part1['Content-Disposition'] = 'inline' + part2 = MIMEMessage(omsg) + part2['Content-Disposition'] = 'inline' + msg['Content-Type'] = 'multipart/mixed' + msg.set_payload([part1, part2]) + else: + msg['Content-Type'] = 'message/rfc822' + msg['Content-Disposition'] = 'inline' + msg.set_payload([omsg]) + return + + +def process(mlist, msg, msgdata): + """Process DMARC actions.""" + if ((not msgdata.get('dmarc') or + mlist.dmarc_moderation_action == DMARCModerationAction.none) and + mlist.from_is_list == FromIsList.none): + return + if mlist.anonymous_list: + # DMARC mitigation is not required for anonymous lists. + return + if (mlist.dmarc_moderation_action != DMARCModerationAction.none and + msgdata.get('dmarc')): + if mlist.dmarc_moderation_action == DMARCModerationAction.munge_from: + munge_from(mlist, msg, msgdata) + elif (mlist.dmarc_moderation_action == + DMARCModerationAction.wrap_message): + wrap_message(mlist, msg, msgdata, dmarc_wrap=True) + else: + assert False, ( + 'handlers/dmarc.py: dmarc_moderation_action = {0}'.format( + mlist.dmarc_moderation_action)) + else: + if mlist.from_is_list == FromIsList.munge_from: + munge_from(mlist, msg, msgdata) + elif mlist.from_is_list == FromIsList.wrap_message: + wrap_message(mlist, msg, msgdata) + else: + assert False, ( + 'handlers/dmarc.py: from_is_list = {0}'.format( + mlist.from_is_list)) + + +@public +@implementer(IHandler) +class DMARC: + """Apply DMARC mitigations.""" + + name = 'dmarc' + description = _('Apply DMARC mitigations.') + + def process(self, mlist, msg, msgdata): + """See `IHandler`.""" + process(mlist, msg, msgdata) diff --git a/src/mailman/handlers/docs/dmarc-mitigations.rst b/src/mailman/handlers/docs/dmarc-mitigations.rst new file mode 100644 index 000000000..ee7dd98e0 --- /dev/null +++ b/src/mailman/handlers/docs/dmarc-mitigations.rst @@ -0,0 +1,145 @@ +================= +DMARC Mitigations +================= + +In order to mitigate the effects of DMARC on mailing list traffic, list +administrators have the ability to apply transformations to messages delivered +to list members. These transformations are applied only to individual messages +sent to list members and not to messages in digests, archives or gated to +usenet. + +The messages can be transformed by either munging the From: header and putting +original From: in Cc: or Reply-To: or by wrapping the original message in an +outer message From: the list. + +Exactly what transformations are applied depends on a number of list settings. + +The settings and their effects are: + + * anonymous_list: If True, no mitigations are ever applied because the message + is already From: the list. + * dmarc_moderation_action: The action to apply to messages From: a domain + publishing a DMARC policy of reject and possibly quarantine or none. + * dmarc_quarantine_moderation_action: A flag to apply dmarc_moderation_action + to messages From: a domain publishing a DMARC policy of quarantine. + * dmarc_none_moderation_action: A flag to apply dmarc_moderation_action to + messages From: a domain publishing a DMARC policy of none, but only when + dmarc_quarantine_moderation_action is also true. + * dmarc_moderation_notice: Text to include in any rejection notice to be sent + when dmarc_moderation_action of reject applies. + * dmarc_wrapped_message_text: Text to be added as a separate text/plain MIME + part preceding the original message part in the wrapped message when + dmarc_moderation_action of wrap_message applies. + * from_is_list: The action to be applied to all messages for which + dmarc_moderation_action is none or not applicable. + * reply_goes_to_list: If this is set to other than no_munging of Reply-To, + the original From: goes in Cc: rather than Reply-To:. This is intended to + make MUA functions of reply and reply-all have the same effect with + messages to which mitigations have been applied as they do with other + messages. + +The possible actions for both dmarc_moderation_action and from_is_list are: + + * none: Make no transformation to the message. + * munge_from: Change the From: header and put the original From: in Reply-To: + or in some cases Cc: + * wrap_message: Wrap the message in an outer message with headers as in + munge_from. + +In addition, there are two more possible actions (actually processed by the +dmarc-moderation rule) for dmarc_moderation_action only: + + * reject: Bounce the message back to the sender with a default reason or one + supplied in dmarc_moderation_notice. + * discard: Silently discard the message. + +Here's what happens when we munge the From. + + >>> from mailman.interfaces.mailinglist import (DMARCModerationAction, + ... ReplyToMunging) + >>> mlist = create_list('test@example.com') + >>> mlist.dmarc_moderation_action = DMARCModerationAction.munge_from + >>> mlist.reply_goes_to_list = ReplyToMunging.no_munging + >>> msg = message_from_string("""\ + ... From: A Person <aperson@example.com> + ... To: test@example.com + ... + ... A message of great import. + ... """) + >>> msgdata = dict(dmarc=True, original_sender='aperson@example.com') + >>> from mailman.handlers.dmarc import process + >>> process(mlist, msg, msgdata) + >>> print(msg.as_string()) + To: test@example.com + From: A Person via Test <test@example.com> + Reply-To: A Person <aperson@example.com> + <BLANKLINE> + A message of great import. + <BLANKLINE> + +Here we wrap the message without adding a text part. + + >>> mlist.dmarc_moderation_action = DMARCModerationAction.wrap_message + >>> mlist.dmarc_wrapped_message_text = '' + >>> msg = message_from_string("""\ + ... From: A Person <aperson@example.com> + ... To: test@example.com + ... + ... A message of great import. + ... """) + >>> msgdata = dict(dmarc=True, original_sender='aperson@example.com') + >>> process(mlist, msg, msgdata) + >>> print(msg.as_string()) + To: test@example.com + MIME-Version: 1.0 + Message-ID: <...> + From: A Person via Test <test@example.com> + Reply-To: A Person <aperson@example.com> + Content-Type: message/rfc822 + Content-Disposition: inline + <BLANKLINE> + From: A Person <aperson@example.com> + To: test@example.com + <BLANKLINE> + A message of great import. + <BLANKLINE> + +And here's a wrapped message with an added text part. + + >>> mlist.dmarc_wrapped_message_text = 'The original message is attached.' + >>> msg = message_from_string("""\ + ... From: A Person <aperson@example.com> + ... To: test@example.com + ... + ... A message of great import. + ... """) + >>> msgdata = dict(dmarc=True, original_sender='aperson@example.com') + >>> process(mlist, msg, msgdata) + >>> print(msg.as_string()) + To: test@example.com + MIME-Version: 1.0 + Message-ID: <...> + From: A Person via Test <test@example.com> + Reply-To: A Person <aperson@example.com> + Content-Type: multipart/mixed; boundary="..." + <BLANKLINE> + --... + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: inline + <BLANKLINE> + The original message is attached. + --... + Content-Type: message/rfc822 + MIME-Version: 1.0 + Content-Disposition: inline + <BLANKLINE> + From: A Person <aperson@example.com> + To: test@example.com + <BLANKLINE> + A message of great import. + <BLANKLINE> + --...-- + <BLANKLINE> + diff --git a/src/mailman/handlers/tests/test_dmarc.py b/src/mailman/handlers/tests/test_dmarc.py new file mode 100644 index 000000000..648fbab0e --- /dev/null +++ b/src/mailman/handlers/tests/test_dmarc.py @@ -0,0 +1,244 @@ +# Copyright (C) 2016 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test the dmarc handler.""" + +import unittest + +from mailman.app.lifecycle import create_list +from mailman.handlers import dmarc +from mailman.interfaces.mailinglist import ( + DMARCModerationAction, FromIsList, ReplyToMunging) +from mailman.testing.helpers import specialized_message_from_string as mfs +from mailman.testing.layers import ConfigLayer + + +class TestDMARCMitigations(unittest.TestCase): + """Test the dmarc handler.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('ant@example.com') + self._mlist.anonymous_list = False + self._mlist.dmarc_moderation_action = DMARCModerationAction.none + self._mlist.dmarc_wrapped_message_text = '' + self._mlist.from_is_list = FromIsList.none + self._mlist.reply_goes_to_list = ReplyToMunging.no_munging + # We can use the same message text for all tests. + self._text = """\ +From: anne@example.com +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Message-ID: <some-id@example.com> +Date: Fri, 1 Jan 2016 00:00:01 +0000 +Another-Header: To test removal in wrapper +MIME-Version: 1.0 +Content-Type: multipart/alternative; boundary="=====abc==" + +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +Some things to say. +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +<html><head></head><body>Some things to say.</body></html> +--=====abc==-- +""" + + def test_default_no_change(self): + msg = mfs(self._text) + dmarc.process(self._mlist, msg, {}) + self.assertMultiLineEqual(msg.as_string(), self._text) + + def test_anonymous_no_change(self): + self._mlist.anonymous_list = True + self._mlist.dmarc_moderation_action = DMARCModerationAction.munge_from + msgdata = {'dmarc': True} + msg = mfs(self._text) + dmarc.process(self._mlist, msg, msgdata) + self.assertMultiLineEqual(msg.as_string(), self._text) + + def test_action_munge_from(self): + self._mlist.dmarc_moderation_action = DMARCModerationAction.munge_from + msgdata = {'dmarc': True} + msg = mfs(self._text) + dmarc.process(self._mlist, msg, msgdata) + self.assertMultiLineEqual(msg.as_string(), """\ +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Message-ID: <some-id@example.com> +Date: Fri, 1 Jan 2016 00:00:01 +0000 +Another-Header: To test removal in wrapper +MIME-Version: 1.0 +Content-Type: multipart/alternative; boundary="=====abc==" +From: anne--- via Ant <ant@example.com> +Reply-To: anne@example.com + +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +Some things to say. +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +<html><head></head><body>Some things to say.</body></html> +--=====abc==-- +""") + + def test_no_action_without_msgdata(self): + self._mlist.dmarc_moderation_action = DMARCModerationAction.munge_from + msg = mfs(self._text) + dmarc.process(self._mlist, msg, {}) + self.assertMultiLineEqual(msg.as_string(), self._text) + + def test_from_is_list_no_msgdata(self): + self._mlist.from_is_list = FromIsList.munge_from + msg = mfs(self._text) + dmarc.process(self._mlist, msg, {}) + self.assertMultiLineEqual(msg.as_string(), """\ +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Message-ID: <some-id@example.com> +Date: Fri, 1 Jan 2016 00:00:01 +0000 +Another-Header: To test removal in wrapper +MIME-Version: 1.0 +Content-Type: multipart/alternative; boundary="=====abc==" +From: anne--- via Ant <ant@example.com> +Reply-To: anne@example.com + +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +Some things to say. +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +<html><head></head><body>Some things to say.</body></html> +--=====abc==-- +""") + + def test_from_in_cc(self): + self._mlist.dmarc_moderation_action = DMARCModerationAction.munge_from + self._mlist.reply_goes_to_list = ReplyToMunging.point_to_list + msgdata = {'dmarc': True} + msg = mfs(self._text) + dmarc.process(self._mlist, msg, msgdata) + self.assertMultiLineEqual(msg.as_string(), """\ +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Message-ID: <some-id@example.com> +Date: Fri, 1 Jan 2016 00:00:01 +0000 +Another-Header: To test removal in wrapper +MIME-Version: 1.0 +Content-Type: multipart/alternative; boundary="=====abc==" +From: anne--- via Ant <ant@example.com> +Cc: anne@example.com + +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +Some things to say. +--=====abc== +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +<html><head></head><body>Some things to say.</body></html> +--=====abc==-- +""") + + def test_wrap_message_1(self): + self._mlist.from_is_list = FromIsList.wrap_message + msg = mfs(self._text) + dmarc.process(self._mlist, msg, {}) + # We can't predict the Message-ID in the wrapper so delete it, but + # ensure we have one. + self.assertIsNotNone(msg.get('message-id')) + del msg['message-id'] + self.assertMultiLineEqual(msg.as_string(), """\ +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Date: Fri, 1 Jan 2016 00:00:01 +0000 +MIME-Version: 1.0 +From: anne--- via Ant <ant@example.com> +Reply-To: anne@example.com +Content-Type: message/rfc822 +Content-Disposition: inline + +""" + self._text) + + def test_wrap_message_2(self): + self._mlist.dmarc_moderation_action = ( + DMARCModerationAction.wrap_message) + msgdata = {'dmarc': True} + msg = mfs(self._text) + dmarc.process(self._mlist, msg, msgdata) + # We can't predict the Message-ID in the wrapper so delete it, but + # ensure we have one. + self.assertIsNotNone(msg.get('message-id')) + del msg['message-id'] + self.maxDiff = None + self.assertMultiLineEqual(msg.as_string(), """\ +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Date: Fri, 1 Jan 2016 00:00:01 +0000 +MIME-Version: 1.0 +From: anne--- via Ant <ant@example.com> +Reply-To: anne@example.com +Content-Type: message/rfc822 +Content-Disposition: inline + +""" + self._text) + + def test_wrap_message_cc(self): + self._mlist.dmarc_moderation_action = ( + DMARCModerationAction.wrap_message) + self._mlist.reply_goes_to_list = ReplyToMunging.point_to_list + msgdata = {'dmarc': True} + msg = mfs(self._text) + dmarc.process(self._mlist, msg, msgdata) + # We can't predict the Message-ID in the wrapper so delete it, but + # ensure we have one. + self.assertIsNotNone(msg.get('message-id')) + del msg['message-id'] + self.maxDiff = None + self.assertMultiLineEqual(msg.as_string(), """\ +To: ant@example.com +Subject: A subject +X-Mailman-Version: X.Y +Date: Fri, 1 Jan 2016 00:00:01 +0000 +MIME-Version: 1.0 +From: anne--- via Ant <ant@example.com> +Cc: anne@example.com +Content-Type: message/rfc822 +Content-Disposition: inline + +""" + self._text) diff --git a/src/mailman/interfaces/mailinglist.py b/src/mailman/interfaces/mailinglist.py index 359cd5069..09f7d0259 100644 --- a/src/mailman/interfaces/mailinglist.py +++ b/src/mailman/interfaces/mailinglist.py @@ -24,6 +24,37 @@ from zope.interface import Attribute, Interface @public +class DMARCModerationAction(Enum): + none = 0 + # No DMARC mitigations. + munge_from = 1 + # Messages From: domains with DMARC policy will have From: replaced by + # the list posting address and the original From: added to Reply-To: + # or Cc:. + wrap_message = 2 + # Messages From: domains with DMARC policy will be wrapped in an outer + # message From: the list posting address. + reject = 3 + # Messages From: domains with DMARC policy will be rejected. + discard = 4 + # Messages From: domains with DMARC policy will be discarded. + + +@public +class FromIsList(Enum): + none = 0 + # No DMARC transformations will be applied to all messages. + munge_from = 1 + # All messages will have From: replaced by the list posting address and + # the original From: added to Reply-To: except DMARCModerationAction + # wrap_message, reject or discard takes precedence if applicable. + wrap_message = 2 + # All messages will be wrapped in an outer message From: the list posting + # address except DMARCModerationAction reject or discard takes + # precedence if applicable. + + +@public class Personalization(Enum): none = 0 # Everyone gets a unique copy of the message, and there are a few more @@ -216,6 +247,39 @@ class IMailingList(Interface): def confirm_address(cookie=''): """The address used for various forms of email confirmation.""" + # DMARC attributes. + + dmarc_moderation_action = Attribute( + """The DMARCModerationAction to be applied to messages From: a + domain publishing DMARC p=reject and possibly quarantine or none. + """) + + dmarc_quarantine_moderation_action = Attribute( + """Flag to apply DMARCModerationAction to messages From: a domain + publishing DMARC p=quarantine. + """) + + dmarc_none_moderation_action = Attribute( + """Flag to apply DMARCModerationAction to messages From: a domain + publishing DMARC p=none, but only when + dmarc_quarantine_moderation_action is also true. + """) + dmarc_moderation_notice = Attribute( + """Text to include in any rejection notice to be sent when + DMARCModerationAction of reject applies. + """) + + dmarc_wrapped_message_text = Attribute( + """Text to be added as a separate text/plain MIME part preceding the + original message part in the wrapped message when + DMARCModerationAction of wrap_message applies. + """) + + from_is_list = Attribute( + """The FromIsList action to be applied to all messages for which + DMARCModerationAction is none or not applicable. + """) + # Rosters and subscriptions. owners = Attribute( diff --git a/src/mailman/model/mailinglist.py b/src/mailman/model/mailinglist.py index bd6538aaf..51d322dbc 100644 --- a/src/mailman/model/mailinglist.py +++ b/src/mailman/model/mailinglist.py @@ -32,9 +32,9 @@ from mailman.interfaces.digests import DigestFrequency from mailman.interfaces.domain import IDomainManager from mailman.interfaces.languages import ILanguageManager from mailman.interfaces.mailinglist import ( - IAcceptableAlias, IAcceptableAliasSet, IHeaderMatch, IHeaderMatchList, - IListArchiver, IListArchiverSet, IMailingList, Personalization, - ReplyToMunging, SubscriptionPolicy) + DMARCModerationAction, FromIsList, IAcceptableAlias, IAcceptableAliasSet, + IHeaderMatch, IHeaderMatchList, IListArchiver, IListArchiverSet, + IMailingList, Personalization, ReplyToMunging, SubscriptionPolicy) from mailman.interfaces.member import ( AlreadySubscribedError, MemberRole, MissingPreferredAddressError, SubscriptionEvent) @@ -127,6 +127,13 @@ class MailingList(Model): forward_unrecognized_bounces_to = Column( Enum(UnrecognizedBounceDisposition)) process_bounces = Column(Boolean) + # DMARC + dmarc_moderation_action = Column(Enum(DMARCModerationAction)) + dmarc_quarantine_moderation_action = Column(Boolean) + dmarc_none_moderation_action = Column(Boolean) + dmarc_moderation_notice = Column(SAUnicode) + dmarc_wrapped_message_text = Column(SAUnicode) + from_is_list = Column(Enum(FromIsList)) # Miscellaneous default_member_action = Column(Enum(Action)) default_nonmember_action = Column(Enum(Action)) diff --git a/src/mailman/model/pending.py b/src/mailman/model/pending.py index 5889f9cc1..60be56e8f 100644 --- a/src/mailman/model/pending.py +++ b/src/mailman/model/pending.py @@ -23,7 +23,7 @@ from lazr.config import as_timedelta from mailman.config import config from mailman.database.model import Model from mailman.database.transaction import dbconnection -from mailman.database.types import SAUnicode +from mailman.database.types import SAUnicode, SAUnicodeLarge from mailman.interfaces.pending import ( IPendable, IPended, IPendedKeyValue, IPendings) from mailman.utilities.datetime import now @@ -47,7 +47,7 @@ class PendedKeyValue(Model): id = Column(Integer, primary_key=True) key = Column(SAUnicode, index=True) - value = Column(SAUnicode, index=True) + value = Column(SAUnicodeLarge, index=True) pended_id = Column(Integer, ForeignKey('pended.id'), index=True) def __init__(self, key, value): diff --git a/src/mailman/rest/docs/listconf.rst b/src/mailman/rest/docs/listconf.rst index 3434f2a3d..da9f1e299 100644 --- a/src/mailman/rest/docs/listconf.rst +++ b/src/mailman/rest/docs/listconf.rst @@ -44,10 +44,16 @@ All readable attributes for a list are available on a sub-resource. digest_volume_frequency: monthly digests_enabled: True display_name: Ant + dmarc_moderation_action: none + dmarc_moderation_notice: + dmarc_none_moderation_action: False + dmarc_quarantine_moderation_action: True + dmarc_wrapped_message_text: filter_content: False first_strip_reply_to: False footer_uri: fqdn_listname: ant@example.com + from_is_list: none goodbye_message_uri: header_uri: http_etag: "..." @@ -110,9 +116,15 @@ When using ``PUT``, all writable attributes must be included. ... digest_size_threshold=10.5, ... digest_volume_frequency='yearly', ... digests_enabled=False, + ... dmarc_moderation_action='munge_from', + ... dmarc_moderation_notice='Some moderation notice', + ... dmarc_none_moderation_action=True, + ... dmarc_quarantine_moderation_action=False, + ... dmarc_wrapped_message_text='some message text', ... posting_pipeline='virgin', ... filter_content=True, ... first_strip_reply_to=True, + ... from_is_list='none', ... convert_html_to_plaintext=True, ... collapse_alternatives=False, ... reply_goes_to_list='point_to_list', @@ -162,6 +174,11 @@ These values are changed permanently. digest_volume_frequency: yearly digests_enabled: False display_name: Fnords + dmarc_moderation_action: munge_from + dmarc_moderation_notice: Some moderation notice + dmarc_none_moderation_action: True + dmarc_quarantine_moderation_action: False + dmarc_wrapped_message_text: some message text filter_content: True first_strip_reply_to: True footer_uri: diff --git a/src/mailman/rest/docs/systemconf.rst b/src/mailman/rest/docs/systemconf.rst index 385588077..24c6bef4a 100644 --- a/src/mailman/rest/docs/systemconf.rst +++ b/src/mailman/rest/docs/systemconf.rst @@ -14,6 +14,9 @@ You can also get all the values for a particular section. >>> dump_json('http://localhost:9001/3.0/system/configuration/mailman') cache_life: 7d default_language: en + dmarc_org_domain_data: https://publicsuffix.org/list/public_suffix_list.dat + dmarc_resolver_lifetime: 5s + dmarc_resolver_timeout: 3s email_commands_max_lines: 10 filtered_messages_are_preservable: no html_to_plain_text_command: /usr/bin/lynx -dump $filename diff --git a/src/mailman/rest/listconf.py b/src/mailman/rest/listconf.py index 94598b0ab..416336142 100644 --- a/src/mailman/rest/listconf.py +++ b/src/mailman/rest/listconf.py @@ -24,7 +24,8 @@ from mailman.interfaces.archiver import ArchivePolicy from mailman.interfaces.autorespond import ResponseAction from mailman.interfaces.digests import DigestFrequency from mailman.interfaces.mailinglist import ( - IAcceptableAliasSet, IMailingList, ReplyToMunging, SubscriptionPolicy) + DMARCModerationAction, FromIsList, IAcceptableAliasSet, IMailingList, + ReplyToMunging, SubscriptionPolicy) from mailman.interfaces.template import ITemplateManager from mailman.rest.helpers import ( GetterSetter, bad_request, etag, no_content, not_found, okay) @@ -128,7 +129,9 @@ ATTRIBUTES = dict( admin_notify_mchanges=GetterSetter(as_boolean), administrivia=GetterSetter(as_boolean), advertised=GetterSetter(as_boolean), + allow_list_posts=GetterSetter(as_boolean), anonymous_list=GetterSetter(as_boolean), + archive_policy=GetterSetter(enum_validator(ArchivePolicy)), autorespond_owner=GetterSetter(enum_validator(ResponseAction)), autorespond_postings=GetterSetter(enum_validator(ResponseAction)), autorespond_requests=GetterSetter(enum_validator(ResponseAction)), @@ -136,7 +139,6 @@ ATTRIBUTES = dict( autoresponse_owner_text=GetterSetter(str), autoresponse_postings_text=GetterSetter(str), autoresponse_request_text=GetterSetter(str), - archive_policy=GetterSetter(enum_validator(ArchivePolicy)), bounces_address=GetterSetter(None), collapse_alternatives=GetterSetter(as_boolean), convert_html_to_plaintext=GetterSetter(as_boolean), @@ -144,22 +146,29 @@ ATTRIBUTES = dict( default_member_action=GetterSetter(enum_validator(Action)), default_nonmember_action=GetterSetter(enum_validator(Action)), description=GetterSetter(no_newlines_validator), + display_name=GetterSetter(str), digest_last_sent_at=GetterSetter(None), digest_send_periodic=GetterSetter(as_boolean), digest_size_threshold=GetterSetter(float), digest_volume_frequency=GetterSetter(enum_validator(DigestFrequency)), digests_enabled=GetterSetter(as_boolean), + dmarc_moderation_action=GetterSetter( + enum_validator(DMARCModerationAction)), + dmarc_moderation_notice=GetterSetter(str), + dmarc_none_moderation_action=GetterSetter(as_boolean), + dmarc_quarantine_moderation_action=GetterSetter(as_boolean), + dmarc_wrapped_message_text=GetterSetter(str), filter_content=GetterSetter(as_boolean), first_strip_reply_to=GetterSetter(as_boolean), fqdn_listname=GetterSetter(None), - mail_host=GetterSetter(None), - allow_list_posts=GetterSetter(as_boolean), + from_is_list=GetterSetter(enum_validator(FromIsList)), include_rfc2369_headers=GetterSetter(as_boolean), info=GetterSetter(str), join_address=GetterSetter(None), last_post_at=GetterSetter(None), leave_address=GetterSetter(None), list_name=GetterSetter(None), + mail_host=GetterSetter(None), moderator_password=GetterSetter(password_bytes_validator), next_digest_number=GetterSetter(None), no_reply_address=GetterSetter(None), @@ -167,7 +176,6 @@ ATTRIBUTES = dict( post_id=GetterSetter(None), posting_address=GetterSetter(None), posting_pipeline=GetterSetter(pipeline_validator), - display_name=GetterSetter(str), reply_goes_to_list=GetterSetter(enum_validator(ReplyToMunging)), reply_to_address=GetterSetter(str), request_address=GetterSetter(None), diff --git a/src/mailman/rest/tests/test_listconf.py b/src/mailman/rest/tests/test_listconf.py index 3260d6afb..d9eb91c02 100644 --- a/src/mailman/rest/tests/test_listconf.py +++ b/src/mailman/rest/tests/test_listconf.py @@ -62,8 +62,14 @@ RESOURCE = dict( digest_volume_frequency='monthly', digests_enabled=True, display_name='Fnords', + dmarc_moderation_action='munge_from', + dmarc_moderation_notice='Some moderation notice', + dmarc_none_moderation_action=True, + dmarc_quarantine_moderation_action=False, + dmarc_wrapped_message_text='some message text', filter_content=True, first_strip_reply_to=True, + from_is_list='none', goodbye_message_uri='mailman:///goodbye.txt', include_rfc2369_headers=False, info='This is the mailing list info', diff --git a/src/mailman/rest/tests/test_systemconf.py b/src/mailman/rest/tests/test_systemconf.py index e76c082df..28fa315a9 100644 --- a/src/mailman/rest/tests/test_systemconf.py +++ b/src/mailman/rest/tests/test_systemconf.py @@ -39,6 +39,10 @@ class TestSystemConfiguration(unittest.TestCase): self.assertEqual(json, dict( cache_life='7d', default_language='en', + dmarc_org_domain_data= # noqa E251 + 'https://publicsuffix.org/list/public_suffix_list.dat', + dmarc_resolver_lifetime='5s', + dmarc_resolver_timeout='3s', email_commands_max_lines='10', filtered_messages_are_preservable='no', html_to_plain_text_command='/usr/bin/lynx -dump $filename', diff --git a/src/mailman/rules/dmarc.py b/src/mailman/rules/dmarc.py new file mode 100644 index 000000000..d7e826fe8 --- /dev/null +++ b/src/mailman/rules/dmarc.py @@ -0,0 +1,270 @@ +# Copyright (C) 2016 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""DMARC moderation rule.""" + +import re +import logging +import dns.resolver + +from dns.exception import DNSException +from email.utils import parseaddr +from lazr.config import as_timedelta +from mailman import public +from mailman.config import config +from mailman.core.i18n import _ +from mailman.interfaces.mailinglist import DMARCModerationAction +from mailman.interfaces.rules import IRule +from mailman.utilities.string import wrap +from urllib import error, request +from zope.interface import implementer + + +elog = logging.getLogger('mailman.error') +vlog = logging.getLogger('mailman.vette') +s_dict = dict() + + +def _get_suffixes(url): + # This loads and parses the data from the url argument into s_dict for + # use by _get_org_dom. + global s_dict + if s_dict: + return + if not url: + return + try: + d = request.urlopen(url) + except error.URLError as e: + elog.error('Unable to retrieve data from %s: %s', url, e) + return + for line in d.readlines(): + line = str(line, encoding='utf-8') + if not line.strip() or line.startswith(' ') or line.startswith('//'): + continue + line = re.sub(' .*', '', line.strip()) + if not line: + continue + parts = line.lower().split('.') + if parts[0].startswith('!'): + exc = True + parts = [parts[0][1:]] + parts[1:] + else: + exc = False + parts.reverse() + k = '.'.join(parts) + s_dict[k] = exc + + +def _get_dom(d, l): + # A helper to get a domain name consisting of the first l+1 labels + # in d. + dom = d[:min(l+1, len(d))] + dom.reverse() + return '.'.join(dom) + + +def _get_org_dom(domain): + # Given a domain name, this returns the corresponding Organizational + # Domain which may be the same as the input. + global s_dict + if not s_dict: + _get_suffixes(config.mailman.dmarc_org_domain_data) + hits = [] + d = domain.lower().split('.') + d.reverse() + for k in s_dict.keys(): + ks = k.split('.') + if len(d) >= len(ks): + for i in range(len(ks)-1): + if d[i] != ks[i] and ks[i] != '*': + break + else: + if d[len(ks)-1] == ks[-1] or ks[-1] == '*': + hits.append(k) + if not hits: + return _get_dom(d, 1) + l = 0 + for k in hits: + if s_dict[k]: + # It's an exception + return _get_dom(d, len(k.split('.'))-1) + if len(k.split('.')) > l: + l = len(k.split('.')) + return _get_dom(d, l) + + +def _IsDMARCProhibited(mlist, email): + # This takes an email address, and returns True if DMARC policy is + # p=reject or possibly quarantine or none. + email = email.lower() + # Scan from the right in case quoted local part has an '@'. + at_sign = email.rfind('@') + if at_sign < 1: + return False + f_dom = email[at_sign+1:] + x = _DMARCProhibited(mlist, email, '_dmarc.' + f_dom) + if x != 'continue': + return x + o_dom = _get_org_dom(f_dom) + if o_dom != f_dom: + x = _DMARCProhibited(mlist, email, '_dmarc.' + o_dom, org=True) + if x != 'continue': + return x + return False + + +def _DMARCProhibited(mlist, email, dmarc_domain, org=False): + + try: + resolver = dns.resolver.Resolver() + resolver.timeout = as_timedelta( + config.mailman.dmarc_resolver_timeout).total_seconds() + resolver.lifetime = as_timedelta( + config.mailman.dmarc_resolver_lifetime).total_seconds() + txt_recs = resolver.query(dmarc_domain, dns.rdatatype.TXT) + except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): + return 'continue' + except DNSException as e: + elog.error( + 'DNSException: Unable to query DMARC policy for %s (%s). %s', + email, dmarc_domain, e.__doc__) + return 'continue' + else: + # Be as robust as possible in parsing the result. + results_by_name = {} + cnames = {} + want_names = set([dmarc_domain + '.']) + for txt_rec in txt_recs.response.answer: + if txt_rec.rdtype == dns.rdatatype.CNAME: + cnames[txt_rec.name.to_text()] = ( + txt_rec.items[0].target.to_text()) + if txt_rec.rdtype != dns.rdatatype.TXT: + continue + results_by_name.setdefault( + txt_rec.name.to_text(), []).append( + "".join( + [str(x, encoding='utf-8') + for x in txt_rec.items[0].strings])) + expands = list(want_names) + seen = set(expands) + while expands: + item = expands.pop(0) + if item in cnames: + if cnames[item] in seen: + continue # cname loop + expands.append(cnames[item]) + seen.add(cnames[item]) + want_names.add(cnames[item]) + want_names.discard(item) + + if len(want_names) != 1: + elog.error( + """multiple DMARC entries in results for %s, + processing each to be strict""", + dmarc_domain) + for name in want_names: + if name not in results_by_name: + continue + dmarcs = [x for x in results_by_name[name] + if x.startswith('v=DMARC1;')] + if len(dmarcs) == 0: + return 'continue' + if len(dmarcs) > 1: + elog.error( + """RRset of TXT records for %s has %d v=DMARC1 entries; + testing them all""", + dmarc_domain, len(dmarcs)) + for entry in dmarcs: + mo = re.search(r'\bsp=(\w*)\b', entry, re.IGNORECASE) + if org and mo: + policy = mo.group(1).lower() + else: + mo = re.search(r'\bp=(\w*)\b', entry, re.IGNORECASE) + if mo: + policy = mo.group(1).lower() + else: + continue + if policy == 'reject': + vlog.info( + """%s: DMARC lookup for %s (%s) + found p=reject in %s = %s""", + mlist.list_name, email, dmarc_domain, name, entry) + return True + + if (mlist.dmarc_quarantine_moderation_action and + policy == 'quarantine'): + vlog.info( + """%s: DMARC lookup for %s (%s) + found p=quarantine in %s = %s""", + mlist.list_name, email, dmarc_domain, name, entry) + return True + + if (mlist.dmarc_none_moderation_action and + mlist.dmarc_quarantine_moderation_action and + mlist.dmarc_moderation_action in ( + DMARCModerationAction.munge_from, + DMARCModerationAction.wrap_message) and + policy == 'none'): + vlog.info( + '%s: DMARC lookup for %s (%s) found p=none in %s = %s', + mlist.list_name, email, dmarc_domain, name, entry) + return True + + return False + + +@public +@implementer(IRule) +class DMARCModeration: + """The DMARC moderation rule.""" + + name = 'dmarc-moderation' + description = _('Find DMARC policy of From: domain.') + record = True + + def check(self, mlist, msg, msgdata): + """See `IRule`.""" + if mlist.dmarc_moderation_action == DMARCModerationAction.none: + # Don't bother to check if we're not going to do anything. + return False + dn, addr = parseaddr(msg.get('from')) + if _IsDMARCProhibited(mlist, addr): + # If dmarc_moderation_action is discard or reject, this rule fires + # and jumps to the 'moderation' chain to do the actual discard. + # Otherwise, the rule misses but sets a flag for the dmarc handler + # to do the appropriate action. + msgdata['dmarc'] = True + if mlist.dmarc_moderation_action == DMARCModerationAction.discard: + msgdata['moderation_action'] = 'discard' + msgdata['moderation_reasons'] = [_('DMARC moderation')] + elif mlist.dmarc_moderation_action == DMARCModerationAction.reject: + listowner = mlist.owner_address # noqa F841 + reason = (mlist.dmarc_moderation_notice or + _('You are not allowed to post to this mailing ' + 'list From: a domain which publishes a DMARC ' + 'policy of reject or quarantine, and your message' + ' has been automatically rejected. If you think ' + 'that your messages are being rejected in error, ' + 'contact the mailing list owner at ${listowner}.')) + msgdata['moderation_reasons'] = [wrap(reason)] + msgdata['moderation_action'] = 'reject' + else: + return False + msgdata['moderation_sender'] = addr + return True + return False diff --git a/src/mailman/rules/docs/dmarc-moderation.rst b/src/mailman/rules/docs/dmarc-moderation.rst new file mode 100644 index 000000000..8aab08161 --- /dev/null +++ b/src/mailman/rules/docs/dmarc-moderation.rst @@ -0,0 +1,147 @@ +================ +DMARC moderation +================ + +This rule only matches in order to jump to the moderation chain to reject +or discard the message. The rule looks at the list's dmarc_moderation_policy +and if it is other than 'none', it checks the domain of the From: address for +a DMARC policy and depending on settings may reject or discard the message or +just flag it for the dmarc handler to apply DMARC mitigations to the message. + + >>> mlist = create_list('_xtest@example.com') + >>> rule = config.rules['dmarc-moderation'] + >>> print(rule.name) + dmarc-moderation + +First we set up a mock patcher to return predictable responses to DNS lookups. +This returns p=reject for the example.biz domain and not for any others. + + >>> from mailman.rules.tests.test_dmarc import get_dns_resolver + >>> patcher = get_dns_resolver() + +A message From: a domain without a DMARC policy does not set any flags. + + >>> from mailman.interfaces.mailinglist import DMARCModerationAction + >>> mlist.dmarc_moderation_action = DMARCModerationAction.munge_from + >>> msg = message_from_string("""\ + ... From: aperson@example.org + ... To: _xtest@example.com + ... Subject: A posted message + ... + ... """) + >>> msgdata = {} + >>> with patcher as Resolver: + ... rule.check(mlist, msg, msgdata) + False + >>> msgdata == {} + True + +Even if the From: domain publishes p=reject, no flags are set if the list's +action is none. + + >>> mlist.dmarc_moderation_action = DMARCModerationAction.none + >>> msg = message_from_string("""\ + ... From: aperson@example.biz + ... To: _xtest@example.com + ... Subject: A posted message + ... + ... """) + >>> msgdata = {} + >>> with patcher as Resolver: + ... rule.check(mlist, msg, msgdata) + False + >>> msgdata == {} + True + +But with a different list setting, the message is flagged. + + >>> mlist.dmarc_moderation_action = DMARCModerationAction.munge_from + >>> msg = message_from_string("""\ + ... From: aperson@example.biz + ... To: _xtest@example.com + ... Subject: A posted message + ... + ... """) + >>> msgdata = {} + >>> with patcher as Resolver: + ... rule.check(mlist, msg, msgdata) + False + >>> msgdata['dmarc'] + True + +Subdomains which don't have a policy will check the organizational domain. + + >>> msg = message_from_string("""\ + ... From: aperson@sub.domain.example.biz + ... To: _xtest@example.com + ... Subject: A posted message + ... + ... """) + >>> msgdata = {} + >>> with patcher as Resolver: + ... rule.check(mlist, msg, msgdata) + False + >>> msgdata['dmarc'] + True + +The list's action can also be set to immediately discard or reject the +message. + + >>> mlist.dmarc_moderation_action = DMARCModerationAction.discard + >>> msg = message_from_string("""\ + ... From: aperson@example.biz + ... To: _xtest@example.com + ... Subject: A posted message + ... Message-ID: <xxx_message_id@example.biz> + ... + ... """) + >>> msgdata = {} + >>> with patcher as Resolver: + ... rule.check(mlist, msg, msgdata) + True + >>> msgdata['dmarc'] + True + >>> msgdata['moderation_action'] + 'discard' + +We can reject the message with a default reason. + + >>> mlist.dmarc_moderation_action = DMARCModerationAction.reject + >>> msg = message_from_string("""\ + ... From: aperson@example.biz + ... To: _xtest@example.com + ... Subject: A posted message + ... Message-ID: <xxx_message_id@example.biz> + ... + ... """) + >>> msgdata = {} + >>> with patcher as Resolver: + ... rule.check(mlist, msg, msgdata) + True + >>> msgdata['dmarc'] + True + >>> msgdata['moderation_action'] + 'reject' + >>> msgdata['moderation_reasons'] + ['You are not allowed to post to this mailing list From: a domain ... + +And, we can reject with a custom message. + + >>> mlist.dmarc_moderation_notice = 'A silly reason' + >>> msg = message_from_string("""\ + ... From: aperson@example.biz + ... To: _xtest@example.com + ... Subject: A posted message + ... Message-ID: <xxx_message_id@example.biz> + ... + ... """) + >>> msgdata = {} + >>> with patcher as Resolver: + ... rule.check(mlist, msg, msgdata) + True + >>> msgdata['dmarc'] + True + >>> msgdata['moderation_action'] + 'reject' + >>> msgdata['moderation_reasons'] + ['A silly reason'] diff --git a/src/mailman/rules/docs/rules.rst b/src/mailman/rules/docs/rules.rst index 812486b45..e00889061 100644 --- a/src/mailman/rules/docs/rules.rst +++ b/src/mailman/rules/docs/rules.rst @@ -22,6 +22,7 @@ names to rule objects. any True approved True banned-address True + dmarc-moderation True emergency True implicit-dest True loop True diff --git a/src/mailman/rules/tests/test_dmarc.py b/src/mailman/rules/tests/test_dmarc.py new file mode 100644 index 000000000..49aeea8c6 --- /dev/null +++ b/src/mailman/rules/tests/test_dmarc.py @@ -0,0 +1,81 @@ +# Copyright (C) 2016 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Support for mocking dnspython calls from dmarc rules.""" + +from dns.rdatatype import TXT +from dns.resolver import NXDOMAIN, NoAnswer +from mailman import public +from unittest import mock + + +@public +def get_dns_resolver(): + """Create a dns.resolver.Resolver mock. + + This is used to return a predictable response to a _dmarc query. It + returns p=reject for the example.biz domain and raises either NXDOMAIN + or NoAnswer for any other. + + It only implements those classes and attributes used by the dmarc rule. + """ + class Name: + # mock answer.name + def __init__(self): + pass + + def to_text(self): + return '_dmarc.example.biz.' + + class Item: + # mock answer.items + def __init__(self): + self.strings = [b'v=DMARC1; p=reject;'] + + class Ans_e: + # mock answer element + def __init__(self): + self.rdtype = TXT + self.items = [Item()] + self.name = Name() + + class Answer: + # mock answer + def __init__(self): + self.answer = [Ans_e()] + + class Resolver: + # mock dns.resolver.Resolver class. + def __init__(self): + pass + + def query(self, domain, data_type): + if data_type != TXT: + raise NoAnswer + dparts = domain.split('.') + if len(dparts) < 3: + raise NXDOMAIN + if len(dparts) > 3: + raise NoAnswer + if dparts[0] != '_dmarc': + raise NoAnswer + if dparts[1] != 'example' or dparts[2] != 'biz': + raise NXDOMAIN + self.response = Answer() + return self + patcher = mock.patch('dns.resolver.Resolver', Resolver) + return patcher diff --git a/src/mailman/runners/docs/incoming.rst b/src/mailman/runners/docs/incoming.rst index 70b870d18..18b723ab4 100644 --- a/src/mailman/runners/docs/incoming.rst +++ b/src/mailman/runners/docs/incoming.rst @@ -128,9 +128,9 @@ Now the message is in the pipeline queue. Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB Date: ... - X-Mailman-Rule-Misses: approved; emergency; loop; banned-address; - member-moderation; nonmember-moderation; administrivia; implicit-dest; - max-recipients; max-size; news-moderation; no-subject; + X-Mailman-Rule-Misses: dmarc-moderation; approved; emergency; loop; + banned-address; member-moderation; nonmember-moderation; administrivia; + implicit-dest; max-recipients; max-size; news-moderation; no-subject; suspicious-header <BLANKLINE> First post! diff --git a/src/mailman/styles/base.py b/src/mailman/styles/base.py index 02a17d54d..9530b0ea3 100644 --- a/src/mailman/styles/base.py +++ b/src/mailman/styles/base.py @@ -31,7 +31,8 @@ from mailman.interfaces.autorespond import ResponseAction from mailman.interfaces.bounce import UnrecognizedBounceDisposition from mailman.interfaces.digests import DigestFrequency from mailman.interfaces.mailinglist import ( - Personalization, ReplyToMunging, SubscriptionPolicy) + DMARCModerationAction, FromIsList, Personalization, ReplyToMunging, + SubscriptionPolicy) from mailman.interfaces.nntp import NewsgroupModeration from public import public @@ -85,6 +86,13 @@ class BasicOperation: mlist.digest_send_periodic = True mlist.digest_volume_frequency = DigestFrequency.monthly mlist.next_digest_number = 1 + # DMARC + mlist.dmarc_moderation_action = DMARCModerationAction.none + mlist.dmarc_quarantine_moderation_action = True + mlist.dmarc_none_moderation_action = False + mlist.dmarc_moderation_notice = '' + mlist.dmarc_wrapped_message_text = '' + mlist.from_is_list = FromIsList.none # NNTP gateway mlist.nntp_host = '' mlist.linked_newsgroup = '' |