diff options
| -rw-r--r-- | setup.py | 1 | ||||
| -rw-r--r-- | src/mailman/chains/builtin.py | 2 | ||||
| -rw-r--r-- | src/mailman/config/schema.cfg | 15 | ||||
| -rw-r--r-- | src/mailman/core/pipelines.py | 1 | ||||
| -rw-r--r-- | src/mailman/database/alembic/versions/3002bac0c25a_dmarc_attributes.py | 83 | ||||
| -rw-r--r-- | src/mailman/handlers/dmarc.py | 206 | ||||
| -rw-r--r-- | src/mailman/handlers/docs/dmarc-mitigations.rst | 142 | ||||
| -rw-r--r-- | src/mailman/interfaces/mailinglist.py | 64 | ||||
| -rw-r--r-- | src/mailman/model/mailinglist.py | 13 | ||||
| -rw-r--r-- | src/mailman/rest/docs/systemconf.rst | 3 | ||||
| -rw-r--r-- | src/mailman/rest/tests/test_systemconf.py | 4 | ||||
| -rw-r--r-- | src/mailman/rules/dmarc.py | 269 | ||||
| -rw-r--r-- | src/mailman/rules/docs/dmarc-moderation.rst | 210 | ||||
| -rw-r--r-- | src/mailman/rules/docs/rules.rst | 1 | ||||
| -rw-r--r-- | src/mailman/styles/base.py | 10 |
15 files changed, 1020 insertions, 4 deletions
@@ -105,6 +105,7 @@ case second `m'. Any other spelling is incorrect.""", }, install_requires = [ 'alembic', + 'dnspython>=1.14.0', 'falcon>=1.0.0rc1', 'flufl.bounce', 'flufl.i18n', diff --git a/src/mailman/chains/builtin.py b/src/mailman/chains/builtin.py index b805fca0f..f31fb6ed2 100644 --- a/src/mailman/chains/builtin.py +++ b/src/mailman/chains/builtin.py @@ -38,6 +38,8 @@ class BuiltInChain: description = _('The built-in moderation chain.') _link_descriptions = ( + # First check DMARC and maybe reject or discard. + ('dmarc-moderation', LinkAction.defer, None), ('approved', LinkAction.jump, 'accept'), ('emergency', LinkAction.jump, 'hold'), ('loop', LinkAction.jump, 'discard'), diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg index e3ddb6f8e..91be0742d 100644 --- a/src/mailman/config/schema.cfg +++ b/src/mailman/config/schema.cfg @@ -74,6 +74,21 @@ filtered_messages_are_preservable: no # The command should print the converted text to stdout. html_to_plain_text_command: /usr/bin/lynx -dump $filename +# Parameters for DMARC DNS lookups. If you are seeing 'DNSException: +# Unable to query DMARC policy ...' entries in your error log, you may need +# to adjust these. +# The time to wait for a response from a name server before timeout. +dmarc_resolver_timeout: 3s +# The total time to spend trying to get an answer to the question. +dmarc_resolver_lifetime: 5s + +# A URL from which to retrieve the data for the algorithm that computes +# Organizational Domains for DMARC policy lookup purposes. This can be +# anything handled by the Python urllib.request.urlopen function. See +# https://publicsuffix.org/list/ for info. +dmarc_org_domain_data: https://publicsuffix.org/list/public_suffix_list.dat + + [shell] # `mailman shell` (also `withlist`) gives you an interactive prompt that you diff --git a/src/mailman/core/pipelines.py b/src/mailman/core/pipelines.py index 5df21ed15..8b55daf57 100644 --- a/src/mailman/core/pipelines.py +++ b/src/mailman/core/pipelines.py @@ -115,6 +115,7 @@ class PostingPipeline(BasePipeline): 'after-delivery', 'acknowledge', 'decorate', + 'dmarc', 'to-outgoing', ) diff --git a/src/mailman/database/alembic/versions/3002bac0c25a_dmarc_attributes.py b/src/mailman/database/alembic/versions/3002bac0c25a_dmarc_attributes.py new file mode 100644 index 000000000..06fcecf0a --- /dev/null +++ b/src/mailman/database/alembic/versions/3002bac0c25a_dmarc_attributes.py @@ -0,0 +1,83 @@ +"""dmarc_attributes + +Revision ID: 3002bac0c25a +Revises: 448a93984c35 +Create Date: 2016-10-30 22:05:17.881880 + +""" + +import sqlalchemy as sa + +from alembic import op +from mailman.database.helpers import exists_in_db +from mailman.database.types import Enum, SAUnicode +from mailman.interfaces.mailinglist import DMARCModerationAction, FromIsList + + +# revision identifiers, used by Alembic. +revision = '3002bac0c25a' +down_revision = '448a93984c35' + + +def upgrade(): + if not exists_in_db(op.get_bind(), + 'mailinglist', + 'dmarc_moderation_action' + ): + # SQLite may not have removed it when downgrading. It should be OK + # to just test one. + op.add_column('mailinglist', sa.Column( + 'dmarc_moderation_action', + Enum(DMARCModerationAction), + nullable=True)) + op.add_column('mailinglist', sa.Column( + 'dmarc_quarantine_moderation_action', + sa.Boolean(), + nullable=True)) + op.add_column('mailinglist', sa.Column( + 'dmarc_none_moderation_action', + sa.Boolean(), + nullable=True)) + op.add_column('mailinglist', sa.Column( + 'dmarc_moderation_notice', + SAUnicode(), + nullable=True)) + op.add_column('mailinglist', sa.Column( + 'dmarc_wrapped_message_text', + SAUnicode(), + nullable=True)) + op.add_column('mailinglist', sa.Column( + 'from_is_list', + Enum(FromIsList), + nullable=True)) + # Now migrate the data. Don't import the table definition from the + # models, it may break this migration when the model is updated in the + # future (see the Alembic doc). + mlist = sa.sql.table( + 'mailinglist', + sa.sql.column('dmarc_moderation_action', Enum(DMARCModerationAction)), + sa.sql.column('dmarc_quarantine_moderation_action', sa.Boolean()), + sa.sql.column('dmarc_none_moderation_action', sa.Boolean()), + sa.sql.column('dmarc_moderation_notice', SAUnicode()), + sa.sql.column('dmarc_wrapped_message_text', SAUnicode()), + sa.sql.column('from_is_list', Enum(FromIsList)) + ) + # These are all new attributes so just set defaults. + op.execute(mlist.update().values(dict( + dmarc_moderation_action=op.inline_literal(DMARCModerationAction.none), + dmarc_quarantine_moderation_action=op.inline_literal(True), + dmarc_none_moderation_action=op.inline_literal(False), + dmarc_moderation_notice=op.inline_literal(''), + dmarc_wrapped_message_text=op.inline_literal(''), + from_is_list=op.inline_literal(FromIsList.none), + ))) + + +def downgrade(): + with op.batch_alter_table('mailinglist') as batch_op: + batch_op.drop_column('dmarc_moderation_action') + batch_op.drop_column('dmarc_quarantine_moderation_action') + batch_op.drop_column('dmarc_none_moderation_action') + batch_op.drop_column('dmarc_moderation_notice') + batch_op.drop_column('dmarc_wrapped_message_text') + batch_op.drop_column('from_is_list') diff --git a/src/mailman/handlers/dmarc.py b/src/mailman/handlers/dmarc.py new file mode 100644 index 000000000..29a90565b --- /dev/null +++ b/src/mailman/handlers/dmarc.py @@ -0,0 +1,206 @@ +# Copyright (C) 2016 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Do DMARC Munge From and Wrap Message actions.""" + +import re +import copy +import logging + +from email.header import Header +from email.mime.message import MIMEMessage +from email.mime.text import MIMEText +from email.utils import formataddr, getaddresses, make_msgid +from mailman import public +from mailman.core.i18n import _ +from mailman.interfaces.handler import IHandler +from mailman.interfaces.mailinglist import ( + DMARCModerationAction, FromIsList, ReplyToMunging) +from mailman.utilities.string import wrap +from zope.interface import implementer + + +log = logging.getLogger('mailman.error') + +COMMASPACE = ', ' +MAXLINELEN = 78 +NONASCII = re.compile('[^\s!-~]') +# Headers from the original that we want to keep in the wrapper. These are +# actually regexps matched with re.match so they match anything that starts +# with the given string unless they end with '$'. +KEEPERS = ('archived-at', + 'date', + 'in-reply-to', + 'list-', + 'precedence', + 'references', + 'to', + 'x-mailman-', + ) + + +@public +def uheader(mlist, s, header_name=None, continuation_ws='\t', maxlinelen=None): + """Get the charset to encode the string in. + + Then search if there is any non-ascii character is in the string. If + there is and the charset is us-ascii then we use iso-8859-1 instead. If + the string is ascii only we use 'us-ascii' if another charset is + specified. + + If the header contains a newline, truncate it (see GL#273). + """ + charset = mlist.preferred_language.charset + if NONASCII.search(s): + # use list charset but ... + if charset == 'us-ascii': + charset = 'iso-8859-1' + else: + # there is no non-ascii so ... + charset = 'us-ascii' + if '\n' in s: + s = '{} [...]'.format(s.split('\n')[0]) + log.warning('Header {} contains a newline, truncating it.'.format( + header_name, s)) + return Header(s, charset, maxlinelen, header_name, continuation_ws) + + +def munged_headers(mlist, msg, msgdata): + # Be as robust as possible here. + faddrs = getaddresses(msg.get_all('from', [])) + # Strip the nulls and bad emails. + faddrs = [x for x in faddrs if x[1].find('@') > 0] + if len(faddrs) == 1: + realname, email = o_from = faddrs[0] + else: + # No From: or multiple addresses. Just punt and take + # the get_sender result. + realname = '' + email = msgdata['original_sender'] + o_from = (realname, email) + if not realname: + member = mlist.members.get_member(email) + if member: + realname = member.display_name or email + else: + realname = email + # Remove domain from realname if it looks like an email address + realname = re.sub(r'@([^ .]+\.)+[^ .]+$', '---', realname) + # RFC 2047 encode realname if necessary. + realname = str(uheader(mlist, realname)) + lrn = mlist.display_name # noqa F841 + retn = [('From', formataddr((_('$realname via $lrn'), + mlist.posting_address)))] + # We've made the munged From:. Now put the original in Reply-To: or Cc: + if mlist.reply_goes_to_list == ReplyToMunging.no_munging: + # Add original from to Reply-To: + add_to = 'Reply-To' + else: + # Add original from to Cc: + add_to = 'Cc' + orig = getaddresses(msg.get_all(add_to, [])) + if o_from[1] not in [x[1] for x in orig]: + orig.append(o_from) + retn.append((add_to, COMMASPACE.join([formataddr(x) for x in orig]))) + return retn + + +def munge_from(mlist, msg, msgdata): + for k, v in munged_headers(mlist, msg, msgdata): + del msg[k] + msg[k] = v + return + + +def wrap_message(mlist, msg, msgdata, dmarc_wrap=False): + # Create a wrapper message around the original. + # There are various headers in msg that we don't want, so we basically + # make a copy of the msg, then delete almost everything and set/copy + # what we want. + omsg = copy.deepcopy(msg) + for key in msg.keys(): + keep = False + for keeper in KEEPERS: + if re.match(keeper, key, re.I): + keep = True + break + if not keep: + del msg[key] + msg['MIME-Version'] = '1.0' + msg['Message-ID'] = make_msgid() + for k, v in munged_headers(mlist, omsg, msgdata): + msg[k] = v + # Are we including dmarc_wrapped_message_text? I.e., do we have text and + # are we wrapping because of dmarc_moderation_action? + if mlist.dmarc_wrapped_message_text and dmarc_wrap: + part1 = MIMEText(wrap(mlist.dmarc_wrapped_message_text), + 'plain', + mlist.preferred_language.charset) + part1['Content-Disposition'] = 'inline' + part2 = MIMEMessage(omsg) + part2['Content-Disposition'] = 'inline' + msg['Content-Type'] = 'multipart/mixed' + msg.set_payload([part1, part2]) + else: + msg['Content-Type'] = 'message/rfc822' + msg['Content-Disposition'] = 'inline' + msg.set_payload([omsg]) + return + + +def process(mlist, msg, msgdata): + """Process DMARC actions.""" + if ((not msgdata.get('dmarc') or + mlist.dmarc_moderation_action == DMARCModerationAction.none) and + mlist.from_is_list == FromIsList.none): + return + if mlist.anonymous_list: + # DMARC mitigation is not required for anonymous lists. + return + if (mlist.dmarc_moderation_action != DMARCModerationAction.none and + msgdata.get('dmarc')): + if mlist.dmarc_moderation_action == DMARCModerationAction.munge_from: + munge_from(mlist, msg, msgdata) + elif (mlist.dmarc_moderation_action == + DMARCModerationAction.wrap_message): + wrap_message(mlist, msg, msgdata, dmarc_wrap=True) + else: + assert False, ( + 'handlers/dmarc.py: dmarc_moderation_action = {0}'.format( + mlist.dmarc_moderation_action)) + else: + if mlist.from_is_list == FromIsList.munge_from: + munge_from(mlist, msg, msgdata) + elif mlist.from_is_list == FromIsList.wrap_message: + wrap_message(mlist, msg, msgdata) + else: + assert False, ( + 'handlers/dmarc.py: from_is_list = {0}'.format( + mlist.from_is_list)) + + +@public +@implementer(IHandler) +class DMARC: + """Apply DMARC mitigations.""" + + name = 'dmarc' + description = _('Apply DMARC mitigations.') + + def process(self, mlist, msg, msgdata): + """See `IHandler`.""" + process(mlist, msg, msgdata) diff --git a/src/mailman/handlers/docs/dmarc-mitigations.rst b/src/mailman/handlers/docs/dmarc-mitigations.rst new file mode 100644 index 000000000..f538182ad --- /dev/null +++ b/src/mailman/handlers/docs/dmarc-mitigations.rst @@ -0,0 +1,142 @@ +================= +DMARC Mitigations +================= + +In order to mitigate the effects of DMARC on mailing list traffic, list +administrators have the ability to apply transformations to messages delivered +to list members. These transformations are applied only to individual messages +sent to list members and not to messages in digests, archives or gated to +usenet. + +The messages can be transformed by either munging the From: header and putting +original From: in Cc: or Reply-To: or by wrapping the original message in an +outer message From: the list. + +Exactly what transformations are applied depends on a number of list settings. + +The settings and their effects are: + + * anonymous_list: If True, no mitigations are ever applied because the message + is already From: the list. + * dmarc_moderation_action: The action to apply to messages From: a domain + publishing a DMARC policy of reject and possibly quarantine or none. + * dmarc_quarantine_moderation_action: A flag to apply dmarc_moderation_action + to messages From: a domain publishing a DMARC policy of quarantine. + * dmarc_none_moderation_action: A flag to apply dmarc_moderation_action to + messages From: a domain publishing a DMARC policy of none, but only when + dmarc_quarantine_moderation_action is also true. + * dmarc_moderation_notice: Text to include in any rejection notice to be sent + when dmarc_moderation_action of reject applies. + * dmarc_wrapped_message_text: Text to be added as a separate text/plain MIME + part preceding the original message part in the wrapped message when + dmarc_moderation_action of wrap_message applies. + * from_is_list: The action to be applied to all messages for which + dmarc_moderation_action is none or not applicable. + * reply_goes_to_list: If this is set to other than no_munging of Reply-To, + the original From: goes in Cc: rather than Reply-To:. + +The possible actions for both dmarc_moderation_action and from_is_list are: + + * none: Make no transformation to the message. + * munge_from: Change the From: header and put the original From: in Reply-To: + or in some cases Cc: + * wrap_message: Wrap the message in an outer message with headers as in + munge_from. + +In addition, there are two more possible actions for dmarc_moderation_action +only: + + * reject: Bounce the message back to the sender with a default reason or one + supplied in dmarc_moderation_notice. + * discard: Silently discard the message. + +Here's what happens when we munge the From: + + >>> from mailman.interfaces.mailinglist import (DMARCModerationAction, + ... ReplyToMunging) + >>> mlist = create_list('test@example.com') + >>> mlist.dmarc_moderation_action = DMARCModerationAction.munge_from + >>> mlist.reply_goes_to_list = ReplyToMunging.no_munging + >>> msg = message_from_string("""\ + ... From: A Person <aperson@example.com> + ... To: test@example.com + ... + ... A message of great import. + ... """) + >>> msgdata = dict(dmarc=True, original_sender='aperson@example.com') + >>> from mailman.handlers.dmarc import process + >>> process(mlist, msg, msgdata) + >>> print(msg.as_string()) + To: test@example.com + From: A Person via Test <test@example.com> + Reply-To: A Person <aperson@example.com> + <BLANKLINE> + A message of great import. + <BLANKLINE> + +Here we wrap the message without adding a text part. + + >>> mlist.dmarc_moderation_action = DMARCModerationAction.wrap_message + >>> mlist.dmarc_wrapped_message_text = '' + >>> msg = message_from_string("""\ + ... From: A Person <aperson@example.com> + ... To: test@example.com + ... + ... A message of great import. + ... """) + >>> msgdata = dict(dmarc=True, original_sender='aperson@example.com') + >>> process(mlist, msg, msgdata) + >>> print(msg.as_string()) + To: test@example.com + MIME-Version: 1.0 + Message-ID: <...> + From: A Person via Test <test@example.com> + Reply-To: A Person <aperson@example.com> + Content-Type: message/rfc822 + Content-Disposition: inline + <BLANKLINE> + From: A Person <aperson@example.com> + To: test@example.com + <BLANKLINE> + A message of great import. + <BLANKLINE> + +And here's a wrapped message with an added text part. + + >>> mlist.dmarc_wrapped_message_text = 'The original message is attached.' + >>> msg = message_from_string("""\ + ... From: A Person <aperson@example.com> + ... To: test@example.com + ... + ... A message of great import. + ... """) + >>> msgdata = dict(dmarc=True, original_sender='aperson@example.com') + >>> process(mlist, msg, msgdata) + >>> print(msg.as_string()) + To: test@example.com + MIME-Version: 1.0 + Message-ID: <...> + From: A Person via Test <test@example.com> + Reply-To: A Person <aperson@example.com> + Content-Type: multipart/mixed; boundary="..." + <BLANKLINE> + --... + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: inline + <BLANKLINE> + The original message is attached. + --... + Content-Type: message/rfc822 + MIME-Version: 1.0 + Content-Disposition: inline + <BLANKLINE> + From: A Person <aperson@example.com> + To: test@example.com + <BLANKLINE> + A message of great import. + <BLANKLINE> + --...-- + <BLANKLINE> + diff --git a/src/mailman/interfaces/mailinglist.py b/src/mailman/interfaces/mailinglist.py index 877016f41..08b1f4032 100644 --- a/src/mailman/interfaces/mailinglist.py +++ b/src/mailman/interfaces/mailinglist.py @@ -24,6 +24,37 @@ from zope.interface import Attribute, Interface @public +class DMARCModerationAction(Enum): + none = 0 + # No DMARC mitigations. + munge_from = 1 + # Messages From: domains with DMARC policy will have From: replaced by + # the list posting address and the original From: added to Reply-To: + # or Cc:. + wrap_message = 2 + # Messages From: domains with DMARC policy will be wrapped in an outer + # message From: the list posting address. + reject = 3 + # Messages From: domains with DMARC policy will be rejected. + discard = 4 + # Messages From: domains with DMARC policy will be discarded. + + +@public +class FromIsList(Enum): + none = 0 + # No DMARC transformations will be applied to all messages. + munge_from = 1 + # All messages will have From: replaced by the list posting address and + # the original From: added to Reply-To: except DMARCModerationAction + # wrap_message, reject or discard takes precedence if applicable. + wrap_message = 2 + # All messages will be wrapped in an outer message From: the list posting + # address except DMARCModerationAction reject or discard takes + # precedence if applicable. + + +@public class Personalization(Enum): none = 0 # Everyone gets a unique copy of the message, and there are a few more @@ -216,6 +247,39 @@ class IMailingList(Interface): def confirm_address(cookie=''): """The address used for various forms of email confirmation.""" + # DMARC attributes. + + dmarc_moderation_action = Attribute( + """The DMARCModerationAction to be applied to messages From: a + domain publishing DMARC p=reject and possibly quarantine or none. + """) + + dmarc_quarantine_moderation_action = Attribute( + """Flag to apply DMARCModerationAction to messages From: a domain + publishing DMARC p=quarantine. + """) + + dmarc_none_moderation_action = Attribute( + """Flag to apply DMARCModerationAction to messages From: a domain + publishing DMARC p=none, but only when + dmarc_quarantine_moderation_action is also true. + """) + dmarc_moderation_notice = Attribute( + """Text to include in any rejection notice to be sent when + DMARCModerationAction of reject applies. + """) + + dmarc_wrapped_message_text = Attribute( + """Text to be added as a separate text/plain MIME part preceding the + original message part in the wrapped message when + DMARCModerationAction of wrap_message applies. + """) + + from_is_list = Attribute( + """The FromIsList action to be applied to all messages for which + DMARCModerationAction is none or not applicable. + """) + # Rosters and subscriptions. owners = Attribute( diff --git a/src/mailman/model/mailinglist.py b/src/mailman/model/mailinglist.py index eec58ad04..1a97d6506 100644 --- a/src/mailman/model/mailinglist.py +++ b/src/mailman/model/mailinglist.py @@ -33,9 +33,9 @@ from mailman.interfaces.digests import DigestFrequency from mailman.interfaces.domain import IDomainManager from mailman.interfaces.languages import ILanguageManager from mailman.interfaces.mailinglist import ( - IAcceptableAlias, IAcceptableAliasSet, IHeaderMatch, IHeaderMatchList, - IListArchiver, IListArchiverSet, IMailingList, Personalization, - ReplyToMunging, SubscriptionPolicy) + DMARCModerationAction, FromIsList, IAcceptableAlias, IAcceptableAliasSet, + IHeaderMatch, IHeaderMatchList, IListArchiver, IListArchiverSet, + IMailingList, Personalization, ReplyToMunging, SubscriptionPolicy) from mailman.interfaces.member import ( AlreadySubscribedError, MemberRole, MissingPreferredAddressError, SubscriptionEvent) @@ -127,6 +127,13 @@ class MailingList(Model): forward_unrecognized_bounces_to = Column( Enum(UnrecognizedBounceDisposition)) process_bounces = Column(Boolean) + # DMARC + dmarc_moderation_action = Column(Enum(DMARCModerationAction)) + dmarc_quarantine_moderation_action = Column(Boolean) + dmarc_none_moderation_action = Column(Boolean) + dmarc_moderation_notice = Column(SAUnicode) + dmarc_wrapped_message_text = Column(SAUnicode) + from_is_list = Column(Enum(FromIsList)) # Miscellaneous default_member_action = Column(Enum(Action)) default_nonmember_action = Column(Enum(Action)) diff --git a/src/mailman/rest/docs/systemconf.rst b/src/mailman/rest/docs/systemconf.rst index 385588077..24c6bef4a 100644 --- a/src/mailman/rest/docs/systemconf.rst +++ b/src/mailman/rest/docs/systemconf.rst @@ -14,6 +14,9 @@ You can also get all the values for a particular section. >>> dump_json('http://localhost:9001/3.0/system/configuration/mailman') cache_life: 7d default_language: en + dmarc_org_domain_data: https://publicsuffix.org/list/public_suffix_list.dat + dmarc_resolver_lifetime: 5s + dmarc_resolver_timeout: 3s email_commands_max_lines: 10 filtered_messages_are_preservable: no html_to_plain_text_command: /usr/bin/lynx -dump $filename diff --git a/src/mailman/rest/tests/test_systemconf.py b/src/mailman/rest/tests/test_systemconf.py index e76c082df..28fa315a9 100644 --- a/src/mailman/rest/tests/test_systemconf.py +++ b/src/mailman/rest/tests/test_systemconf.py @@ -39,6 +39,10 @@ class TestSystemConfiguration(unittest.TestCase): self.assertEqual(json, dict( cache_life='7d', default_language='en', + dmarc_org_domain_data= # noqa E251 + 'https://publicsuffix.org/list/public_suffix_list.dat', + dmarc_resolver_lifetime='5s', + dmarc_resolver_timeout='3s', email_commands_max_lines='10', filtered_messages_are_preservable='no', html_to_plain_text_command='/usr/bin/lynx -dump $filename', diff --git a/src/mailman/rules/dmarc.py b/src/mailman/rules/dmarc.py new file mode 100644 index 000000000..ba42ef70a --- /dev/null +++ b/src/mailman/rules/dmarc.py @@ -0,0 +1,269 @@ +# Copyright (C) 2016 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""DMARC moderation rule.""" + +import re +import logging +import dns.resolver + +from dns.exception import DNSException +from email.utils import parseaddr +from lazr.config import as_timedelta +from mailman import public +from mailman.chains.discard import DiscardChain +from mailman.chains.reject import RejectChain +from mailman.config import config +from mailman.core.i18n import _ +from mailman.interfaces.mailinglist import DMARCModerationAction +from mailman.interfaces.rules import IRule +from mailman.utilities.string import wrap +from urllib import error, request +from zope.interface import implementer + + +elog = logging.getLogger('mailman.error') +vlog = logging.getLogger('mailman.vette') +s_dict = dict() + + +def _get_suffixes(url): + # This loads and parses the data from the url argument into s_dict for + # use by _get_org_dom. + global s_dict + if s_dict: + return + if not url: + return + try: + d = request.urlopen(url) + except error.URLError as e: + elog.error('Unable to retrieve data from %s: %s', url, e) + return + for line in d.readlines(): + line = str(line, encoding='utf-8') + if not line.strip() or line.startswith(' ') or line.startswith('//'): + continue + line = re.sub(' .*', '', line.strip()) + if not line: + continue + parts = line.lower().split('.') + if parts[0].startswith('!'): + exc = True + parts = [parts[0][1:]] + parts[1:] + else: + exc = False + parts.reverse() + k = '.'.join(parts) + s_dict[k] = exc + + +def _get_dom(d, l): + # A helper to get a domain name consisting of the first l+1 labels + # in d. + dom = d[:min(l+1, len(d))] + dom.reverse() + return '.'.join(dom) + + +def _get_org_dom(domain): + # Given a domain name, this returns the corresponding Organizational + # Domain which may be the same as the input. + global s_dict + if not s_dict: + _get_suffixes(config.mailman.dmarc_org_domain_data) + hits = [] + d = domain.lower().split('.') + d.reverse() + for k in s_dict.keys(): + ks = k.split('.') + if len(d) >= len(ks): + for i in range(len(ks)-1): + if d[i] != ks[i] and ks[i] != '*': + break + else: + if d[len(ks)-1] == ks[-1] or ks[-1] == '*': + hits.append(k) + if not hits: + return _get_dom(d, 1) + l = 0 + for k in hits: + if s_dict[k]: + # It's an exception + return _get_dom(d, len(k.split('.'))-1) + if len(k.split('.')) > l: + l = len(k.split('.')) + return _get_dom(d, l) + + +def _IsDMARCProhibited(mlist, email): + # This takes an email address, and returns True if DMARC policy is + # p=reject or possibly quarantine or none. + email = email.lower() + # Scan from the right in case quoted local part has an '@'. + at_sign = email.rfind('@') + if at_sign < 1: + return False + f_dom = email[at_sign+1:] + x = _DMARCProhibited(mlist, email, '_dmarc.' + f_dom) + if x != 'continue': + return x + o_dom = _get_org_dom(f_dom) + if o_dom != f_dom: + x = _DMARCProhibited(mlist, email, '_dmarc.' + o_dom, org=True) + if x != 'continue': + return x + return False + + +def _DMARCProhibited(mlist, email, dmarc_domain, org=False): + + try: + resolver = dns.resolver.Resolver() + resolver.timeout = as_timedelta( + config.mailman.dmarc_resolver_timeout).total_seconds() + resolver.lifetime = as_timedelta( + config.mailman.dmarc_resolver_lifetime).total_seconds() + txt_recs = resolver.query(dmarc_domain, dns.rdatatype.TXT) + except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): + return 'continue' + except DNSException as e: + elog.error( + 'DNSException: Unable to query DMARC policy for %s (%s). %s', + email, dmarc_domain, e.__doc__) + return 'continue' + else: + # Be as robust as possible in parsing the result. + results_by_name = {} + cnames = {} + want_names = set([dmarc_domain + '.']) + for txt_rec in txt_recs.response.answer: + if txt_rec.rdtype == dns.rdatatype.CNAME: + cnames[txt_rec.name.to_text()] = ( + txt_rec.items[0].target.to_text()) + if txt_rec.rdtype != dns.rdatatype.TXT: + continue + results_by_name.setdefault( + txt_rec.name.to_text(), []).append( + "".join( + [str(x, encoding='utf-8') + for x in txt_rec.items[0].strings])) + expands = list(want_names) + seen = set(expands) + while expands: + item = expands.pop(0) + if item in cnames: + if cnames[item] in seen: + continue # cname loop + expands.append(cnames[item]) + seen.add(cnames[item]) + want_names.add(cnames[item]) + want_names.discard(item) + + if len(want_names) != 1: + elog.error( + """multiple DMARC entries in results for %s, + processing each to be strict""", + dmarc_domain) + for name in want_names: + if name not in results_by_name: + continue + dmarcs = [x for x in results_by_name[name] + if x.startswith('v=DMARC1;')] + if len(dmarcs) == 0: + return 'continue' + if len(dmarcs) > 1: + elog.error( + """RRset of TXT records for %s has %d v=DMARC1 entries; + testing them all""", + dmarc_domain, len(dmarcs)) + for entry in dmarcs: + mo = re.search(r'\bsp=(\w*)\b', entry, re.IGNORECASE) + if org and mo: + policy = mo.group(1).lower() + else: + mo = re.search(r'\bp=(\w*)\b', entry, re.IGNORECASE) + if mo: + policy = mo.group(1).lower() + else: + continue + if policy == 'reject': + vlog.info( + """%s: DMARC lookup for %s (%s) + found p=reject in %s = %s""", + mlist.list_name, email, dmarc_domain, name, entry) + return True + + if (mlist.dmarc_quarantine_moderation_action and + policy == 'quarantine'): + vlog.info( + """%s: DMARC lookup for %s (%s) + found p=quarantine in %s = %s""", + mlist.list_name, email, dmarc_domain, name, entry) + return True + + if (mlist.dmarc_none_moderation_action and + mlist.dmarc_quarantine_moderation_action and + mlist.dmarc_moderation_action in ( + DMARCModerationAction.munge_from, + DMARCModerationAction.wrap_message) and + policy == 'none'): + vlog.info( + '%s: DMARC lookup for %s (%s) found p=none in %s = %s', + mlist.list_name, email, dmarc_domain, name, entry) + return True + + return False + + +@public +@implementer(IRule) +class DMARCModeration: + """The DMARC moderation rule.""" + + name = 'dmarc-moderation' + description = _('Find DMARC policy of From: domain.') + record = True + + def check(self, mlist, msg, msgdata): + """See `IRule`.""" + if mlist.dmarc_moderation_action == DMARCModerationAction.none: + # Don't bother to check if we're not going to do anything. + return False + dn, addr = parseaddr(msg.get('from')) + if _IsDMARCProhibited(mlist, addr): + # This is something of a kludge, but we can't have this rule be + # a chain in the normal way, because a hit will cause the message + # to be held. We just flag the hit for the handler, but jump + # to the reject or discard chain if appropriate. + msgdata['dmarc'] = True + if mlist.dmarc_moderation_action == DMARCModerationAction.discard: + DiscardChain()._process(mlist, msg, msgdata) + if mlist.dmarc_moderation_action == DMARCModerationAction.reject: + listowner = mlist.owner_address # noqa F841 + reason = (mlist.dmarc_moderation_notice or + _('You are not allowed to post to this mailing ' + 'list From: a domain which publishes a DMARC ' + 'policy of reject or quarantine, and your message' + ' has been automatically rejected. If you think ' + 'that your messages are being rejected in error, ' + 'contact the mailing list owner at $listowner.')) + msgdata['moderation_reasons'] = [wrap(reason)] + # Add the hit for the reject notice. + msgdata.setdefault('rule_hits', []).append('dmarc-moderation') + RejectChain()._process(mlist, msg, msgdata) + return False diff --git a/src/mailman/rules/docs/dmarc-moderation.rst b/src/mailman/rules/docs/dmarc-moderation.rst new file mode 100644 index 000000000..e0bfe6b07 --- /dev/null +++ b/src/mailman/rules/docs/dmarc-moderation.rst @@ -0,0 +1,210 @@ +================ +DMARC moderation +================ + +This rule is different from others in that it never matches bucause a match +would cause the message to be held. The rule looks at the list's +dmarc_moderation_policy and if it is other than 'none', it checks the domain +of the From: address for a DMARC policy and depending on settings may reject +or discard the message or just flag in for the dmarc handler to apply DMARC +mitigations to the message. + + >>> mlist = create_list('_xtest@example.com') + >>> rule = config.rules['dmarc-moderation'] + >>> print(rule.name) + dmarc-moderation + +A message From: a domain without a DMARC policy does not set any flags. + + >>> from mailman.interfaces.mailinglist import DMARCModerationAction + >>> mlist.dmarc_moderation_action = DMARCModerationAction.munge_from + >>> msg = message_from_string("""\ + ... From: aperson@example.org + ... To: _xtest@example.com + ... Subject: A posted message + ... + ... """) + >>> msgdata = {} + >>> rule.check(mlist, msg, msgdata) + False + >>> msgdata == {} + True + +Even if the From: domain publishes p=reject, no flags are set if the list's +action is none. + + >>> mlist.dmarc_moderation_action = DMARCModerationAction.none + >>> msg = message_from_string("""\ + ... From: aperson@yahoo.com + ... To: _xtest@example.com + ... Subject: A posted message + ... + ... """) + >>> msgdata = {} + >>> rule.check(mlist, msg, msgdata) + False + >>> msgdata == {} + True + +But with a different list setting, the message is flagged. + + >>> mlist.dmarc_moderation_action = DMARCModerationAction.munge_from + >>> msg = message_from_string("""\ + ... From: aperson@yahoo.com + ... To: _xtest@example.com + ... Subject: A posted message + ... + ... """) + >>> msgdata = {} + >>> rule.check(mlist, msg, msgdata) + False + >>> msgdata['dmarc'] + True + +Subdomains which don't have a policy will check the organizational domain. + + >>> msg = message_from_string("""\ + ... From: aperson@sub.domain.yahoo.com + ... To: _xtest@example.com + ... Subject: A posted message + ... + ... """) + >>> msgdata = {} + >>> rule.check(mlist, msg, msgdata) + False + >>> msgdata['dmarc'] + True + +The list's action can also be set to immediately discard or reject the +message. + + >>> mlist.dmarc_moderation_action = DMARCModerationAction.discard + >>> msg = message_from_string("""\ + ... From: aperson@yahoo.com + ... To: _xtest@example.com + ... Subject: A posted message + ... + ... """) + >>> msgdata = {} + >>> rule.check(mlist, msg, msgdata) + False + >>> msgdata['dmarc'] + True + +The above needs to test that the message was discarded. + +We can reject the message with a default reason. + + >>> mlist.dmarc_moderation_action = DMARCModerationAction.reject + >>> msg = message_from_string("""\ + ... From: aperson@yahoo.com + ... To: _xtest@example.com + ... Subject: A posted message + ... + ... """) + >>> msgdata = {} + >>> rule.check(mlist, msg, msgdata) + False + >>> msgdata['dmarc'] + True + +There is now a reject message in the virgin queue. + + >>> from mailman.testing.helpers import get_queue_messages + >>> messages = get_queue_messages('virgin') + >>> len(messages) + 1 + >>> print(messages[0].msg.as_string()) + Subject: A posted message + From: _xtest-owner@example.com + To: aperson@yahoo.com + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="..." + Message-ID: <...> + Date: ... + Precedence: bulk + <BLANKLINE> + --... + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + <BLANKLINE> + <BLANKLINE> + Your message to the _xtest mailing-list was rejected for the following + reasons: + <BLANKLINE> + You are not allowed to post to this mailing list From: a domain which + publishes a DMARC policy of reject or quarantine, and your message has + been automatically rejected. If you think that your messages are + being rejected in error, contact the mailing list owner at $listowner. + <BLANKLINE> + The original message as received by Mailman is attached. + <BLANKLINE> + --... + Content-Type: message/rfc822 + MIME-Version: 1.0 + <BLANKLINE> + From: aperson@yahoo.com + To: _xtest@example.com + Subject: A posted message + X-Mailman-Rule-Hits: dmarc-moderation + <BLANKLINE> + <BLANKLINE> + --...-- + <BLANKLINE> + +And, we can reject with a custom message. + + >>> mlist.dmarc_moderation_notice = 'A silly reason' + >>> msg = message_from_string("""\ + ... From: aperson@yahoo.com + ... To: _xtest@example.com + ... Subject: A posted message + ... + ... """) + >>> msgdata = {} + >>> rule.check(mlist, msg, msgdata) + False + >>> msgdata['dmarc'] + True + +Check the the virgin queue. + + >>> messages = get_queue_messages('virgin') + >>> len(messages) + 1 + >>> print(messages[0].msg.as_string()) + Subject: A posted message + From: _xtest-owner@example.com + To: aperson@yahoo.com + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="..." + Message-ID: <...> + Date: ... + Precedence: bulk + <BLANKLINE> + --... + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + <BLANKLINE> + <BLANKLINE> + Your message to the _xtest mailing-list was rejected for the following + reasons: + <BLANKLINE> + A silly reason + <BLANKLINE> + The original message as received by Mailman is attached. + <BLANKLINE> + --... + Content-Type: message/rfc822 + MIME-Version: 1.0 + <BLANKLINE> + From: aperson@yahoo.com + To: _xtest@example.com + Subject: A posted message + X-Mailman-Rule-Hits: dmarc-moderation + <BLANKLINE> + <BLANKLINE> + --...-- + <BLANKLINE> diff --git a/src/mailman/rules/docs/rules.rst b/src/mailman/rules/docs/rules.rst index 812486b45..e00889061 100644 --- a/src/mailman/rules/docs/rules.rst +++ b/src/mailman/rules/docs/rules.rst @@ -22,6 +22,7 @@ names to rule objects. any True approved True banned-address True + dmarc-moderation True emergency True implicit-dest True loop True diff --git a/src/mailman/styles/base.py b/src/mailman/styles/base.py index 7226d762a..07f6f57dc 100644 --- a/src/mailman/styles/base.py +++ b/src/mailman/styles/base.py @@ -32,7 +32,8 @@ from mailman.interfaces.autorespond import ResponseAction from mailman.interfaces.bounce import UnrecognizedBounceDisposition from mailman.interfaces.digests import DigestFrequency from mailman.interfaces.mailinglist import ( - Personalization, ReplyToMunging, SubscriptionPolicy) + DMARCModerationAction, FromIsList, Personalization, ReplyToMunging, + SubscriptionPolicy) from mailman.interfaces.nntp import NewsgroupModeration @@ -85,6 +86,13 @@ class BasicOperation: mlist.digest_send_periodic = True mlist.digest_volume_frequency = DigestFrequency.monthly mlist.next_digest_number = 1 + # DMARC + mlist.dmarc_moderation_action = DMARCModerationAction.none + mlist.dmarc_quarantine_moderation_action = True + mlist.dmarc_none_moderation_action = False + mlist.dmarc_moderation_notice = '' + mlist.dmarc_wrapped_message_text = '' + mlist.from_is_list = FromIsList.none # NNTP gateway mlist.nntp_host = '' mlist.linked_newsgroup = '' |