3 files changed, 480 insertions, 0 deletions

diff --git a/src/mailman/rules/dmarc.py b/src/mailman/rules/dmarc.py
new file mode 100644
index 000000000..ba42ef70a
--- /dev/null
+++ b/src/mailman/rules/dmarc.py
@@ -0,0 +1,269 @@
+# Copyright (C) 2016 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""DMARC moderation rule."""
+
+import re
+import logging
+import dns.resolver
+
+from dns.exception import DNSException
+from email.utils import parseaddr
+from lazr.config import as_timedelta
+from mailman import public
+from mailman.chains.discard import DiscardChain
+from mailman.chains.reject import RejectChain
+from mailman.config import config
+from mailman.core.i18n import _
+from mailman.interfaces.mailinglist import DMARCModerationAction
+from mailman.interfaces.rules import IRule
+from mailman.utilities.string import wrap
+from urllib import error, request
+from zope.interface import implementer
+
+
+elog = logging.getLogger('mailman.error')
+vlog = logging.getLogger('mailman.vette')
+s_dict = dict()
+
+
+def _get_suffixes(url):
+    # This loads and parses the data from the url argument into s_dict for
+    # use by _get_org_dom.
+    global s_dict
+    if s_dict:
+        return
+    if not url:
+        return
+    try:
+        d = request.urlopen(url)
+    except error.URLError as e:
+        elog.error('Unable to retrieve data from %s: %s', url, e)
+        return
+    for line in d.readlines():
+        line = str(line, encoding='utf-8')
+        if not line.strip() or line.startswith(' ') or line.startswith('//'):
+            continue
+        line = re.sub(' .*', '', line.strip())
+        if not line:
+            continue
+        parts = line.lower().split('.')
+        if parts[0].startswith('!'):
+            exc = True
+            parts = [parts[0][1:]] + parts[1:]
+        else:
+            exc = False
+        parts.reverse()
+        k = '.'.join(parts)
+        s_dict[k] = exc
+
+
+def _get_dom(d, l):
+    # A helper to get a domain name consisting of the first l+1 labels
+    # in d.
+    dom = d[:min(l+1, len(d))]
+    dom.reverse()
+    return '.'.join(dom)
+
+
+def _get_org_dom(domain):
+    # Given a domain name, this returns the corresponding Organizational
+    # Domain which may be the same as the input.
+    global s_dict
+    if not s_dict:
+        _get_suffixes(config.mailman.dmarc_org_domain_data)
+    hits = []
+    d = domain.lower().split('.')
+    d.reverse()
+    for k in s_dict.keys():
+        ks = k.split('.')
+        if len(d) >= len(ks):
+            for i in range(len(ks)-1):
+                if d[i] != ks[i] and ks[i] != '*':
+                    break
+            else:
+                if d[len(ks)-1] == ks[-1] or ks[-1] == '*':
+                    hits.append(k)
+    if not hits:
+        return _get_dom(d, 1)
+    l = 0
+    for k in hits:
+        if s_dict[k]:
+            # It's an exception
+            return _get_dom(d, len(k.split('.'))-1)
+        if len(k.split('.')) > l:
+            l = len(k.split('.'))
+    return _get_dom(d, l)
+
+
+def _IsDMARCProhibited(mlist, email):
+    # This takes an email address, and returns True if DMARC policy is
+    # p=reject or possibly quarantine or none.
+    email = email.lower()
+    # Scan from the right in case quoted local part has an '@'.
+    at_sign = email.rfind('@')
+    if at_sign < 1:
+        return False
+    f_dom = email[at_sign+1:]
+    x = _DMARCProhibited(mlist, email, '_dmarc.' + f_dom)
+    if x != 'continue':
+        return x
+    o_dom = _get_org_dom(f_dom)
+    if o_dom != f_dom:
+        x = _DMARCProhibited(mlist, email, '_dmarc.' + o_dom, org=True)
+        if x != 'continue':
+            return x
+    return False
+
+
+def _DMARCProhibited(mlist, email, dmarc_domain, org=False):
+
+    try:
+        resolver = dns.resolver.Resolver()
+        resolver.timeout = as_timedelta(
+            config.mailman.dmarc_resolver_timeout).total_seconds()
+        resolver.lifetime = as_timedelta(
+            config.mailman.dmarc_resolver_lifetime).total_seconds()
+        txt_recs = resolver.query(dmarc_domain, dns.rdatatype.TXT)
+    except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
+        return 'continue'
+    except DNSException as e:
+        elog.error(
+            'DNSException: Unable to query DMARC policy for %s (%s). %s',
+            email, dmarc_domain, e.__doc__)
+        return 'continue'
+    else:
+        # Be as robust as possible in parsing the result.
+        results_by_name = {}
+        cnames = {}
+        want_names = set([dmarc_domain + '.'])
+        for txt_rec in txt_recs.response.answer:
+            if txt_rec.rdtype == dns.rdatatype.CNAME:
+                cnames[txt_rec.name.to_text()] = (
+                    txt_rec.items[0].target.to_text())
+            if txt_rec.rdtype != dns.rdatatype.TXT:
+                continue
+            results_by_name.setdefault(
+                txt_rec.name.to_text(), []).append(
+                    "".join(
+                       [str(x, encoding='utf-8')
+                           for x in txt_rec.items[0].strings]))
+        expands = list(want_names)
+        seen = set(expands)
+        while expands:
+            item = expands.pop(0)
+            if item in cnames:
+                if cnames[item] in seen:
+                    continue  # cname loop
+                expands.append(cnames[item])
+                seen.add(cnames[item])
+                want_names.add(cnames[item])
+                want_names.discard(item)
+
+        if len(want_names) != 1:
+            elog.error(
+                """multiple DMARC entries in results for %s,
+                processing each to be strict""",
+                dmarc_domain)
+        for name in want_names:
+            if name not in results_by_name:
+                continue
+            dmarcs = [x for x in results_by_name[name]
+                      if x.startswith('v=DMARC1;')]
+            if len(dmarcs) == 0:
+                return 'continue'
+            if len(dmarcs) > 1:
+                elog.error(
+                    """RRset of TXT records for %s has %d v=DMARC1 entries;
+                    testing them all""",
+                    dmarc_domain, len(dmarcs))
+            for entry in dmarcs:
+                mo = re.search(r'\bsp=(\w*)\b', entry, re.IGNORECASE)
+                if org and mo:
+                    policy = mo.group(1).lower()
+                else:
+                    mo = re.search(r'\bp=(\w*)\b', entry, re.IGNORECASE)
+                    if mo:
+                        policy = mo.group(1).lower()
+                    else:
+                        continue
+                if policy == 'reject':
+                    vlog.info(
+                        """%s: DMARC lookup for %s (%s)
+                        found p=reject in %s = %s""",
+                        mlist.list_name, email, dmarc_domain, name, entry)
+                    return True
+
+                if (mlist.dmarc_quarantine_moderation_action and
+                        policy == 'quarantine'):
+                    vlog.info(
+                        """%s: DMARC lookup for %s (%s)
+                        found p=quarantine in %s = %s""",
+                        mlist.list_name, email, dmarc_domain, name, entry)
+                    return True
+
+                if (mlist.dmarc_none_moderation_action and
+                        mlist.dmarc_quarantine_moderation_action and
+                        mlist.dmarc_moderation_action in (
+                            DMARCModerationAction.munge_from,
+                            DMARCModerationAction.wrap_message) and
+                        policy == 'none'):
+                    vlog.info(
+                        '%s: DMARC lookup for %s (%s) found p=none in %s = %s',
+                        mlist.list_name, email, dmarc_domain, name, entry)
+                    return True
+
+    return False
+
+
+@public
+@implementer(IRule)
+class DMARCModeration:
+    """The DMARC moderation rule."""
+
+    name = 'dmarc-moderation'
+    description = _('Find DMARC policy of From: domain.')
+    record = True
+
+    def check(self, mlist, msg, msgdata):
+        """See `IRule`."""
+        if mlist.dmarc_moderation_action == DMARCModerationAction.none:
+            # Don't bother to check if we're not going to do anything.
+            return False
+        dn, addr = parseaddr(msg.get('from'))
+        if _IsDMARCProhibited(mlist, addr):
+            # This is something of a kludge, but we can't have this rule be
+            # a chain in the normal way, because a hit will cause the message
+            # to be held.  We just flag the hit for the handler, but jump
+            # to the reject or discard chain if appropriate.
+            msgdata['dmarc'] = True
+            if mlist.dmarc_moderation_action == DMARCModerationAction.discard:
+                DiscardChain()._process(mlist, msg, msgdata)
+            if mlist.dmarc_moderation_action == DMARCModerationAction.reject:
+                listowner = mlist.owner_address       # noqa F841
+                reason = (mlist.dmarc_moderation_notice or
+                          _('You are not allowed to post to this mailing '
+                            'list From: a domain which publishes a DMARC '
+                            'policy of reject or quarantine, and your message'
+                            ' has been automatically rejected.  If you think '
+                            'that your messages are being rejected in error, '
+                            'contact the mailing list owner at $listowner.'))
+                msgdata['moderation_reasons'] = [wrap(reason)]
+                # Add the hit for the reject notice.
+                msgdata.setdefault('rule_hits', []).append('dmarc-moderation')
+                RejectChain()._process(mlist, msg, msgdata)
+        return False
diff --git a/src/mailman/rules/docs/dmarc-moderation.rst b/src/mailman/rules/docs/dmarc-moderation.rst
new file mode 100644
index 000000000..e0bfe6b07
--- /dev/null
+++ b/src/mailman/rules/docs/dmarc-moderation.rst
@@ -0,0 +1,210 @@
+================
+DMARC moderation
+================
+
+This rule is different from others in that it never matches bucause a match
+would cause the message to be held.  The rule looks at the list's
+dmarc_moderation_policy and if it is other than 'none', it checks the domain
+of the From: address for a DMARC policy and depending on settings may reject
+or discard the message or just flag in for the dmarc handler to apply DMARC
+mitigations to the message.
+
+    >>> mlist = create_list('_xtest@example.com')
+    >>> rule = config.rules['dmarc-moderation']
+    >>> print(rule.name)
+    dmarc-moderation
+
+A message From: a domain without a DMARC policy does not set any flags.
+
+    >>> from mailman.interfaces.mailinglist import DMARCModerationAction
+    >>> mlist.dmarc_moderation_action = DMARCModerationAction.munge_from
+    >>> msg = message_from_string("""\
+    ... From: aperson@example.org
+    ... To: _xtest@example.com
+    ... Subject: A posted message
+    ...
+    ... """)
+    >>> msgdata = {}
+    >>> rule.check(mlist, msg, msgdata)
+    False
+    >>> msgdata == {}
+    True
+
+Even if the From: domain publishes p=reject, no flags are set if the list's
+action is none.
+
+    >>> mlist.dmarc_moderation_action = DMARCModerationAction.none
+    >>> msg = message_from_string("""\
+    ... From: aperson@yahoo.com
+    ... To: _xtest@example.com
+    ... Subject: A posted message
+    ...
+    ... """)
+    >>> msgdata = {}
+    >>> rule.check(mlist, msg, msgdata)
+    False
+    >>> msgdata == {}
+    True
+
+But with a different list setting, the message is flagged.
+
+    >>> mlist.dmarc_moderation_action = DMARCModerationAction.munge_from
+    >>> msg = message_from_string("""\
+    ... From: aperson@yahoo.com
+    ... To: _xtest@example.com
+    ... Subject: A posted message
+    ...
+    ... """)
+    >>> msgdata = {}
+    >>> rule.check(mlist, msg, msgdata)
+    False
+    >>> msgdata['dmarc']
+    True
+
+Subdomains which don't have a policy will check the organizational domain.
+
+    >>> msg = message_from_string("""\
+    ... From: aperson@sub.domain.yahoo.com
+    ... To: _xtest@example.com
+    ... Subject: A posted message
+    ...
+    ... """)
+    >>> msgdata = {}
+    >>> rule.check(mlist, msg, msgdata)
+    False
+    >>> msgdata['dmarc']
+    True
+
+The list's action can also be set to immediately discard or reject the
+message.
+
+    >>> mlist.dmarc_moderation_action = DMARCModerationAction.discard
+    >>> msg = message_from_string("""\
+    ... From: aperson@yahoo.com
+    ... To: _xtest@example.com
+    ... Subject: A posted message
+    ...
+    ... """)
+    >>> msgdata = {}
+    >>> rule.check(mlist, msg, msgdata)
+    False
+    >>> msgdata['dmarc']
+    True
+
+The above needs to test that the message was discarded.
+
+We can reject the message with a default reason.
+
+    >>> mlist.dmarc_moderation_action = DMARCModerationAction.reject
+    >>> msg = message_from_string("""\
+    ... From: aperson@yahoo.com
+    ... To: _xtest@example.com
+    ... Subject: A posted message
+    ...
+    ... """)
+    >>> msgdata = {}
+    >>> rule.check(mlist, msg, msgdata)
+    False
+    >>> msgdata['dmarc']
+    True
+
+There is now a reject message in the virgin queue.
+
+    >>> from mailman.testing.helpers import get_queue_messages
+    >>> messages = get_queue_messages('virgin')
+    >>> len(messages)
+    1
+    >>> print(messages[0].msg.as_string())
+    Subject: A posted message
+    From: _xtest-owner@example.com
+    To: aperson@yahoo.com
+    MIME-Version: 1.0
+    Content-Type: multipart/mixed; boundary="..."
+    Message-ID: <...>
+    Date: ...
+    Precedence: bulk
+    <BLANKLINE>
+    --...
+    Content-Type: text/plain; charset="us-ascii"
+    MIME-Version: 1.0
+    Content-Transfer-Encoding: 7bit
+    <BLANKLINE>
+    <BLANKLINE>
+    Your message to the _xtest mailing-list was rejected for the following
+    reasons:
+    <BLANKLINE>
+    You are not allowed to post to this mailing list From: a domain which
+    publishes a DMARC policy of reject or quarantine, and your message has
+    been automatically rejected.  If you think that your messages are
+    being rejected in error, contact the mailing list owner at $listowner.
+    <BLANKLINE>
+    The original message as received by Mailman is attached.
+    <BLANKLINE>
+    --...
+    Content-Type: message/rfc822
+    MIME-Version: 1.0
+    <BLANKLINE>
+    From: aperson@yahoo.com
+    To: _xtest@example.com
+    Subject: A posted message
+    X-Mailman-Rule-Hits: dmarc-moderation
+    <BLANKLINE>
+    <BLANKLINE>
+    --...--
+    <BLANKLINE>
+
+And, we can reject with a custom message.
+
+    >>> mlist.dmarc_moderation_notice = 'A silly reason'
+    >>> msg = message_from_string("""\
+    ... From: aperson@yahoo.com
+    ... To: _xtest@example.com
+    ... Subject: A posted message
+    ...
+    ... """)
+    >>> msgdata = {}
+    >>> rule.check(mlist, msg, msgdata)
+    False
+    >>> msgdata['dmarc']
+    True
+
+Check the the virgin queue.
+
+    >>> messages = get_queue_messages('virgin')
+    >>> len(messages)
+    1
+    >>> print(messages[0].msg.as_string())
+    Subject: A posted message
+    From: _xtest-owner@example.com
+    To: aperson@yahoo.com
+    MIME-Version: 1.0
+    Content-Type: multipart/mixed; boundary="..."
+    Message-ID: <...>
+    Date: ...
+    Precedence: bulk
+    <BLANKLINE>
+    --...
+    Content-Type: text/plain; charset="us-ascii"
+    MIME-Version: 1.0
+    Content-Transfer-Encoding: 7bit
+    <BLANKLINE>
+    <BLANKLINE>
+    Your message to the _xtest mailing-list was rejected for the following
+    reasons:
+    <BLANKLINE>
+    A silly reason
+    <BLANKLINE>
+    The original message as received by Mailman is attached.
+    <BLANKLINE>
+    --...
+    Content-Type: message/rfc822
+    MIME-Version: 1.0
+    <BLANKLINE>
+    From: aperson@yahoo.com
+    To: _xtest@example.com
+    Subject: A posted message
+    X-Mailman-Rule-Hits: dmarc-moderation
+    <BLANKLINE>
+    <BLANKLINE>
+    --...--
+    <BLANKLINE>
diff --git a/src/mailman/rules/docs/rules.rst b/src/mailman/rules/docs/rules.rst
index 812486b45..e00889061 100644
--- a/src/mailman/rules/docs/rules.rst
+++ b/src/mailman/rules/docs/rules.rst
@@ -22,6 +22,7 @@ names to rule objects.
     any True
     approved True
     banned-address True
+    dmarc-moderation True
     emergency True
     implicit-dest True
     loop True