Merge bounce cleanup branch. This is a major cleansing and refactoring of the

bounce processor.  Note that one thing this does not do is implement any of
the policy around bounce scores.  It "merely" cleans up all the crud in the
BounceRunner, and all the logic around registering bounce events.  This means
the bounce runner can actually be enabled again.  More code needs to be
written to process bounces events occasionally.

Details:

* maybe_forward() moved to app/bounces.py, and it can now discard the
  message, forward it to the list administrators (moderators + owners), or
  site owners.  See UnrecognizedBounceDisposition.
* scan_message() always returns a set of addresses.
* The DSN bounce detector is cleaned up.
* An IBounceProcessor utility is added.
* IBounceEvents are added, with database support.
* bounce_unrecognized_goes_to_list_owner -> forward_unrecognized_bounces_to
* bounce_processing -> process_bounces
* ReopenableFileHandler.filename is exposed as a public attribute.  This aids
  in testing.
* Fix the signature of UserNotification.__init__() to be more PEP 8 compliant.
* Change the OwnerNotification.__init__() signature to take a roster object
  instead of `tomoderators`.  When the roster is None, the site owner is used
  instead.
* Fix the default style setting; s/personalization/personalize/
* BounceMixin is gone, baby gone.
* Add tests for the OutgoingRunner.
* make_testable_runner() can now take a predicate object, which can change how
  often the runner goes through its main loop.  Use this e.g. to go through
  only once when a message does not get dequeued.
* A new LogFileMark helper for testing messages to a log file.

1 files changed, 57 insertions, 75 deletions

diff --git a/src/mailman/bouncers/dsn.py b/src/mailman/bouncers/dsn.py
index 49803ef88..c9d7803ee 100644
--- a/src/mailman/bouncers/dsn.py
+++ b/src/mailman/bouncers/dsn.py
@@ -37,84 +37,66 @@ from mailman.interfaces.bounce import IBounceDetector, Stop
 
 
 
-def check(msg):
-    # Iterate over each message/delivery-status subpart.
-    failed_addresses = []
-    delayed_addresses = []
-    for part in typed_subpart_iterator(msg, 'message', 'delivery-status'):
-        if not part.is_multipart():
-            # Huh?
-            continue
-        # Each message/delivery-status contains a list of Message objects
-        # which are the header blocks.  Iterate over those too.
-        for msgblock in part.get_payload():
-            address_set = None
-            # We try to dig out the Original-Recipient (which is optional) and
-            # Final-Recipient (which is mandatory, but may not exactly match
-            # an address on our list).  Some MTA's also use X-Actual-Recipient
-            # as a synonym for Original-Recipient, but some apparently use
-            # that for other purposes :(
-            #
-            # Also grok out Action so we can do something with that too.
-            action = msgblock.get('action', '').lower()
-            # Some MTAs have been observed that put comments on the action.
-            if action.startswith('delayed'):
-                address_set = delayed_addresses
-            elif action.startswith('fail'):
-                address_set = failed_addresses
-            else:
-                # Some non-permanent failure, so ignore this block.
-                continue
-            params = []
-            foundp = False
-            for header in ('original-recipient', 'final-recipient'):
-                for k, v in msgblock.get_params([], header):
-                    if k.lower() == 'rfc822':
-                        foundp = True
-                    else:
-                        params.append(k)
-                if foundp:
-                    # Note that params should already be unquoted.
-                    address_set.extend(params)
-                    break
-                else:
-                    # MAS: This is a kludge, but SMTP-GATEWAY01.intra.home.dk
-                    # has a final-recipient with an angle-addr and no
-                    # address-type parameter at all. Non-compliant, but ...
-                    for param in params:
-                        if param.startswith('<') and param.endswith('>'):
-                            address_set.append(param[1:-1])
-    # There may be both delayed and failed addresses.  If there are any failed
-    # addresses, return those, otherwise just stop processing.
-    if len(failed_addresses) == 0:
-        if len(delayed_addresses) == 0:
-            return set()
-        else:
-            return Stop
-    return set(parseaddr(address)[1] for address in failed_addresses
-               if address is not None)
-
-
-
 class DSN:
     """Parse RFC 3464 (i.e. DSN) bounce formats."""
 
     implements(IBounceDetector)
 
     def process(self, msg):
-        return check(msg)
-        ## # A DSN has been seen wrapped with a "legal disclaimer" by an outgoing
-        ## # MTA in a multipart/mixed outer part.
-        ## if msg.is_multipart() and msg.get_content_subtype() == 'mixed':
-        ##     msg = msg.get_payload()[0]
-        ## # The above will suffice if the original message 'parts' were wrapped
-        ## # with the disclaimer added, but the original DSN can be wrapped as a
-        ## # message/rfc822 part.  We need to test that too.
-        ## if msg.is_multipart() and msg.get_content_type() == 'message/rfc822':
-        ##     msg = msg.get_payload()[0]
-        ## # The report-type parameter should be "delivery-status", but it seems
-        ## # that some DSN generating MTAs don't include this on the
-        ## # Content-Type: header, so let's relax the test a bit.
-        ## if not msg.is_multipart() or msg.get_content_subtype() <> 'report':
-        ##     return set()
-        ## return check(msg)
+        """See `IBounceDetector`."""
+        # Iterate over each message/delivery-status subpart.
+        failed_addresses = []
+        delayed_addresses = []
+        for part in typed_subpart_iterator(msg, 'message', 'delivery-status'):
+            if not part.is_multipart():
+                # Huh?
+                continue
+            # Each message/delivery-status contains a list of Message objects
+            # which are the header blocks.  Iterate over those too.
+            for msgblock in part.get_payload():
+                address_set = None
+                # We try to dig out the Original-Recipient (which is optional)
+                # and Final-Recipient (which is mandatory, but may not exactly
+                # match an address on our list).  Some MTA's also use
+                # X-Actual-Recipient as a synonym for Original-Recipient, but
+                # some apparently use that for other purposes :(
+                #
+                # Also grok out Action so we can do something with that too.
+                action = msgblock.get('action', '').lower()
+                # Some MTAs have been observed that put comments on the action.
+                if action.startswith('delayed'):
+                    address_set = delayed_addresses
+                elif action.startswith('fail'):
+                    address_set = failed_addresses
+                else:
+                    # Some non-permanent failure, so ignore this block.
+                    continue
+                params = []
+                foundp = False
+                for header in ('original-recipient', 'final-recipient'):
+                    for k, v in msgblock.get_params([], header):
+                        if k.lower() == 'rfc822':
+                            foundp = True
+                        else:
+                            params.append(k)
+                    if foundp:
+                        # Note that params should already be unquoted.
+                        address_set.extend(params)
+                        break
+                    else:
+                        # MAS: This is a kludge, but
+                        # SMTP-GATEWAY01.intra.home.dk has a final-recipient
+                        # with an angle-addr and no address-type parameter at
+                        # all. Non-compliant, but ...
+                        for param in params:
+                            if param.startswith('<') and param.endswith('>'):
+                                address_set.append(param[1:-1])
+        # There may be both delayed and failed addresses.  If there are any
+        # failed addresses, return those, otherwise just stop processing.
+        if len(failed_addresses) == 0:
+            if len(delayed_addresses) == 0:
+                return set()
+            else:
+                return Stop
+        return set(parseaddr(address)[1] for address in failed_addresses
+                   if address is not None)