5 files changed, 204 insertions, 9 deletions

diff --git a/src/mailman/interfaces/bounce.py b/src/mailman/interfaces/bounce.py
index 168ca7ee0..0b301aa98 100644
--- a/src/mailman/interfaces/bounce.py
+++ b/src/mailman/interfaces/bounce.py
@@ -113,3 +113,9 @@ class IBounceProcessor(Interface):
         :return: The registered bounce event.
         :rtype: IBounceEvent
         """
+
+    events = Attribute(
+        """An iterator over all events.""")
+
+    unprocessed = Attribute(
+        """An iterator over all unprocessed bounce events.""")
diff --git a/src/mailman/model/bounce.py b/src/mailman/model/bounce.py
index 8abe3d149..20953b0ff 100644
--- a/src/mailman/model/bounce.py
+++ b/src/mailman/model/bounce.py
@@ -29,10 +29,11 @@ __all__ = [
 from storm.locals import Bool, Int, DateTime, Unicode
 from zope.interface import implements
 
-from mailman.interfaces.bounce import (
-    BounceContext, IBounceEvent, IBounceProcessor)
+from mailman.config import config
 from mailman.database.model import Model
 from mailman.database.types import Enum
+from mailman.interfaces.bounce import (
+    BounceContext, IBounceEvent, IBounceProcessor)
 from mailman.utilities.datetime import now
 
 
@@ -63,4 +64,19 @@ class BounceProcessor:
 
     def register(self, mlist, email, msg, where=None):
         """See `IBounceProcessor`."""
-        return BounceEvent(mlist.fqdn_listname, email, msg, where)
+        event = BounceEvent(mlist.fqdn_listname, email, msg, where)
+        config.db.store.add(event)
+        return event
+
+    @property
+    def events(self):
+        """See `IBounceProcessor`."""
+        for event in config.db.store.find(BounceEvent):
+            yield event
+
+    @property
+    def unprocessed(self):
+        """See `IBounceProcessor`."""
+        for event in config.db.store.find(BounceEvent,
+                                          BounceEvent.processed == False):
+            yield event
diff --git a/src/mailman/model/tests/test_bounce.py b/src/mailman/model/tests/test_bounce.py
index fb0bf0875..a232b37fd 100644
--- a/src/mailman/model/tests/test_bounce.py
+++ b/src/mailman/model/tests/test_bounce.py
@@ -27,7 +27,14 @@ __all__ = [
 
 import unittest
 
-from mailman.model.bounce import BounceEvent
+from datetime import datetime
+from zope.component import getUtility
+
+from mailman.app.lifecycle import create_list
+from mailman.config import config
+from mailman.interfaces.bounce import BounceContext, IBounceProcessor
+from mailman.testing.helpers import (
+    specialized_message_from_string as message_from_string)
 from mailman.testing.layers import ConfigLayer
 
 
@@ -35,8 +42,60 @@ from mailman.testing.layers import ConfigLayer
 class TestBounceEvents(unittest.TestCase):
     layer = ConfigLayer
 
-    
+    def setUp(self):
+        self._processor = getUtility(IBounceProcessor)
+        self._mlist = create_list('test@example.com')
+        self._msg = message_from_string("""\
+From: mail-daemon@example.com
+To: test-bounces@example.com
+Message-Id: <first>
+
+""")
+
+    def test_events_iterator(self):
+        self._processor.register(self._mlist, 'anne@example.com', self._msg)
+        config.db.commit()
+        events = list(self._processor.events)
+        self.assertEqual(len(events), 1)
+        event = events[0]
+        self.assertEqual(event.list_name, 'test@example.com')
+        self.assertEqual(event.email, 'anne@example.com')
+        self.assertEqual(event.timestamp, datetime(2005, 8, 1, 7, 49, 23))
+        self.assertEqual(event.message_id, '<first>')
+        self.assertEqual(event.context, BounceContext.normal)
+        self.assertEqual(event.processed, False)
+        # The unprocessed list will be exactly the same right now.
+        unprocessed = list(self._processor.unprocessed)
+        self.assertEqual(len(unprocessed), 1)
+        event = unprocessed[0]
+        self.assertEqual(event.list_name, 'test@example.com')
+        self.assertEqual(event.email, 'anne@example.com')
+        self.assertEqual(event.timestamp, datetime(2005, 8, 1, 7, 49, 23))
+        self.assertEqual(event.message_id, '<first>')
+        self.assertEqual(event.context, BounceContext.normal)
+        self.assertEqual(event.processed, False)
 
+    def test_unprocessed_events_iterator(self):
+        self._processor.register(self._mlist, 'anne@example.com', self._msg)
+        self._processor.register(self._mlist, 'bart@example.com', self._msg)
+        config.db.commit()
+        events = list(self._processor.events)
+        self.assertEqual(len(events), 2)
+        unprocessed = list(self._processor.unprocessed)
+        # The unprocessed list will be exactly the same right now.
+        self.assertEqual(len(unprocessed), 2)
+        # Process one of the events.
+        events[0].processed = True
+        config.db.commit()
+        # Now there will be only one unprocessed event.
+        unprocessed = list(self._processor.unprocessed)
+        self.assertEqual(len(unprocessed), 1)
+        # Process the other event.
+        events[1].processed = True
+        config.db.commit()
+        # Now there will be no unprocessed events.
+        unprocessed = list(self._processor.unprocessed)
+        self.assertEqual(len(unprocessed), 0)
 
 
 
diff --git a/src/mailman/queue/outgoing.py b/src/mailman/queue/outgoing.py
index 6a8e2b08d..f20adc270 100644
--- a/src/mailman/queue/outgoing.py
+++ b/src/mailman/queue/outgoing.py
@@ -22,10 +22,14 @@ import logging
 
 from datetime import datetime
 from lazr.config import as_boolean, as_timedelta
+from zope.component import getUtility
 
 from mailman.config import config
+from mailman.interfaces.bounce import BounceContext, IBounceProcessor
 from mailman.interfaces.mailinglist import Personalization
+from mailman.interfaces.membership import ISubscriptionService
 from mailman.interfaces.mta import SomeRecipientsFailed
+from mailman.interfaces.pending import IPendings
 from mailman.queue import Runner
 from mailman.queue.bounce import BounceMixin
 from mailman.utilities.datetime import now
@@ -98,9 +102,24 @@ class OutgoingRunner(Runner, BounceMixin):
                 self._logged = True
             return True
         except SomeRecipientsFailed as error:
-            # Handle local rejects of probe messages differently.
-            if msgdata.get('probe_token') and error.permanent_failures:
-                self._probe_bounce(mlist, msgdata['probe_token'])
+            if 'probe_token' in msgdata:
+                # This is a failure of our local MTA to deliver to a probe
+                # message recipient.  Register the bounce event for permanent
+                # failures.  Start by grabbing and confirming (i.e. removing)
+                # the pendable record associated with this bounce token,
+                # regardless of what address was actually failing.
+                if len(error.permanent_failures) > 0:
+                    pended = getUtility(IPendings).confirm(
+                        msgdata['probe_token'])
+                    # It's possible the token has been confirmed out of the
+                    # database.  Just ignore that.
+                    if pended is not None:
+                        member = getUtility(ISubscriptionService).get_member(
+                            pended['member_id'])
+                        processor = getUtility(IBounceProcessor)
+                        processor.register(
+                            mlist, member.address.email, msg,
+                            BounceContext.probe)
             else:
                 # Delivery failed at SMTP time for some or all of the
                 # recipients.  Permanent failures are registered as bounces,
diff --git a/src/mailman/queue/tests/test_outgoing.py b/src/mailman/queue/tests/test_outgoing.py
index c86a1efa7..da45dbdb5 100644
--- a/src/mailman/queue/tests/test_outgoing.py
+++ b/src/mailman/queue/tests/test_outgoing.py
@@ -31,11 +31,18 @@ import logging
 import unittest
 
 from contextlib import contextmanager
-from datetime import timedelta
+from datetime import datetime, timedelta
+from zope.component import getUtility
 
+from mailman.app.bounces import send_probe
 from mailman.app.lifecycle import create_list
 from mailman.config import config
+from mailman.interfaces.bounce import BounceContext, IBounceProcessor
 from mailman.interfaces.mailinglist import Personalization
+from mailman.interfaces.member import MemberRole
+from mailman.interfaces.mta import SomeRecipientsFailed
+from mailman.interfaces.pending import IPendings
+from mailman.interfaces.usermanager import IUserManager
 from mailman.queue.outgoing import OutgoingRunner
 from mailman.testing.helpers import (
     get_queue_messages,
@@ -319,9 +326,97 @@ Message-Id: <first>
 
 
 
+temporary_failures = []
+permanent_failures = []
+
+
+def raise_SomeRecipientsFailed(mlist, msg, msgdata):
+    raise SomeRecipientsFailed(temporary_failures, permanent_failures)
+
+
+class TestSomeRecipientsFailed(unittest.TestCase):
+    """Test socket.error occurring in the delivery function."""
+
+    layer = ConfigLayer
+
+    def setUp(self):
+        global temporary_failures, permanent_failures
+        del temporary_failures[:]
+        del permanent_failures[:]
+        # Push a config where actual delivery is handled by a dummy function.
+        # We generally don't care what this does, since we're just testing the
+        # setting of the 'verp' key in the metadata.
+        config.push('fake outgoing', """
+        [mta]
+        outgoing: mailman.queue.tests.test_outgoing.raise_SomeRecipientsFailed
+        """)
+        self._mlist = create_list('test@example.com')
+        self._outq = config.switchboards['out']
+        self._runner = make_testable_runner(OutgoingRunner, 'out', run_once)
+        self._msg = message_from_string("""\
+From: anne@example.com
+To: test@example.com
+Message-Id: <first>
+
+""")
+
+    def tearDown(self):
+        config.pop('fake outgoing')
+
+    def test_probe_failure(self):
+        # When a probe message fails during SMTP, a bounce event is recorded
+        # with the proper bounce context.
+        anne = getUtility(IUserManager).create_address('anne@example.com')
+        member = self._mlist.subscribe(anne, MemberRole.member)
+        token = send_probe(member, self._msg)
+        msgdata = dict(probe_token=token)
+        permanent_failures.append('anne@example.com')
+        self._outq.enqueue(self._msg, msgdata, listname='test@example.com')
+        self._runner.run()
+        events = list(getUtility(IBounceProcessor).unprocessed)
+        self.assertEqual(len(events), 1)
+        event = events[0]
+        self.assertEqual(event.list_name, 'test@example.com')
+        self.assertEqual(event.email, 'anne@example.com')
+        self.assertEqual(event.timestamp, datetime(2005, 8, 1, 7, 49, 23))
+        self.assertEqual(event.message_id, '<first>')
+        self.assertEqual(event.context, BounceContext.probe)
+        self.assertEqual(event.processed, False)
+
+    def test_confirmed_probe_failure(self):
+        # This time, a probe also fails, but for some reason the probe token
+        # has already been confirmed and no longer exists in the database.
+        anne = getUtility(IUserManager).create_address('anne@example.com')
+        member = self._mlist.subscribe(anne, MemberRole.member)
+        token = send_probe(member, self._msg)
+        getUtility(IPendings).confirm(token)
+        msgdata = dict(probe_token=token)
+        permanent_failures.append('anne@example.com')
+        self._outq.enqueue(self._msg, msgdata, listname='test@example.com')
+        self._runner.run()
+        events = list(getUtility(IBounceProcessor).unprocessed)
+        self.assertEqual(len(events), 0)
+
+    def test_probe_temporary_failure(self):
+        # This time, a probe also fails, but the failures are temporary so
+        # they are not registered.
+        anne = getUtility(IUserManager).create_address('anne@example.com')
+        member = self._mlist.subscribe(anne, MemberRole.member)
+        token = send_probe(member, self._msg)
+        getUtility(IPendings).confirm(token)
+        msgdata = dict(probe_token=token)
+        temporary_failures.append('anne@example.com')
+        self._outq.enqueue(self._msg, msgdata, listname='test@example.com')
+        self._runner.run()
+        events = list(getUtility(IBounceProcessor).unprocessed)
+        self.assertEqual(len(events), 0)
+
+
+
 def test_suite():
     suite = unittest.TestSuite()
     suite.addTest(unittest.makeSuite(TestOnce))
     suite.addTest(unittest.makeSuite(TestVERPSettings))
     suite.addTest(unittest.makeSuite(TestSocketError))
+    suite.addTest(unittest.makeSuite(TestSomeRecipientsFailed))
     return suite