4 files changed, 114 insertions, 12 deletions

diff --git a/src/mailman/interfaces/mta.py b/src/mailman/interfaces/mta.py
index f8dcb64b3..da0943e25 100644
--- a/src/mailman/interfaces/mta.py
+++ b/src/mailman/interfaces/mta.py
@@ -67,4 +67,6 @@ class IMailTransportAgentDelivery(Interface):
         :type msg: `Message`
         :param msgdata: Additional message metadata for this delivery.
         :type msgdata: dictionary
+        :return: delivery failures as defined by `smtplib.SMTP.sendmail`
+        :rtype: dictionary
         """
diff --git a/src/mailman/mta/bulk.py b/src/mailman/mta/bulk.py
index ff00f3d20..21bbd1713 100644
--- a/src/mailman/mta/bulk.py
+++ b/src/mailman/mta/bulk.py
@@ -25,6 +25,9 @@ __all__ = [
     ]
 
 
+import logging
+import smtplib
+
 from itertools import chain
 
 from zope.interface import implements
@@ -34,6 +37,8 @@ from mailman.interfaces.mta import IMailTransportAgentDelivery
 from mailman.mta.connection import Connection
 
 
+log = logging.getLogger('mailman.smtp')
+
 # A mapping of top-level domains to bucket numbers.  The zeroth bucket is
 # reserved for everything else.  At one time, these were the most common
 # domains.
@@ -134,6 +139,12 @@ class BulkDelivery:
                       else mlist.bounces_address)
         msg['Sender'] = sender
         msg['Errors-To'] = sender
+        message_id = msg['message-id']
         for recipients in self.chunkify(msgdata['recipients']):
-            self._connection.sendmail(
-                'foo@example.com', recipients, msg.as_string())
+            try:
+                refused = self._connection.sendmail(
+                    sender, recipients, msg.as_string())
+            except smtplib.SMTPRecipientsRefused as error:
+                log.error('%s recipients refused: %s', message_id, error)
+                refused = error.recipients
+        return refused
diff --git a/src/mailman/mta/docs/bulk.txt b/src/mailman/mta/docs/bulk.txt
index a612e8ac9..fabc265b6 100644
--- a/src/mailman/mta/docs/bulk.txt
+++ b/src/mailman/mta/docs/bulk.txt
@@ -165,6 +165,8 @@ message sent, with all the recipients packed into the envelope recipients
     >>> recipients = set('person_{0:02d}'.format(i) for i in range(100))
     >>> msgdata = dict(recipients=recipients)
     >>> bulk.deliver(mlist, msg, msgdata)
+    {}
+
     >>> messages = list(smtpd.messages)
     >>> len(messages)
     1
@@ -190,6 +192,8 @@ each with 20 addresses in the RCPT TO header.
 
     >>> bulk = BulkDelivery(20)
     >>> bulk.deliver(mlist, msg, msgdata)
+    {}
+
     >>> messages = list(smtpd.messages)
     >>> len(messages)
     5
@@ -231,6 +235,8 @@ message metadata...
     >>> msgdata = dict(recipients=recipients,
     ...                sender='asender@example.org')
     >>> bulk.deliver(mlist, msg, msgdata)
+    {}
+
     >>> message = list(smtpd.messages)[0]
     >>> print message.as_string()
     From: aperson@example.org
@@ -240,7 +246,7 @@ message metadata...
     Sender: asender@example.org
     Errors-To: asender@example.org
     X-Peer: ...
-    X-MailFrom: foo@example.com
+    X-MailFrom: asender@example.org
     X-RcptTo: aperson@example.com
     <BLANKLINE>
     This is a test.
@@ -249,6 +255,8 @@ message metadata...
 
     >>> del msgdata['sender']
     >>> bulk.deliver(mlist, msg, msgdata)
+    {}
+
     >>> message = list(smtpd.messages)[0]
     >>> print message.as_string()
     From: aperson@example.org
@@ -258,7 +266,7 @@ message metadata...
     Sender: test-bounces@example.com
     Errors-To: test-bounces@example.com
     X-Peer: ...
-    X-MailFrom: foo@example.com
+    X-MailFrom: test-bounces@example.com
     X-RcptTo: aperson@example.com
     <BLANKLINE>
     This is a test.
@@ -272,6 +280,8 @@ message.
     ... """)
 
     >>> bulk.deliver(None, msg, msgdata)
+    {}
+
     >>> message = list(smtpd.messages)[0]
     >>> print message.as_string()
     From: aperson@example.org
@@ -281,7 +291,7 @@ message.
     Sender: site-owner@example.com
     Errors-To: site-owner@example.com
     X-Peer: ...
-    X-MailFrom: foo@example.com
+    X-MailFrom: site-owner@example.com
     X-RcptTo: aperson@example.com
     <BLANKLINE>
     This is a test.
@@ -301,6 +311,8 @@ deleted first.
     ... """)
 
     >>> bulk.deliver(mlist, msg, msgdata)
+    {}
+
     >>> message = list(smtpd.messages)[0]
     >>> print message.as_string()
     From: bperson@example.org
@@ -310,9 +322,54 @@ deleted first.
     Sender: test-bounces@example.com
     Errors-To: test-bounces@example.com
     X-Peer: ...
-    X-MailFrom: foo@example.com
+    X-MailFrom: test-bounces@example.com
     X-RcptTo: aperson@example.com
     <BLANKLINE>
     This is a test.
 
     >>> config.pop('site-owner')
+
+
+Delivery failures
+=================
+
+Mailman does not do final delivery.  Instead, it sends mail through a site
+local mail server which manages queuing and final delivery.  However, even
+this local mail server can produce delivery failures visible to Mailman in
+certain situations.
+
+For example, something could be seriously wrong with the mail server and it
+could refuse delivery to all recipients.
+
+    # Tell the mail server to fail on the next 3 RCPT TO commands, one for
+    # each recipient in the following message.
+    >>> smtpd.err_queue.put('rcpt')
+    >>> smtpd.err_queue.put('rcpt')
+    >>> smtpd.err_queue.put('rcpt')
+
+    >>> recipients = set([
+    ...     'aperson@example.org',
+    ...     'bperson@example.org',
+    ...     'cperson@example.org',
+    ...     ])
+    >>> msgdata = dict(recipients=recipients)
+
+    >>> msg = message_from_string("""\
+    ... From: aperson@example.org
+    ... To: test@example.com
+    ... Subject: test three
+    ... Message-ID: <camel>
+    ...
+    ... This is a test.
+    ... """)
+
+    >>> failures = bulk.deliver(mlist, msg, msgdata)
+    >>> for address in sorted(failures):
+    ...     print address, failures[address][0], failures[address][1]
+    aperson@example.org 500 Error: SMTPRecipientsRefused
+    bperson@example.org 500 Error: SMTPRecipientsRefused
+    cperson@example.org 500 Error: SMTPRecipientsRefused
+
+    >>> messages = list(smtpd.messages)
+    >>> len(messages)
+    0
diff --git a/src/mailman/testing/mta.py b/src/mailman/testing/mta.py
index 6e0ee45b9..81852a24b 100644
--- a/src/mailman/testing/mta.py
+++ b/src/mailman/testing/mta.py
@@ -27,7 +27,7 @@ __all__ = [
 
 import logging
 
-from Queue import Queue
+from Queue import Empty, Queue
 
 from lazr.smtptest.controller import QueueController
 from lazr.smtptest.server import Channel, QueueServer
@@ -62,20 +62,51 @@ class StatisticsChannel(Channel):
     def smtp_STAT(self, arg):
         """Cause the server to send statistics to its controller."""
         self._server.send_statistics()
-        self.push('250 Ok')
+        self.push(b'250 Ok')
+
+    def smtp_RCPT(self, arg):
+        """For testing, sometimes cause a non-25x response."""
+        if self._server.next_error == 'rcpt':
+            # The test suite wants this to fail.  The message corresponds to
+            # the exception we expect smtplib.SMTP to raise.
+            self.push(b'500 Error: SMTPRecipientsRefused')
+        else:
+            # Everything's cool.
+            Channel.smtp_RCPT(self, arg)
 
 
 
 class ConnectionCountingServer(QueueServer):
     """Count the number of SMTP connections opened."""
 
-    def __init__(self, host, port, queue, oob_queue):
-        """See `lazr.smtptest.server.QueueServer`."""
+    def __init__(self, host, port, queue, oob_queue, err_queue):
+        """See `lazr.smtptest.server.QueueServer`.
+
+        :param oob_queue: A queue for communicating information back to the
+            controller, e.g. statistics.
+        :type oob_queue: `Queue.Queue`
+        :param err_queue: A queue for allowing the controller to request SMTP
+            errors from the server.
+        :type err_queue: `Queue.Queue`
+        """
         QueueServer.__init__(self, host, port, queue)
         self._connection_count = 0
         # The out-of-band queue is where the server sends statistics to the
         # controller upon request.
         self._oob_queue = oob_queue
+        self._err_queue = err_queue
+
+    @property
+    def next_error(self):
+        """Return the next error, or None if nothing's on the stack.
+
+        :return: The next SMTP command that should error.
+        :rtype: string (lower cased) or None
+        """
+        try:
+            return self._err_queue.get_nowait()
+        except Empty:
+            return None
 
     def handle_accept(self):
         """See `lazr.smtp.server.Server`."""
@@ -103,12 +134,13 @@ class ConnectionCountingController(QueueController):
     def __init__(self, host, port):
         """See `lazr.smtptest.controller.QueueController`."""
         self.oob_queue = Queue()
+        self.err_queue = Queue()
         QueueController.__init__(self, host, port)
 
     def _make_server(self, host, port):
         """See `lazr.smtptest.controller.QueueController`."""
         self.server = ConnectionCountingServer(
-            host, port, self.queue, self.oob_queue)
+            host, port, self.queue, self.oob_queue, self.err_queue)
 
     def start(self):
         """See `lazr.smtptest.controller.QueueController`."""
@@ -124,7 +156,7 @@ class ConnectionCountingController(QueueController):
         :rtype: integer
         """
         smtpd = self._connect()
-        smtpd.docmd('STAT')
+        smtpd.docmd(b'STAT')
         # An Empty exception will occur if the data isn't available in 10
         # seconds.  Let that propagate.
         return self.oob_queue.get(block=True, timeout=10)