6 files changed, 365 insertions, 76 deletions

diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg
index 1520c4726..b0e7d52d9 100644
--- a/src/mailman/config/schema.cfg
+++ b/src/mailman/config/schema.cfg
@@ -306,11 +306,10 @@ delivery_retry_period: 5d
 #
 # http://cr.yp.to/proto/verp.txt
 #
-# This involves encoding the address of the recipient as we (Mailman) know it
-# into the envelope sender address (i.e. the SMTP `MAIL FROM:' address).
-# Thus, no matter what kind of forwarding the recipient has in place, should
-# it eventually bounce, we will receive an unambiguous notice of the bouncing
-# address.
+# This involves encoding the address of the recipient as Mailman knows it into
+# the envelope sender address (i.e. RFC 5321 MAIL FROM).  Thus, no matter what
+# kind of forwarding the recipient has in place, should it eventually bounce,
+# we will receive an unambiguous notice of the bouncing address.
 #
 # However, we're technically only "VERP-like" because we're doing the envelope
 # sender encoding in Mailman, not in the MTA.  We do require cooperation from
@@ -318,11 +317,11 @@ delivery_retry_period: 5d
 # semantics.
 #
 # The first variable describes how to encode VERP envelopes.  It must contain
-# these three string interpolations:
+# these three string interpolations, which get filled in by Mailman:
 #
-# $bounces -- the list-bounces mailbox will be set here
-# $mailbox -- the recipient's mailbox will be set here
-# $host    -- the recipient's host name will be set here
+# $bounces -- the list's -bounces robot address will be set here
+# $local   -- the recipient address's local mailbox part will be set here
+# $domain  -- the recipient address's domain name will be set here
 #
 # This example uses the default below.
 #
@@ -334,7 +333,7 @@ delivery_retry_period: 5d
 # Note that your MTA /must/ be configured to deliver such an addressed message
 # to mylist-bounces!
 verp_delimiter: +
-verp_format: ${bounces}+${mailbox}=${host}
+verp_format: ${bounces}+${local}=${domain}
 
 # For nicer confirmation emails, use a VERP-like format which encodes the
 # confirmation cookie in the reply address.  This lets us put a more user
diff --git a/src/mailman/mta/base.py b/src/mailman/mta/base.py
new file mode 100644
index 000000000..5942386a6
--- /dev/null
+++ b/src/mailman/mta/base.py
@@ -0,0 +1,127 @@
+# Copyright (C) 2009 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Base delivery class."""
+
+from __future__ import absolute_import, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    'BaseDelivery',
+    ]
+
+
+import logging
+import smtplib
+
+from zope.interface import implements
+
+from mailman.config import config
+from mailman.interfaces.mta import IMailTransportAgentDelivery
+from mailman.mta.connection import Connection
+
+
+log = logging.getLogger('mailman.smtp')
+
+
+
+class BaseDelivery:
+    """Base delivery class."""
+
+    implements(IMailTransportAgentDelivery)
+
+    def __init__(self, max_recipients=None):
+        """Create a basic deliverer.
+
+        :param max_recipients: The maximum number of recipients per delivery
+            chunk.  None, zero or less means to group all recipients into one
+            big chunk.
+        :type max_recipients: integer
+        """
+        self._max_recipients = (max_recipients
+                                if max_recipients is not None
+                                else 0)
+        self._connection = Connection(
+            config.mta.smtp_host, int(config.mta.smtp_port),
+            self._max_recipients)
+
+    def _deliver_to_recipients(self, mlist, msg, msgdata,
+                               sender, recipients):
+        """Low-level delivery to a set of recipients.
+
+        :param mlist: The mailing list being delivered to.
+        :type mlist: `IMailingList`
+        :param msg: The original message being delivered.
+        :type msg: `Message`
+        :param msgdata: Additional message metadata for this delivery.
+        :type msgdata: dictionary
+        :param sender: The envelope sender.
+        :type sender: string
+        :param recipients: The recipients of this message.
+        :type recipients: sequence
+        :return: delivery failures as defined by `smtplib.SMTP.sendmail`
+        :rtype: dictionary
+        """
+        message_id = msg['message-id']
+        try:
+            refused = self._connection.sendmail(
+                sender, recipients, msg.as_string())
+        except smtplib.SMTPRecipientsRefused as error:
+            log.error('%s recipients refused: %s', message_id, error)
+            refused = error.recipients
+        except smtplib.SMTPResponseException as error:
+            log.error('%s response exception: %s', message_id, error)
+            refused = dict(
+                # recipient -> (code, error)
+                (recipient, (error.smtp_code, error.smtp_error))
+                for recipient in recipients)
+        except (socket.error, IOError, smtplib.SMTPException) as error:
+            # MTA not responding, or other socket problems, or any other
+            # kind of SMTPException.  In that case, nothing got delivered,
+            # so treat this as a temporary failure.  We use error code 444
+            # for this (temporary, unspecified failure, cf RFC 5321).
+            log.error('%s low level smtp error: %s', message_id, error)
+            error = str(error)
+            refused = dict(
+                # recipient -> (code, error)
+                (recipient, (444, error))
+                for recipient in recipients)
+        return refused
+
+    def _get_sender(self, mlist, msg, msgdata):
+        """Return the envelope sender to use.
+
+        The message metadata can override the calculation of the sender, but
+        otherwise it falls to the list's -bounces robot.  If this message is
+        not intended for any specific mailing list, the site owner's address
+        is used.
+
+        :param mlist: The mailing list being delivered to.
+        :type mlist: `IMailingList`
+        :param msg: The original message being delivered.
+        :type msg: `Message`
+        :param msgdata: Additional message metadata for this delivery.
+        :type msgdata: dictionary
+        :return: The envelope sender.
+        :rtype: string
+        """
+        sender = msgdata.get('sender')
+        if sender is None:
+            return (config.mailman.site_owner
+                    if mlist is None
+                    else mlist.bounces_address)
+        return sender
diff --git a/src/mailman/mta/bulk.py b/src/mailman/mta/bulk.py
index 13a6fb369..e516594d4 100644
--- a/src/mailman/mta/bulk.py
+++ b/src/mailman/mta/bulk.py
@@ -15,7 +15,7 @@
 # You should have received a copy of the GNU General Public License along with
 # GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Module stuff."""
+"""Bulk message delivery."""
 
 from __future__ import absolute_import, unicode_literals
 
@@ -25,19 +25,8 @@ __all__ = [
     ]
 
 
-import logging
-import smtplib
+from mailman.mta.base import BaseDelivery
 
-from itertools import chain
-
-from zope.interface import implements
-
-from mailman.config import config
-from mailman.interfaces.mta import IMailTransportAgentDelivery
-from mailman.mta.connection import Connection
-
-
-log = logging.getLogger('mailman.smtp')
 
 # A mapping of top-level domains to bucket numbers.  The zeroth bucket is
 # reserved for everything else.  At one time, these were the most common
@@ -53,25 +42,8 @@ CHUNKMAP = dict(
 
 
 
-class BulkDelivery:
-    """Deliver messages to the MTA in as few sessions as possible."""
-
-    implements(IMailTransportAgentDelivery)
-
-    def __init__(self, max_recipients=None):
-        """Create a bulk deliverer.
-
-        :param max_recipients: The maximum number of recipients per delivery
-            chunk.  None, zero or less means to group all recipients into one
-            big chunk.
-        :type max_recipients: integer
-        """
-        self._max_recipients = (max_recipients
-                                if max_recipients is not None
-                                else 0)
-        self._connection = Connection(
-            config.mta.smtp_host, int(config.mta.smtp_port),
-            self._max_recipients)
+class BulkDelivery(BaseDelivery):
+    """Deliver messages to the MSA in as few sessions as possible."""
 
     def chunkify(self, recipients):
         """Split a set of recipients into chunks.
@@ -118,7 +90,8 @@ class BulkDelivery:
     def deliver(self, mlist, msg, msgdata):
         """See `IMailTransportAgentDelivery`."""
         recipients = msgdata.get('recipients')
-        if recipients is None:
+        if not recipients:
+            # Could be None, could be an empty sequence.
             return
         # Blow away any existing Sender and Errors-To headers and substitute
         # our own.  Our interpretation of RFC 5322 $3.6.2 is that Mailman is
@@ -126,42 +99,15 @@ class BulkDelivery:
         # because what we send to list members is different than what the
         # original author sent.  RFC 2076 says Errors-To is "non-standard,
         # discouraged" but we include it for historical purposes.
+        sender = self._get_sender(mlist, msg, msgdata)
         del msg['sender']
         del msg['errors-to']
-        # The message metadata can override the calculation of the sender, but
-        # otherwise it falls to the list's -bounces robot.  If this message is
-        # not intended for any specific mailing list, the site owner's address
-        # is used.
-        sender = msgdata.get('sender')
-        if sender is None:
-            sender = (config.mailman.site_owner
-                      if mlist is None
-                      else mlist.bounces_address)
         msg['Sender'] = sender
         msg['Errors-To'] = sender
-        message_id = msg['message-id']
+        refused = {}
         for recipients in self.chunkify(msgdata['recipients']):
-            try:
-                refused = self._connection.sendmail(
-                    sender, recipients, msg.as_string())
-            except smtplib.SMTPRecipientsRefused as error:
-                log.error('%s recipients refused: %s', message_id, error)
-                refused = error.recipients
-            except smtplib.SMTPResponseException as error:
-                log.error('%s response exception: %s', message_id, error)
-                refused = dict(
-                    # recipient -> (code, error)
-                    (recipient, (error.smtp_code, error.smtp_error))
-                    for recipient in recipients)
-            except (socket.error, IOError, smtplib.SMTPException) as error:
-                # MTA not responding, or other socket problems, or any other
-                # kind of SMTPException.  In that case, nothing got delivered,
-                # so treat this as a temporary failure.  We use error code 444
-                # for this (temporary, unspecified failure, cf RFC 5321).
-                log.error('%s low level smtp error: %s', message_id, error)
-                error = str(error)
-                refused = dict(
-                    # recipient -> (code, error)
-                    (recipient, (444, error))
-                    for recipient in recipients)
+            chunk_refused = self._deliver_to_recipients(
+                mlist, msg, msgdata, sender, recipients)
+            refused.update(chunk_refused)
         return refused
+
diff --git a/src/mailman/mta/docs/bulk.txt b/src/mailman/mta/docs/bulk.txt
index f1533ffe8..e40bdce16 100644
--- a/src/mailman/mta/docs/bulk.txt
+++ b/src/mailman/mta/docs/bulk.txt
@@ -158,6 +158,10 @@ there are no calculated recipients, nothing gets sent.
     >>> len(list(smtpd.messages))
     0
 
+    >>> bulk.deliver(mlist, msg, dict(recipients=set()))
+    >>> len(list(smtpd.messages))
+    0
+
 With bulk delivery and no maximum number of recipients, there will be just one
 message sent, with all the recipients packed into the envelope recipients
 (i.e. RCTP TO).
diff --git a/src/mailman/mta/docs/verp.txt b/src/mailman/mta/docs/verp.txt
new file mode 100644
index 000000000..ac8d70e29
--- /dev/null
+++ b/src/mailman/mta/docs/verp.txt
@@ -0,0 +1,129 @@
+======================
+Standard VERP delivery
+======================
+
+'VERP' delivery is an alternative to bulk_ delivery, where an individual
+message is crafted uniquely for each recipient.  Our use of the term `VERP`_
+is actually incorrect, since the term is narrowly defined by the technique of
+setting the return path so that bounces can be unambiguously decoded.
+
+.. _bulk: bulk.txt
+.. _VERP: http://en.wikipedia.org/wiki/Variable_envelope_return_path
+
+The cost of enabling VERP is that Mailman must send to the upstream MTA, one
+message per recipient.  Under bulk delivery, an exact copy of one message can
+be sent to many recipients, greatly reducing the bandwidth for delivery.
+
+In Mailman, enabling VERP delivery for bounce detection brings with it a side
+benefit: the message which must be crafted uniquely for each recipient, can be
+further personalized to include all kinds of information unique to that
+recipient.  In the simplest case, the message can contain footer information,
+e.g.  pointing the user to their account URL or including a user-specific
+unsubscription link.  In theory, VERP delivery means we can do sophisticated
+`mail merge`_ operations.
+
+.. _`mail merge`: http://en.wikipedia.org/wiki/Mail_merge
+
+Mailman's use of the term VERP really means "message personalization".
+
+    >>> from mailman.mta.verp import VERPDelivery
+    >>> verp = VERPDelivery()
+
+Delivery strategies must implement the proper interface.
+
+    >>> from mailman.interfaces.mta import IMailTransportAgentDelivery
+    >>> from zope.interface.verify import verifyObject
+    >>> verifyObject(IMailTransportAgentDelivery, verp)
+    True
+
+
+No recipients
+=============
+
+The message metadata specifies the set of recipients to send this message to.
+If there are no recipients, there's nothing to do.
+
+    >>> smtpd.clear()
+    >>> mlist = create_list('test@example.com')
+    >>> msg = message_from_string("""\
+    ... From: aperson@example.org
+    ... To: test@example.com
+    ... Subject: test one
+    ... Message-ID: <aardvark>
+    ...
+    ... This is a test.
+    ... """)
+
+    >>> verp.deliver(mlist, msg, {})
+    >>> len(list(smtpd.messages))
+    0
+
+    >>> verp.deliver(mlist, msg, dict(recipients=set()))
+    >>> len(list(smtpd.messages))
+    0
+
+
+Individual copy
+===============
+
+Each recipient of the message gets an individual, personalized copy of the
+message, with their email address encoded into the envelope sender.  This is
+so the return path will point back to Mailman but allow for decoding of the
+intended recipient's delivery address.
+
+    >>> recipients = set([
+    ...     'aperson@example.com',
+    ...     'bperson@example.com',
+    ...     'cperson@example.com',
+    ...     ])
+
+    >>> verp.deliver(mlist, msg, dict(recipients=recipients))
+    >>> messages = list(smtpd.messages)
+    >>> len(messages)
+    3
+
+    >>> from operator import itemgetter
+    >>> for message in sorted(messages, key=itemgetter('x-rcptto')):
+    ...     print message.as_string()
+    ...     print '----------'
+    From: aperson@example.org
+    To: test@example.com
+    Subject: test one
+    Message-ID: <aardvark>
+    X-Peer: ...
+    X-MailFrom: test-bounces+aperson=example.com@example.com
+    X-RcptTo: aperson@example.com
+    <BLANKLINE>
+    This is a test.
+    ----------
+    From: aperson@example.org
+    To: test@example.com
+    Subject: test one
+    Message-ID: <aardvark>
+    X-Peer: ...
+    X-MailFrom: test-bounces+bperson=example.com@example.com
+    X-RcptTo: bperson@example.com
+    <BLANKLINE>
+    This is a test.
+    ----------
+    From: aperson@example.org
+    To: test@example.com
+    Subject: test one
+    Message-ID: <aardvark>
+    X-Peer: ...
+    X-MailFrom: test-bounces+cperson=example.com@example.com
+    X-RcptTo: cperson@example.com
+    <BLANKLINE>
+    This is a test.
+    ----------
+
+The deliverer made a copy of the original message, so it wasn't changed.
+
+    >>> print msg.as_string()
+    From: aperson@example.org
+    To: test@example.com
+    Subject: test one
+    Message-ID: <aardvark>
+    <BLANKLINE>
+    This is a test.
+    <BLANKLINE>
diff --git a/src/mailman/mta/verp.py b/src/mailman/mta/verp.py
new file mode 100644
index 000000000..be44afd10
--- /dev/null
+++ b/src/mailman/mta/verp.py
@@ -0,0 +1,84 @@
+# Copyright (C) 2009 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""VERP (i.e. personalized) message delivery."""
+
+from __future__ import absolute_import, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    'VERPDelivery',
+    ]
+
+
+import copy
+import logging
+
+from mailman.config import config
+from mailman.email.utils import split_email
+from mailman.mta.base import BaseDelivery
+from mailman.utilities.string import expand
+
+
+DOT = '.'
+log = logging.getLogger('mailman.smtp')
+
+
+
+class VERPDelivery(BaseDelivery):
+    """Deliver a unique message to the MSA for each recipient."""
+
+    def deliver(self, mlist, msg, msgdata):
+        """See `IMailTransportAgentDelivery`.
+
+        Craft a unique message for every recipient.  Encode the recipient's
+        delivery address in the return envelope so there can be no ambiguity
+        in bounce processing.
+        """
+        recipients = msgdata.get('recipients')
+        if not recipients:
+            # Could be None, could be an empty sequence.
+            return
+        sender = self._get_sender(mlist, msg, msgdata)
+        sender_mailbox, sender_domain = split_email(sender)
+        for recipient in recipients:
+            # Make a copy of the original messages and operator on it, since
+            # we're going to munge it repeatedly for each recipient.
+            message_copy = copy.deepcopy(msg)
+            # Encode the recipient's address for VERP.
+            recipient_mailbox, recipient_domain = split_email(recipient)
+            if recipient_domain is None:
+                # The recipient address is not fully-qualified.  We can't
+                # deliver it to this person, nor can we craft a valid verp
+                # header.  I don't think there's much we can do except ignore
+                # this recipient.
+                log.info('Skipping VERP delivery to unqual recip: %s', recip)
+                continue
+            verp_sender = '{0}@{1}'.format(
+                expand(config.mta.verp_format, dict(
+                    bounces=sender_mailbox,
+                    local=recipient_mailbox,
+                    domain=DOT.join(recipient_domain))),
+                DOT.join(sender_domain))
+            # We can flag the mail as a duplicate for each member, if they've
+            # already received this message, as calculated by Message-ID.  See
+            # AvoidDuplicates.py for details.
+            del message_copy['x-mailman-copy']
+            if recipient in msgdata.get('add-dup-header', {}):
+                message_copy['X-Mailman-Copy'] = 'yes'
+            self._deliver_to_recipients(mlist, msg, msgdata,
+                                        verp_sender, [recipient])