diff options
| -rw-r--r-- | src/mailman/config/schema.cfg | 19 | ||||
| -rw-r--r-- | src/mailman/mta/base.py | 127 | ||||
| -rw-r--r-- | src/mailman/mta/bulk.py | 78 | ||||
| -rw-r--r-- | src/mailman/mta/docs/bulk.txt | 4 | ||||
| -rw-r--r-- | src/mailman/mta/docs/verp.txt | 129 | ||||
| -rw-r--r-- | src/mailman/mta/verp.py | 84 |
6 files changed, 365 insertions, 76 deletions
diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg index 1520c4726..b0e7d52d9 100644 --- a/src/mailman/config/schema.cfg +++ b/src/mailman/config/schema.cfg @@ -306,11 +306,10 @@ delivery_retry_period: 5d # # http://cr.yp.to/proto/verp.txt # -# This involves encoding the address of the recipient as we (Mailman) know it -# into the envelope sender address (i.e. the SMTP `MAIL FROM:' address). -# Thus, no matter what kind of forwarding the recipient has in place, should -# it eventually bounce, we will receive an unambiguous notice of the bouncing -# address. +# This involves encoding the address of the recipient as Mailman knows it into +# the envelope sender address (i.e. RFC 5321 MAIL FROM). Thus, no matter what +# kind of forwarding the recipient has in place, should it eventually bounce, +# we will receive an unambiguous notice of the bouncing address. # # However, we're technically only "VERP-like" because we're doing the envelope # sender encoding in Mailman, not in the MTA. We do require cooperation from @@ -318,11 +317,11 @@ delivery_retry_period: 5d # semantics. # # The first variable describes how to encode VERP envelopes. It must contain -# these three string interpolations: +# these three string interpolations, which get filled in by Mailman: # -# $bounces -- the list-bounces mailbox will be set here -# $mailbox -- the recipient's mailbox will be set here -# $host -- the recipient's host name will be set here +# $bounces -- the list's -bounces robot address will be set here +# $local -- the recipient address's local mailbox part will be set here +# $domain -- the recipient address's domain name will be set here # # This example uses the default below. # @@ -334,7 +333,7 @@ delivery_retry_period: 5d # Note that your MTA /must/ be configured to deliver such an addressed message # to mylist-bounces! verp_delimiter: + -verp_format: ${bounces}+${mailbox}=${host} +verp_format: ${bounces}+${local}=${domain} # For nicer confirmation emails, use a VERP-like format which encodes the # confirmation cookie in the reply address. This lets us put a more user diff --git a/src/mailman/mta/base.py b/src/mailman/mta/base.py new file mode 100644 index 000000000..5942386a6 --- /dev/null +++ b/src/mailman/mta/base.py @@ -0,0 +1,127 @@ +# Copyright (C) 2009 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Base delivery class.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'BaseDelivery', + ] + + +import logging +import smtplib + +from zope.interface import implements + +from mailman.config import config +from mailman.interfaces.mta import IMailTransportAgentDelivery +from mailman.mta.connection import Connection + + +log = logging.getLogger('mailman.smtp') + + + +class BaseDelivery: + """Base delivery class.""" + + implements(IMailTransportAgentDelivery) + + def __init__(self, max_recipients=None): + """Create a basic deliverer. + + :param max_recipients: The maximum number of recipients per delivery + chunk. None, zero or less means to group all recipients into one + big chunk. + :type max_recipients: integer + """ + self._max_recipients = (max_recipients + if max_recipients is not None + else 0) + self._connection = Connection( + config.mta.smtp_host, int(config.mta.smtp_port), + self._max_recipients) + + def _deliver_to_recipients(self, mlist, msg, msgdata, + sender, recipients): + """Low-level delivery to a set of recipients. + + :param mlist: The mailing list being delivered to. + :type mlist: `IMailingList` + :param msg: The original message being delivered. + :type msg: `Message` + :param msgdata: Additional message metadata for this delivery. + :type msgdata: dictionary + :param sender: The envelope sender. + :type sender: string + :param recipients: The recipients of this message. + :type recipients: sequence + :return: delivery failures as defined by `smtplib.SMTP.sendmail` + :rtype: dictionary + """ + message_id = msg['message-id'] + try: + refused = self._connection.sendmail( + sender, recipients, msg.as_string()) + except smtplib.SMTPRecipientsRefused as error: + log.error('%s recipients refused: %s', message_id, error) + refused = error.recipients + except smtplib.SMTPResponseException as error: + log.error('%s response exception: %s', message_id, error) + refused = dict( + # recipient -> (code, error) + (recipient, (error.smtp_code, error.smtp_error)) + for recipient in recipients) + except (socket.error, IOError, smtplib.SMTPException) as error: + # MTA not responding, or other socket problems, or any other + # kind of SMTPException. In that case, nothing got delivered, + # so treat this as a temporary failure. We use error code 444 + # for this (temporary, unspecified failure, cf RFC 5321). + log.error('%s low level smtp error: %s', message_id, error) + error = str(error) + refused = dict( + # recipient -> (code, error) + (recipient, (444, error)) + for recipient in recipients) + return refused + + def _get_sender(self, mlist, msg, msgdata): + """Return the envelope sender to use. + + The message metadata can override the calculation of the sender, but + otherwise it falls to the list's -bounces robot. If this message is + not intended for any specific mailing list, the site owner's address + is used. + + :param mlist: The mailing list being delivered to. + :type mlist: `IMailingList` + :param msg: The original message being delivered. + :type msg: `Message` + :param msgdata: Additional message metadata for this delivery. + :type msgdata: dictionary + :return: The envelope sender. + :rtype: string + """ + sender = msgdata.get('sender') + if sender is None: + return (config.mailman.site_owner + if mlist is None + else mlist.bounces_address) + return sender diff --git a/src/mailman/mta/bulk.py b/src/mailman/mta/bulk.py index 13a6fb369..e516594d4 100644 --- a/src/mailman/mta/bulk.py +++ b/src/mailman/mta/bulk.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see <http://www.gnu.org/licenses/>. -"""Module stuff.""" +"""Bulk message delivery.""" from __future__ import absolute_import, unicode_literals @@ -25,19 +25,8 @@ __all__ = [ ] -import logging -import smtplib +from mailman.mta.base import BaseDelivery -from itertools import chain - -from zope.interface import implements - -from mailman.config import config -from mailman.interfaces.mta import IMailTransportAgentDelivery -from mailman.mta.connection import Connection - - -log = logging.getLogger('mailman.smtp') # A mapping of top-level domains to bucket numbers. The zeroth bucket is # reserved for everything else. At one time, these were the most common @@ -53,25 +42,8 @@ CHUNKMAP = dict( -class BulkDelivery: - """Deliver messages to the MTA in as few sessions as possible.""" - - implements(IMailTransportAgentDelivery) - - def __init__(self, max_recipients=None): - """Create a bulk deliverer. - - :param max_recipients: The maximum number of recipients per delivery - chunk. None, zero or less means to group all recipients into one - big chunk. - :type max_recipients: integer - """ - self._max_recipients = (max_recipients - if max_recipients is not None - else 0) - self._connection = Connection( - config.mta.smtp_host, int(config.mta.smtp_port), - self._max_recipients) +class BulkDelivery(BaseDelivery): + """Deliver messages to the MSA in as few sessions as possible.""" def chunkify(self, recipients): """Split a set of recipients into chunks. @@ -118,7 +90,8 @@ class BulkDelivery: def deliver(self, mlist, msg, msgdata): """See `IMailTransportAgentDelivery`.""" recipients = msgdata.get('recipients') - if recipients is None: + if not recipients: + # Could be None, could be an empty sequence. return # Blow away any existing Sender and Errors-To headers and substitute # our own. Our interpretation of RFC 5322 $3.6.2 is that Mailman is @@ -126,42 +99,15 @@ class BulkDelivery: # because what we send to list members is different than what the # original author sent. RFC 2076 says Errors-To is "non-standard, # discouraged" but we include it for historical purposes. + sender = self._get_sender(mlist, msg, msgdata) del msg['sender'] del msg['errors-to'] - # The message metadata can override the calculation of the sender, but - # otherwise it falls to the list's -bounces robot. If this message is - # not intended for any specific mailing list, the site owner's address - # is used. - sender = msgdata.get('sender') - if sender is None: - sender = (config.mailman.site_owner - if mlist is None - else mlist.bounces_address) msg['Sender'] = sender msg['Errors-To'] = sender - message_id = msg['message-id'] + refused = {} for recipients in self.chunkify(msgdata['recipients']): - try: - refused = self._connection.sendmail( - sender, recipients, msg.as_string()) - except smtplib.SMTPRecipientsRefused as error: - log.error('%s recipients refused: %s', message_id, error) - refused = error.recipients - except smtplib.SMTPResponseException as error: - log.error('%s response exception: %s', message_id, error) - refused = dict( - # recipient -> (code, error) - (recipient, (error.smtp_code, error.smtp_error)) - for recipient in recipients) - except (socket.error, IOError, smtplib.SMTPException) as error: - # MTA not responding, or other socket problems, or any other - # kind of SMTPException. In that case, nothing got delivered, - # so treat this as a temporary failure. We use error code 444 - # for this (temporary, unspecified failure, cf RFC 5321). - log.error('%s low level smtp error: %s', message_id, error) - error = str(error) - refused = dict( - # recipient -> (code, error) - (recipient, (444, error)) - for recipient in recipients) + chunk_refused = self._deliver_to_recipients( + mlist, msg, msgdata, sender, recipients) + refused.update(chunk_refused) return refused + diff --git a/src/mailman/mta/docs/bulk.txt b/src/mailman/mta/docs/bulk.txt index f1533ffe8..e40bdce16 100644 --- a/src/mailman/mta/docs/bulk.txt +++ b/src/mailman/mta/docs/bulk.txt @@ -158,6 +158,10 @@ there are no calculated recipients, nothing gets sent. >>> len(list(smtpd.messages)) 0 + >>> bulk.deliver(mlist, msg, dict(recipients=set())) + >>> len(list(smtpd.messages)) + 0 + With bulk delivery and no maximum number of recipients, there will be just one message sent, with all the recipients packed into the envelope recipients (i.e. RCTP TO). diff --git a/src/mailman/mta/docs/verp.txt b/src/mailman/mta/docs/verp.txt new file mode 100644 index 000000000..ac8d70e29 --- /dev/null +++ b/src/mailman/mta/docs/verp.txt @@ -0,0 +1,129 @@ +====================== +Standard VERP delivery +====================== + +'VERP' delivery is an alternative to bulk_ delivery, where an individual +message is crafted uniquely for each recipient. Our use of the term `VERP`_ +is actually incorrect, since the term is narrowly defined by the technique of +setting the return path so that bounces can be unambiguously decoded. + +.. _bulk: bulk.txt +.. _VERP: http://en.wikipedia.org/wiki/Variable_envelope_return_path + +The cost of enabling VERP is that Mailman must send to the upstream MTA, one +message per recipient. Under bulk delivery, an exact copy of one message can +be sent to many recipients, greatly reducing the bandwidth for delivery. + +In Mailman, enabling VERP delivery for bounce detection brings with it a side +benefit: the message which must be crafted uniquely for each recipient, can be +further personalized to include all kinds of information unique to that +recipient. In the simplest case, the message can contain footer information, +e.g. pointing the user to their account URL or including a user-specific +unsubscription link. In theory, VERP delivery means we can do sophisticated +`mail merge`_ operations. + +.. _`mail merge`: http://en.wikipedia.org/wiki/Mail_merge + +Mailman's use of the term VERP really means "message personalization". + + >>> from mailman.mta.verp import VERPDelivery + >>> verp = VERPDelivery() + +Delivery strategies must implement the proper interface. + + >>> from mailman.interfaces.mta import IMailTransportAgentDelivery + >>> from zope.interface.verify import verifyObject + >>> verifyObject(IMailTransportAgentDelivery, verp) + True + + +No recipients +============= + +The message metadata specifies the set of recipients to send this message to. +If there are no recipients, there's nothing to do. + + >>> smtpd.clear() + >>> mlist = create_list('test@example.com') + >>> msg = message_from_string("""\ + ... From: aperson@example.org + ... To: test@example.com + ... Subject: test one + ... Message-ID: <aardvark> + ... + ... This is a test. + ... """) + + >>> verp.deliver(mlist, msg, {}) + >>> len(list(smtpd.messages)) + 0 + + >>> verp.deliver(mlist, msg, dict(recipients=set())) + >>> len(list(smtpd.messages)) + 0 + + +Individual copy +=============== + +Each recipient of the message gets an individual, personalized copy of the +message, with their email address encoded into the envelope sender. This is +so the return path will point back to Mailman but allow for decoding of the +intended recipient's delivery address. + + >>> recipients = set([ + ... 'aperson@example.com', + ... 'bperson@example.com', + ... 'cperson@example.com', + ... ]) + + >>> verp.deliver(mlist, msg, dict(recipients=recipients)) + >>> messages = list(smtpd.messages) + >>> len(messages) + 3 + + >>> from operator import itemgetter + >>> for message in sorted(messages, key=itemgetter('x-rcptto')): + ... print message.as_string() + ... print '----------' + From: aperson@example.org + To: test@example.com + Subject: test one + Message-ID: <aardvark> + X-Peer: ... + X-MailFrom: test-bounces+aperson=example.com@example.com + X-RcptTo: aperson@example.com + <BLANKLINE> + This is a test. + ---------- + From: aperson@example.org + To: test@example.com + Subject: test one + Message-ID: <aardvark> + X-Peer: ... + X-MailFrom: test-bounces+bperson=example.com@example.com + X-RcptTo: bperson@example.com + <BLANKLINE> + This is a test. + ---------- + From: aperson@example.org + To: test@example.com + Subject: test one + Message-ID: <aardvark> + X-Peer: ... + X-MailFrom: test-bounces+cperson=example.com@example.com + X-RcptTo: cperson@example.com + <BLANKLINE> + This is a test. + ---------- + +The deliverer made a copy of the original message, so it wasn't changed. + + >>> print msg.as_string() + From: aperson@example.org + To: test@example.com + Subject: test one + Message-ID: <aardvark> + <BLANKLINE> + This is a test. + <BLANKLINE> diff --git a/src/mailman/mta/verp.py b/src/mailman/mta/verp.py new file mode 100644 index 000000000..be44afd10 --- /dev/null +++ b/src/mailman/mta/verp.py @@ -0,0 +1,84 @@ +# Copyright (C) 2009 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""VERP (i.e. personalized) message delivery.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'VERPDelivery', + ] + + +import copy +import logging + +from mailman.config import config +from mailman.email.utils import split_email +from mailman.mta.base import BaseDelivery +from mailman.utilities.string import expand + + +DOT = '.' +log = logging.getLogger('mailman.smtp') + + + +class VERPDelivery(BaseDelivery): + """Deliver a unique message to the MSA for each recipient.""" + + def deliver(self, mlist, msg, msgdata): + """See `IMailTransportAgentDelivery`. + + Craft a unique message for every recipient. Encode the recipient's + delivery address in the return envelope so there can be no ambiguity + in bounce processing. + """ + recipients = msgdata.get('recipients') + if not recipients: + # Could be None, could be an empty sequence. + return + sender = self._get_sender(mlist, msg, msgdata) + sender_mailbox, sender_domain = split_email(sender) + for recipient in recipients: + # Make a copy of the original messages and operator on it, since + # we're going to munge it repeatedly for each recipient. + message_copy = copy.deepcopy(msg) + # Encode the recipient's address for VERP. + recipient_mailbox, recipient_domain = split_email(recipient) + if recipient_domain is None: + # The recipient address is not fully-qualified. We can't + # deliver it to this person, nor can we craft a valid verp + # header. I don't think there's much we can do except ignore + # this recipient. + log.info('Skipping VERP delivery to unqual recip: %s', recip) + continue + verp_sender = '{0}@{1}'.format( + expand(config.mta.verp_format, dict( + bounces=sender_mailbox, + local=recipient_mailbox, + domain=DOT.join(recipient_domain))), + DOT.join(sender_domain)) + # We can flag the mail as a duplicate for each member, if they've + # already received this message, as calculated by Message-ID. See + # AvoidDuplicates.py for details. + del message_copy['x-mailman-copy'] + if recipient in msgdata.get('add-dup-header', {}): + message_copy['X-Mailman-Copy'] = 'yes' + self._deliver_to_recipients(mlist, msg, msgdata, + verp_sender, [recipient]) |