diff options
| -rw-r--r-- | src/mailman/archiving/docs/common.txt | 9 | ||||
| -rw-r--r-- | src/mailman/config/schema.cfg | 2 | ||||
| -rw-r--r-- | src/mailman/core/errors.py | 13 | ||||
| -rw-r--r-- | src/mailman/interfaces/mta.py | 11 | ||||
| -rw-r--r-- | src/mailman/mta/base.py | 25 | ||||
| -rw-r--r-- | src/mailman/mta/bulk.py | 19 | ||||
| -rw-r--r-- | src/mailman/mta/deliver.py | 154 | ||||
| -rw-r--r-- | src/mailman/mta/docs/bulk.txt | 2 | ||||
| -rw-r--r-- | src/mailman/mta/docs/decorating.txt | 24 | ||||
| -rw-r--r-- | src/mailman/mta/docs/personalized.txt | 36 | ||||
| -rw-r--r-- | src/mailman/mta/docs/verp.txt | 13 | ||||
| -rw-r--r-- | src/mailman/mta/smtp_direct.py | 374 | ||||
| -rw-r--r-- | src/mailman/mta/verp.py | 55 | ||||
| -rw-r--r-- | src/mailman/pipeline/to_outgoing.py | 2 | ||||
| -rw-r--r-- | src/mailman/queue/outgoing.py | 16 |
15 files changed, 285 insertions, 470 deletions
diff --git a/src/mailman/archiving/docs/common.txt b/src/mailman/archiving/docs/common.txt index cde0ada73..42cdaa12e 100644 --- a/src/mailman/archiving/docs/common.txt +++ b/src/mailman/archiving/docs/common.txt @@ -84,7 +84,7 @@ automatically. http://go.mail-archive.dev/ZaXPPxRMM9_hFZL4vTRlQlBx8pc= To archive the message, the archiver actually mails the message to a special -address at the Mail-Archive. +address at the Mail-Archive. The message gets no header or footer decoration. >>> archiver.archive_message(mlist, msg) @@ -104,9 +104,6 @@ address at the Mail-Archive. Subject: An archived message Message-ID: <12345> X-Message-ID-Hash: ZaXPPxRMM9_hFZL4vTRlQlBx8pc= - MIME-Version: 1.0 - Content-Type: text/plain; charset="us-ascii" - Content-Transfer-Encoding: 7bit Sender: test-bounces@example.com Errors-To: test-bounces@example.com X-Peer: 127.0.0.1:... @@ -114,10 +111,6 @@ address at the Mail-Archive. X-RcptTo: archive@mail-archive.dev <BLANKLINE> Here is an archived message. - _______________________________________________ - Test mailing list - test@example.com - http://lists.example.com/listinfo/test@example.com >>> smtpd.clear() diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg index b0e7d52d9..163b61024 100644 --- a/src/mailman/config/schema.cfg +++ b/src/mailman/config/schema.cfg @@ -265,7 +265,7 @@ chain: hold incoming: mailman.mta.postfix.LMTP # The class defining the interface to the outgoing mail transport agent. -outgoing: mailman.mta.smtp_direct.process +outgoing: mailman.mta.deliver.deliver # How to connect to the outgoing MTA. smtp_host: localhost diff --git a/src/mailman/core/errors.py b/src/mailman/core/errors.py index f320dc9d2..8037ea823 100644 --- a/src/mailman/core/errors.py +++ b/src/mailman/core/errors.py @@ -40,14 +40,12 @@ __all__ = [ 'NotAMemberError', 'PasswordError', 'RejectMessage', - 'SomeRecipientsFailed', 'SubscriptionError', ] -# Base class for all exceptions raised in Mailman (XXX except legacy string -# exceptions). +# Base class for all exceptions raised in Mailman. class MailmanException(Exception): pass @@ -94,10 +92,12 @@ class LostHeldMessage(MailmanError): def _(s): return s + # Exceptions for the Handler subsystem class HandlerError(MailmanError): """Base class for all handler errors.""" + class HoldMessage(HandlerError): """Base class for all message-being-held short circuits.""" @@ -113,15 +113,10 @@ class HoldMessage(HandlerError): def rejection_notice(self, mlist): return self.rejection + class DiscardMessage(HandlerError): """The message can be discarded with no further action""" -class SomeRecipientsFailed(HandlerError): - """Delivery to some or all recipients failed""" - def __init__(self, tempfailures, permfailures): - HandlerError.__init__(self) - self.tempfailures = tempfailures - self.permfailures = permfailures class RejectMessage(HandlerError): """The message will be bounced back to the sender""" diff --git a/src/mailman/interfaces/mta.py b/src/mailman/interfaces/mta.py index da0943e25..65696a10d 100644 --- a/src/mailman/interfaces/mta.py +++ b/src/mailman/interfaces/mta.py @@ -28,6 +28,17 @@ __all__ = [ from zope.interface import Interface +from mailman.core.errors import MailmanError + + + +class SomeRecipientsFailed(MailmanError): + """Delivery to some or all recipients failed""" + def __init__(self, temporary_failures, permanent_failures): + HandlerError.__init__(self) + self.temporary_failures = temporary_failures + self.permanent_failures = permanent_failures + class IMailTransportAgentAliases(Interface): diff --git a/src/mailman/mta/base.py b/src/mailman/mta/base.py index 48a3cd8f4..ff56094ac 100644 --- a/src/mailman/mta/base.py +++ b/src/mailman/mta/base.py @@ -52,8 +52,7 @@ class BaseDelivery: config.mta.smtp_host, int(config.mta.smtp_port), int(config.mta.max_sessions_per_connection)) - def _deliver_to_recipients(self, mlist, msg, msgdata, - sender, recipients): + def _deliver_to_recipients(self, mlist, msg, msgdata, recipients): """Low-level delivery to a set of recipients. :param mlist: The mailing list being delivered to. @@ -62,13 +61,23 @@ class BaseDelivery: :type msg: `Message` :param msgdata: Additional message metadata for this delivery. :type msgdata: dictionary - :param sender: The envelope sender. - :type sender: string :param recipients: The recipients of this message. :type recipients: sequence :return: delivery failures as defined by `smtplib.SMTP.sendmail` :rtype: dictionary """ + # Blow away any existing Sender and Errors-To headers and substitute + # our own. Our interpretation of RFC 5322 $3.6.2 is that Mailman is + # the "agent responsible for actual transmission of the message" + # because what we send to list members is different than what the + # original author sent. RFC 2076 says Errors-To is "non-standard, + # discouraged" but we include it for historical purposes. + sender = self._get_sender(mlist, msg, msgdata) + del msg['sender'] + del msg['errors-to'] + msg['Sender'] = sender + msg['Errors-To'] = sender + # Do the actual sending. message_id = msg['message-id'] try: refused = self._connection.sendmail( @@ -147,11 +156,8 @@ class IndividualDelivery(BaseDelivery): delivery address in the return envelope so there can be no ambiguity in bounce processing. """ - recipients = msgdata.get('recipients') - if not recipients: - # Could be None, could be an empty sequence. - return refused = {} + recipients = msgdata.get('recipients', set()) for recipient in recipients: # Make a copy of the original messages and operator on it, since # we're going to munge it repeatedly for each recipient. @@ -161,10 +167,9 @@ class IndividualDelivery(BaseDelivery): # That way the subclass's _get_sender() override can encode the # recipient address in the sender, e.g. for VERP. msgdata_copy['recipient'] = recipient - sender = self._get_sender(mlist, message_copy, msgdata_copy) for callback in self.callbacks: callback(mlist, message_copy, msgdata_copy) status = self._deliver_to_recipients( - mlist, message_copy, msgdata_copy, sender, [recipient]) + mlist, message_copy, msgdata_copy, [recipient]) refused.update(status) return refused diff --git a/src/mailman/mta/bulk.py b/src/mailman/mta/bulk.py index 3247331d3..046f5cc2d 100644 --- a/src/mailman/mta/bulk.py +++ b/src/mailman/mta/bulk.py @@ -102,25 +102,10 @@ class BulkDelivery(BaseDelivery): def deliver(self, mlist, msg, msgdata): """See `IMailTransportAgentDelivery`.""" - recipients = msgdata.get('recipients') - if not recipients: - # Could be None, could be an empty sequence. - return - # Blow away any existing Sender and Errors-To headers and substitute - # our own. Our interpretation of RFC 5322 $3.6.2 is that Mailman is - # the "agent responsible for actual transmission of the message" - # because what we send to list members is different than what the - # original author sent. RFC 2076 says Errors-To is "non-standard, - # discouraged" but we include it for historical purposes. - sender = self._get_sender(mlist, msg, msgdata) - del msg['sender'] - del msg['errors-to'] - msg['Sender'] = sender - msg['Errors-To'] = sender refused = {} - for recipients in self.chunkify(msgdata['recipients']): + for recipients in self.chunkify(msgdata.get('recipients', set())): chunk_refused = self._deliver_to_recipients( - mlist, msg, msgdata, sender, recipients) + mlist, msg, msgdata, recipients) refused.update(chunk_refused) return refused diff --git a/src/mailman/mta/deliver.py b/src/mailman/mta/deliver.py new file mode 100644 index 000000000..696770894 --- /dev/null +++ b/src/mailman/mta/deliver.py @@ -0,0 +1,154 @@ +# Copyright (C) 2009 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Generic delivery.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'deliver', + ] + + +import time +import logging + +from mailman.config import config +from mailman.interfaces.mailinglist import Personalization +from mailman.interfaces.mta import SomeRecipientsFailed +from mailman.mta.decorating import DecoratingMixin +from mailman.mta.personalized import PersonalizedMixin +from mailman.mta.verp import VERPMixin +from mailman.mta.base import IndividualDelivery +from mailman.mta.bulk import BulkDelivery +from mailman.utilities.string import expand + + +COMMA = ',' +log = logging.getLogger('mailman.smtp') + + + +class Deliver(VERPMixin, DecoratingMixin, PersonalizedMixin, + IndividualDelivery): + """Deliver one message to one recipient. + + All current individualized features are avaialble to this + `IMailTransportAgentDelivery` instance: + + * VERP + * Full Personalization + * Header/Footer decoration + """ + + def __init__(self): + super(Deliver, self).__init__() + self.callbacks.extend([ + self.avoid_duplicates, + self.decorate, + self.personalize_to, + ]) + + + +def deliver(mlist, msg, msgdata): + """Deliver a message to the outgoing mail server.""" + # If there are no recipients, there's nothing to do. + recipients = msgdata.get('recipients') + if not recipients: + # Could be None, could be an empty sequence. + return + # Which delivery agent should we use? Several situations can cause us to + # use individual delivery. If not specified, use bulk delivery. See the + # to-outgoing handler for when the 'verp' key is set in the metadata. + if msgdata.get('verp', False): + agent = Deliver() + elif mlist.personalize != Personalization.none: + agent = Deliver() + else: + agent = BulkDelivery(int(config.mta.max_recipients)) + # Keep track of the original recipients and the original sender for + # logging purposes. + original_recipients = msgdata['recipients'] + original_sender = msgdata.get('original-sender', msg.sender) + # Let the agent attempt to deliver to the recipients. Record all failures + # for re-delivery later. + t0 = time.time() + refused = agent.deliver(mlist, msg, msgdata) + t1 = time.time() + # Log this posting. + substitutions = dict( + msgid = msg.get('message-id', 'n/a'), + listname = mlist.fqdn_listname, + sender = original_sender, + recip = len(original_recipients), + size = msg.original_size, + time = t1 - t0, + refused = len(refused), + smtpcode = 'n/a', + smtpmsg = 'n/a', + ) + template = config.logging.smtp.every + if template.lower() != 'no': + log.info('%s', expand(template, substitutions)) + if refused: + template = config.logging.smtp.refused + if template.lower() != 'no': + log.info('%s', expand(template, substitutions)) + else: + # Log the successful post, but if it was not destined to the mailing + # list (e.g. to the owner or admin), print the actual recipients + # instead of just the number. + if not msgdata.get('tolist', False): + recips = msg.get_all('to', []) + recips.extend(msg.get_all('cc', [])) + substitutions['recips'] = COMMA.join(recips) + template = config.logging.smtp.success + if template.lower() != 'no': + log.info('%s', expand(template, substitutions)) + # Process any failed deliveries. + temporary_failures = [] + permanent_failures = [] + for recipient, (code, smtp_message) in refused.items(): + # RFC 5321, $4.5.3.1.10 says: + # + # RFC 821 [1] incorrectly listed the error where an SMTP server + # exhausts its implementation limit on the number of RCPT commands + # ("too many recipients") as having reply code 552. The correct + # reply code for this condition is 452. Clients SHOULD treat a 552 + # code in this case as a temporary, rather than permanent, failure + # so the logic below works. + # + if code >= 500 and code != 552: + # A permanent failure + permanent_failures.append(recipient) + else: + # Deal with persistent transient failures by queuing them up for + # future delivery. TBD: this could generate lots of log entries! + temporary_failures.append(recipient) + template = config.logging.smtp.failure + if template.lower() != 'no': + substitutions.update( + recip = recip, + smtpcode = code, + smtpmsg = smtpmsg, + ) + log.info('%s', expand(template, substitutions)) + # Return the results + if temporary_failures or permanent_failures: + raise SomeRecipientsFailed(temporary_failures, permanent_failures) diff --git a/src/mailman/mta/docs/bulk.txt b/src/mailman/mta/docs/bulk.txt index 99f58e7a7..57431d2f8 100644 --- a/src/mailman/mta/docs/bulk.txt +++ b/src/mailman/mta/docs/bulk.txt @@ -154,10 +154,12 @@ there are no calculated recipients, nothing gets sent. >>> bulk = BulkDelivery() >>> bulk.deliver(mlist, msg, {}) + {} >>> len(list(smtpd.messages)) 0 >>> bulk.deliver(mlist, msg, dict(recipients=set())) + {} >>> len(list(smtpd.messages)) 0 diff --git a/src/mailman/mta/docs/decorating.txt b/src/mailman/mta/docs/decorating.txt index 490974913..935d9e349 100644 --- a/src/mailman/mta/docs/decorating.txt +++ b/src/mailman/mta/docs/decorating.txt @@ -99,8 +99,10 @@ The decorations happen when the message is delivered. MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit + Sender: test-bounces@example.com + Errors-To: test-bounces@example.com X-Peer: ... - X-MailFrom: test-bounces+aperson=example.com@example.com + X-MailFrom: test-bounces@example.com X-RcptTo: aperson@example.com <BLANKLINE> Delivery address: aperson@example.com @@ -118,8 +120,10 @@ The decorations happen when the message is delivered. MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit + Sender: test-bounces@example.com + Errors-To: test-bounces@example.com X-Peer: ... - X-MailFrom: test-bounces+bperson=example.com@example.com + X-MailFrom: test-bounces@example.com X-RcptTo: bperson@example.com <BLANKLINE> Delivery address: bperson@example.com @@ -137,8 +141,10 @@ The decorations happen when the message is delivered. MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit + Sender: test-bounces@example.com + Errors-To: test-bounces@example.com X-Peer: ... - X-MailFrom: test-bounces+cperson=example.com@example.com + X-MailFrom: test-bounces@example.com X-RcptTo: cperson@example.com <BLANKLINE> Delivery address: cperson@example.com @@ -171,8 +177,10 @@ into the message metadata. To: test@example.com Subject: test one Message-ID: <aardvark> + Sender: test-bounces@example.com + Errors-To: test-bounces@example.com X-Peer: ... - X-MailFrom: test-bounces+aperson=example.com@example.com + X-MailFrom: test-bounces@example.com X-RcptTo: aperson@example.com <BLANKLINE> This is a test. @@ -181,8 +189,10 @@ into the message metadata. To: test@example.com Subject: test one Message-ID: <aardvark> + Sender: test-bounces@example.com + Errors-To: test-bounces@example.com X-Peer: ... - X-MailFrom: test-bounces+bperson=example.com@example.com + X-MailFrom: test-bounces@example.com X-RcptTo: bperson@example.com <BLANKLINE> This is a test. @@ -191,8 +201,10 @@ into the message metadata. To: test@example.com Subject: test one Message-ID: <aardvark> + Sender: test-bounces@example.com + Errors-To: test-bounces@example.com X-Peer: ... - X-MailFrom: test-bounces+cperson=example.com@example.com + X-MailFrom: test-bounces@example.com X-RcptTo: cperson@example.com <BLANKLINE> This is a test. diff --git a/src/mailman/mta/docs/personalized.txt b/src/mailman/mta/docs/personalized.txt index 4485fd2fc..7e4331b97 100644 --- a/src/mailman/mta/docs/personalized.txt +++ b/src/mailman/mta/docs/personalized.txt @@ -57,8 +57,10 @@ By default, the To header is not personalized. To: test@example.com Subject: test one Message-ID: <aardvark> + Sender: test-bounces@example.com + Errors-To: test-bounces@example.com X-Peer: ... - X-MailFrom: test-bounces+aperson=example.com@example.com + X-MailFrom: test-bounces@example.com X-RcptTo: aperson@example.com <BLANKLINE> This is a test. @@ -67,8 +69,10 @@ By default, the To header is not personalized. To: test@example.com Subject: test one Message-ID: <aardvark> + Sender: test-bounces@example.com + Errors-To: test-bounces@example.com X-Peer: ... - X-MailFrom: test-bounces+bperson=example.com@example.com + X-MailFrom: test-bounces@example.com X-RcptTo: bperson@example.com <BLANKLINE> This is a test. @@ -77,8 +81,10 @@ By default, the To header is not personalized. To: test@example.com Subject: test one Message-ID: <aardvark> + Sender: test-bounces@example.com + Errors-To: test-bounces@example.com X-Peer: ... - X-MailFrom: test-bounces+cperson=example.com@example.com + X-MailFrom: test-bounces@example.com X-RcptTo: cperson@example.com <BLANKLINE> This is a test. @@ -108,8 +114,10 @@ the recipient's address and name. To: aperson@example.com Subject: test one Message-ID: <aardvark> + Sender: test-bounces@example.com + Errors-To: test-bounces@example.com X-Peer: ... - X-MailFrom: test-bounces+aperson=example.com@example.com + X-MailFrom: test-bounces@example.com X-RcptTo: aperson@example.com <BLANKLINE> This is a test. @@ -118,8 +126,10 @@ the recipient's address and name. To: bperson@example.com Subject: test one Message-ID: <aardvark> + Sender: test-bounces@example.com + Errors-To: test-bounces@example.com X-Peer: ... - X-MailFrom: test-bounces+bperson=example.com@example.com + X-MailFrom: test-bounces@example.com X-RcptTo: bperson@example.com <BLANKLINE> This is a test. @@ -128,8 +138,10 @@ the recipient's address and name. To: cperson@example.com Subject: test one Message-ID: <aardvark> + Sender: test-bounces@example.com + Errors-To: test-bounces@example.com X-Peer: ... - X-MailFrom: test-bounces+cperson=example.com@example.com + X-MailFrom: test-bounces@example.com X-RcptTo: cperson@example.com <BLANKLINE> This is a test. @@ -160,8 +172,10 @@ associated real name, then this name also shows up in the To header. To: aperson@example.com Subject: test one Message-ID: <aardvark> + Sender: test-bounces@example.com + Errors-To: test-bounces@example.com X-Peer: ... - X-MailFrom: test-bounces+aperson=example.com@example.com + X-MailFrom: test-bounces@example.com X-RcptTo: aperson@example.com <BLANKLINE> This is a test. @@ -170,8 +184,10 @@ associated real name, then this name also shows up in the To header. To: Bill Person <bperson@example.com> Subject: test one Message-ID: <aardvark> + Sender: test-bounces@example.com + Errors-To: test-bounces@example.com X-Peer: ... - X-MailFrom: test-bounces+bperson=example.com@example.com + X-MailFrom: test-bounces@example.com X-RcptTo: bperson@example.com <BLANKLINE> This is a test. @@ -180,8 +196,10 @@ associated real name, then this name also shows up in the To header. To: Cate Person <cperson@example.com> Subject: test one Message-ID: <aardvark> + Sender: test-bounces@example.com + Errors-To: test-bounces@example.com X-Peer: ... - X-MailFrom: test-bounces+cperson=example.com@example.com + X-MailFrom: test-bounces@example.com X-RcptTo: cperson@example.com <BLANKLINE> This is a test. diff --git a/src/mailman/mta/docs/verp.txt b/src/mailman/mta/docs/verp.txt index c7c14e714..e31419bdb 100644 --- a/src/mailman/mta/docs/verp.txt +++ b/src/mailman/mta/docs/verp.txt @@ -52,10 +52,12 @@ If there are no recipients, there's nothing to do. ... """) >>> verp.deliver(mlist, msg, {}) + {} >>> len(list(smtpd.messages)) 0 >>> verp.deliver(mlist, msg, dict(recipients=set())) + {} >>> len(list(smtpd.messages)) 0 @@ -74,7 +76,10 @@ intended recipient's delivery address. ... 'cperson@example.com', ... ]) - >>> verp.deliver(mlist, msg, dict(recipients=recipients)) +VERPing is only actually done if the metadata requests it. + + >>> msgdata = dict(recipients=recipients, verp=True) + >>> verp.deliver(mlist, msg, msgdata) {} >>> messages = list(smtpd.messages) >>> len(messages) @@ -88,6 +93,8 @@ intended recipient's delivery address. To: test@example.com Subject: test one Message-ID: <aardvark> + Sender: test-bounces+aperson=example.com@example.com + Errors-To: test-bounces+aperson=example.com@example.com X-Peer: ... X-MailFrom: test-bounces+aperson=example.com@example.com X-RcptTo: aperson@example.com @@ -98,6 +105,8 @@ intended recipient's delivery address. To: test@example.com Subject: test one Message-ID: <aardvark> + Sender: test-bounces+bperson=example.com@example.com + Errors-To: test-bounces+bperson=example.com@example.com X-Peer: ... X-MailFrom: test-bounces+bperson=example.com@example.com X-RcptTo: bperson@example.com @@ -108,6 +117,8 @@ intended recipient's delivery address. To: test@example.com Subject: test one Message-ID: <aardvark> + Sender: test-bounces+cperson=example.com@example.com + Errors-To: test-bounces+cperson=example.com@example.com X-Peer: ... X-MailFrom: test-bounces+cperson=example.com@example.com X-RcptTo: cperson@example.com diff --git a/src/mailman/mta/smtp_direct.py b/src/mailman/mta/smtp_direct.py deleted file mode 100644 index 419d4ce96..000000000 --- a/src/mailman/mta/smtp_direct.py +++ /dev/null @@ -1,374 +0,0 @@ -# Copyright (C) 1998-2009 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Local SMTP direct drop-off. - -This module delivers messages via SMTP to a locally specified daemon. This -should be compatible with any modern SMTP server. It is expected that the MTA -handles all final delivery. We have to play tricks so that the list object -isn't locked while delivery occurs synchronously. - -Note: This file only handles single threaded delivery. See SMTPThreaded.py -for a threaded implementation. -""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'SMTPDirect', - ] - - -import copy -import time -import socket -import logging -import smtplib - -from email.Charset import Charset -from email.Header import Header -from email.Utils import formataddr -from zope.interface import implements - -from mailman.config import config -from mailman.core import errors -from mailman.email.utils import split_email -from mailman.i18n import _ -from mailman.interfaces.handler import IHandler -from mailman.interfaces.mailinglist import Personalization -from mailman.utilities.string import expand - - -DOT = '.' -COMMA = ',' -log = logging.getLogger('mailman.smtp') - - - -def process(mlist, msg, msgdata): - recips = msgdata.get('recips') - if not recips: - # Nobody to deliver to! - return - # Calculate the non-VERP envelope sender. - envsender = msgdata.get('envsender') - if envsender is None: - if mlist: - envsender = mlist.bounces_address - else: - envsender = config.mailman.site_owner - # Time to split up the recipient list. If we're personalizing or VERPing - # then each chunk will have exactly one recipient. We'll then hand craft - # an envelope sender and stitch a message together in memory for each one - # separately. If we're not VERPing, then we'll chunkify based on - # SMTP_MAX_RCPTS. Note that most MTAs have a limit on the number of - # recipients they'll swallow in a single transaction. - deliveryfunc = None - if (not msgdata.has_key('personalize') or msgdata['personalize']) and ( - msgdata.get('verp') or mlist.personalize <> Personalization.none): - chunks = [[recip] for recip in recips] - msgdata['personalize'] = True - deliveryfunc = verpdeliver - elif int(config.mta.max_recipients) <= 0: - chunks = [recips] - else: - chunks = chunkify(recips, int(config.mta.max_recipients)) - # See if this is an unshunted message for which some were undelivered - if msgdata.has_key('undelivered'): - chunks = msgdata['undelivered'] - # If we're doing bulk delivery, then we can stitch up the message now. - if deliveryfunc is None: - # Be sure never to decorate the message more than once! - if not msgdata.get('decorated'): - handler = config.handlers['decorate'] - handler.process(mlist, msg, msgdata) - msgdata['decorated'] = True - deliveryfunc = bulkdeliver - refused = {} - t0 = time.time() - # Open the initial connection - origrecips = msgdata['recips'] - # MAS: get the message sender now for logging. If we're using 'sender' - # and not 'from', bulkdeliver changes it for bounce processing. If we're - # VERPing, it doesn't matter because bulkdeliver is working on a copy, but - # otherwise msg gets changed. If the list is anonymous, the original - # sender is long gone, but Cleanse.py has logged it. - origsender = msgdata.get('original_sender', msg.sender) - # `undelivered' is a copy of chunks that we pop from to do deliveries. - # This seems like a good tradeoff between robustness and resource - # utilization. If delivery really fails (i.e. qfiles/shunt type - # failures), then we'll pick up where we left off with `undelivered'. - # This means at worst, the last chunk for which delivery was attempted - # could get duplicates but not every one, and no recips should miss the - # message. - conn = Connection() - try: - msgdata['undelivered'] = chunks - while chunks: - chunk = chunks.pop() - msgdata['recips'] = chunk - try: - deliveryfunc(mlist, msg, msgdata, envsender, refused, conn) - except Exception: - # If /anything/ goes wrong, push the last chunk back on the - # undelivered list and re-raise the exception. We don't know - # how many of the last chunk might receive the message, so at - # worst, everyone in this chunk will get a duplicate. Sigh. - chunks.append(chunk) - raise - del msgdata['undelivered'] - finally: - conn.quit() - msgdata['recips'] = origrecips - # Log the successful post - t1 = time.time() - substitutions = dict( - msgid = msg.get('message-id', 'n/a'), - listname = mlist.fqdn_listname, - sender = origsender, - recip = len(recips), - size = msg.original_size, - time = t1 - t0, - refused = len(refused), - smtpcode = 'n/a', - smtpmsg = 'n/a', - ) - # Log this message. - template = config.logging.smtp.every - if template != 'no': - log.info('%s', expand(template, substitutions)) - if refused: - template = config.logging.smtp.refused - if template != 'no': - log.info('%s', expand(template, substitutions)) - else: - # Log the successful post, but if it was not destined to the mailing - # list (e.g. to the owner or admin), print the actual recipients - # instead of just the number. - if not msgdata.get('tolist'): - recips = msg.get_all('to', []) - recips.extend(msg.get_all('cc', [])) - substitutions['recips'] = COMMA.join(recips) - template = config.logging.smtp.success - if template != 'no': - log.info('%s', expand(template, substitutions)) - # Process any failed deliveries. - tempfailures = [] - permfailures = [] - for recip, (code, smtpmsg) in refused.items(): - # DRUMS is an internet draft, but it says: - # - # [RFC-821] incorrectly listed the error where an SMTP server - # exhausts its implementation limit on the number of RCPT commands - # ("too many recipients") as having reply code 552. The correct - # reply code for this condition is 452. Clients SHOULD treat a 552 - # code in this case as a temporary, rather than permanent failure - # so the logic below works. - # - if code >= 500 and code <> 552: - # A permanent failure - permfailures.append(recip) - else: - # Deal with persistent transient failures by queuing them up for - # future delivery. TBD: this could generate lots of log entries! - tempfailures.append(recip) - template = config.logging.smtp.failure - if template != 'no': - substitutions.update( - recip = recip, - smtpcode = code, - smtpmsg = smtpmsg, - ) - log.info('%s', expand(template, substitutions)) - # Return the results - if tempfailures or permfailures: - raise errors.SomeRecipientsFailed(tempfailures, permfailures) - - - -def chunkify(recips, chunksize): - # First do a simple sort on top level domain. It probably doesn't buy us - # much to try to sort on MX record -- that's the MTA's job. We're just - # trying to avoid getting a max recips error. Split the chunks along - # these lines (as suggested originally by Chuq Von Rospach and slightly - # elaborated by BAW). - chunkmap = {'com': 1, - 'net': 2, - 'org': 2, - 'edu': 3, - 'us' : 3, - 'ca' : 3, - } - buckets = {} - for r in recips: - tld = None - i = r.rfind('.') - if i >= 0: - tld = r[i+1:] - bin = chunkmap.get(tld, 0) - bucket = buckets.get(bin, []) - bucket.append(r) - buckets[bin] = bucket - # Now start filling the chunks - chunks = [] - currentchunk = [] - chunklen = 0 - for bin in buckets.values(): - for r in bin: - currentchunk.append(r) - chunklen = chunklen + 1 - if chunklen >= chunksize: - chunks.append(currentchunk) - currentchunk = [] - chunklen = 0 - if currentchunk: - chunks.append(currentchunk) - currentchunk = [] - chunklen = 0 - return chunks - - - -def verpdeliver(mlist, msg, msgdata, envsender, failures, conn): - handler = config.handlers['decorate'] - for recip in msgdata['recips']: - # We now need to stitch together the message with its header and - # footer. If we're VERPIng, we have to calculate the envelope sender - # for each recipient. Note that the list of recipients must be of - # length 1. - # - # BAW: ezmlm includes the message number in the envelope, used when - # sending a notification to the user telling her how many messages - # they missed due to bouncing. Neat idea. - msgdata['recips'] = [recip] - # Make a copy of the message and decorate + delivery that - msgcopy = copy.deepcopy(msg) - handler.process(mlist, msgcopy, msgdata) - # Calculate the envelope sender, which we may be VERPing - if msgdata.get('verp'): - bmailbox, bdomain = split_email(envsender) - rmailbox, rdomain = split_email(recip) - if rdomain is None: - # The recipient address is not fully-qualified. We can't - # deliver it to this person, nor can we craft a valid verp - # header. I don't think there's much we can do except ignore - # this recipient. - log.info('Skipping VERP delivery to unqual recip: %s', recip) - continue - envsender = expand(config.mta.verp_format, dict( - bounces=bmailbox, mailbox=rmailbox, - host=DOT.join(rdomain))) + '@' + DOT.join(bdomain) - if mlist.personalize == Personalization.full: - # When fully personalizing, we want the To address to point to the - # recipient, not to the mailing list - del msgcopy['to'] - name = None - if mlist.isMember(recip): - name = mlist.getMemberName(recip) - if name: - # Convert the name to an email-safe representation. If the - # name is a byte string, convert it first to Unicode, given - # the character set of the member's language, replacing bad - # characters for which we can do nothing about. Once we have - # the name as Unicode, we can create a Header instance for it - # so that it's properly encoded for email transport. - charset = mlist.getMemberLanguage(recip).charset - if charset == 'us-ascii': - # Since Header already tries both us-ascii and utf-8, - # let's add something a bit more useful. - charset = 'iso-8859-1' - charset = Charset(charset) - codec = charset.input_codec or 'ascii' - if not isinstance(name, unicode): - name = unicode(name, codec, 'replace') - name = Header(name, charset).encode() - msgcopy['To'] = formataddr((name, recip)) - else: - msgcopy['To'] = recip - # We can flag the mail as a duplicate for each member, if they've - # already received this message, as calculated by Message-ID. See - # AvoidDuplicates.py for details. - del msgcopy['x-mailman-copy'] - if msgdata.get('add-dup-header', {}).has_key(recip): - msgcopy['X-Mailman-Copy'] = 'yes' - # For the final delivery stage, we can just bulk deliver to a party of - # one. ;) - bulkdeliver(mlist, msgcopy, msgdata, envsender, failures, conn) - - - -def bulkdeliver(mlist, msg, msgdata, envsender, failures, conn): - # Do some final cleanup of the message header. Start by blowing away - # any the Sender: and Errors-To: headers so remote MTAs won't be - # tempted to delivery bounces there instead of our envelope sender - # - # BAW An interpretation of RFCs 2822 and 2076 could argue for not touching - # the Sender header at all. Brad Knowles points out that MTAs tend to - # wipe existing Return-Path headers, and old MTAs may still honor - # Errors-To while new ones will at worst ignore the header. - del msg['sender'] - del msg['errors-to'] - msg['Sender'] = envsender - msg['Errors-To'] = envsender - # Get the plain, flattened text of the message, sans unixfrom - msgtext = msg.as_string() - refused = {} - recips = msgdata['recips'] - msgid = msg['message-id'] - try: - # Send the message - refused = conn.sendmail(envsender, recips, msgtext) - except smtplib.SMTPRecipientsRefused as error: - log.error('%s recipients refused: %s', msgid, error) - refused = error.recipients - except smtplib.SMTPResponseException as error: - log.error('%s SMTP session failure: %s, %s', - msgid, error.smtp_code, error.smtp_error) - # If this was a permanent failure, don't add the recipients to the - # refused, because we don't want them to be added to failures. - # Otherwise, if the MTA rejects the message because of the message - # content (e.g. it's spam, virii, or has syntactic problems), then - # this will end up registering a bounce score for every recipient. - # Definitely /not/ what we want. - if error.smtp_code < 500 or error.smtp_code == 552: - # It's a temporary failure - for r in recips: - refused[r] = (error.smtp_code, error.smtp_error) - except (socket.error, IOError, smtplib.SMTPException) as error: - # MTA not responding, or other socket problems, or any other kind of - # SMTPException. In that case, nothing got delivered, so treat this - # as a temporary failure. - log.error('%s low level smtp error: %s', msgid, error) - error = str(error) - for r in recips: - refused[r] = (-1, error) - failures.update(refused) - - - -class SMTPDirect: - """SMTP delivery.""" - - implements(IHandler) - - name = 'smtp-direct' - description = _('SMTP delivery.') - - def process(self, mlist, msg, msgdata): - """See `IHandler`.""" - process(mlist, msg, msgdata) diff --git a/src/mailman/mta/verp.py b/src/mailman/mta/verp.py index a46c42cff..c53276bdc 100644 --- a/src/mailman/mta/verp.py +++ b/src/mailman/mta/verp.py @@ -56,34 +56,27 @@ class VERPMixin: :param msgdata: Additional message metadata for this delivery. :type msgdata: dictionary """ - recipient = msgdata['recipient'] sender = super(VERPMixin, self)._get_sender(mlist, msg, msgdata) - sender_mailbox, sender_domain = split_email(sender) - # Encode the recipient's address for VERP. - recipient_mailbox, recipient_domain = split_email(recipient) - if recipient_domain is None: - # The recipient address is not fully-qualified. We can't - # deliver it to this person, nor can we craft a valid verp - # header. I don't think there's much we can do except ignore - # this recipient. - log.info('Skipping VERP delivery to unqual recip: %s', recip) + if msgdata.get('verp', False): + recipient = msgdata['recipient'] + sender_mailbox, sender_domain = split_email(sender) + # Encode the recipient's address for VERP. + recipient_mailbox, recipient_domain = split_email(recipient) + if recipient_domain is None: + # The recipient address is not fully-qualified. We can't + # deliver it to this person, nor can we craft a valid verp + # header. I don't think there's much we can do except ignore + # this recipient. + log.info('Skipping VERP delivery to unqual recip: %s', recip) + return sender + return '{0}@{1}'.format( + expand(config.mta.verp_format, dict( + bounces=sender_mailbox, + local=recipient_mailbox, + domain=DOT.join(recipient_domain))), + DOT.join(sender_domain)) + else: return sender - return '{0}@{1}'.format( - expand(config.mta.verp_format, dict( - bounces=sender_mailbox, - local=recipient_mailbox, - domain=DOT.join(recipient_domain))), - DOT.join(sender_domain)) - - - -class VERPDelivery(VERPMixin, IndividualDelivery): - """Deliver a unique message to the MSA for each recipient.""" - - def __init__(self): - """See `IndividualDelivery`.""" - super(VERPDelivery, self).__init__() - self.callbacks.append(self.avoid_duplicates) def avoid_duplicates(self, mlist, msg, msgdata): """Flag the message for duplicate avoidance. @@ -96,3 +89,13 @@ class VERPDelivery(VERPMixin, IndividualDelivery): del msg['x-mailman-copy'] if recipient in msgdata.get('add-dup-header', {}): msg['X-Mailman-Copy'] = 'yes' + + + +class VERPDelivery(VERPMixin, IndividualDelivery): + """Deliver a unique message to the MSA for each recipient.""" + + def __init__(self): + """See `IndividualDelivery`.""" + super(VERPDelivery, self).__init__() + self.callbacks.append(self.avoid_duplicates) diff --git a/src/mailman/pipeline/to_outgoing.py b/src/mailman/pipeline/to_outgoing.py index ff27593c4..ef6908ab8 100644 --- a/src/mailman/pipeline/to_outgoing.py +++ b/src/mailman/pipeline/to_outgoing.py @@ -72,7 +72,7 @@ class ToOutgoing: msgdata['verp'] = True else: # VERP every `interval' number of times - msgdata['verp'] = not (int(mlist.post_id) % interval) + msgdata['verp'] = (int(mlist.post_id) % interval == 0) # And now drop the message in qfiles/out config.switchboards['out'].enqueue( msg, msgdata, listname=mlist.fqdn_listname) diff --git a/src/mailman/queue/outgoing.py b/src/mailman/queue/outgoing.py index 7776b1b54..b1adbe411 100644 --- a/src/mailman/queue/outgoing.py +++ b/src/mailman/queue/outgoing.py @@ -26,7 +26,7 @@ from datetime import datetime from lazr.config import as_timedelta from mailman.config import config -from mailman.core import errors +from mailman.interfaces.mta import SomeRecipientsFailed from mailman.queue import Runner from mailman.queue.bounce import BounceMixin from mailman.utilities.modules import find_name @@ -81,9 +81,9 @@ class OutgoingRunner(Runner, BounceMixin): config.mta.host, port) self._logged = True return True - except errors.SomeRecipientsFailed, e: + except SomeRecipientsFailed as error: # Handle local rejects of probe messages differently. - if msgdata.get('probe_token') and e.permfailures: + if msgdata.get('probe_token') and error.permanent_failures: self._probe_bounce(mlist, msgdata['probe_token']) else: # Delivery failed at SMTP time for some or all of the @@ -95,15 +95,15 @@ class OutgoingRunner(Runner, BounceMixin): # this is what's sent to the user in the probe message. Maybe # we should craft a bounce-like message containing information # about the permanent SMTP failure? - if e.permfailures: - self._queue_bounces(mlist.fqdn_listname, e.permfailures, - msg) + if error.permanent_failures: + self._queue_bounces( + mlist.fqdn_listname, error.permanent_failures, msg) # Move temporary failures to the qfiles/retry queue which will # occasionally move them back here for another shot at # delivery. - if e.tempfailures: + if error.temporary_failures: now = datetime.now() - recips = e.tempfailures + recips = error.temporary_failures last_recip_count = msgdata.get('last_recip_count', 0) deliver_until = msgdata.get('deliver_until', now) if len(recips) == last_recip_count: |