diff options
| -rw-r--r-- | mailman/Utils.py | 2 | ||||
| -rw-r--r-- | mailman/app/membership.py | 4 | ||||
| -rw-r--r-- | mailman/bin/arch.py | 2 | ||||
| -rw-r--r-- | mailman/bin/senddigests.py | 2 | ||||
| -rw-r--r-- | mailman/database/mailinglist.py | 14 | ||||
| -rw-r--r-- | mailman/interfaces/mailinglist.py | 7 | ||||
| -rw-r--r-- | mailman/pipeline/decorate.py | 26 | ||||
| -rw-r--r-- | mailman/pipeline/docs/file-recips.txt | 2 | ||||
| -rw-r--r-- | mailman/pipeline/file_recipients.py | 2 | ||||
| -rw-r--r-- | mailman/pipeline/smtp_direct.py | 4 | ||||
| -rw-r--r-- | mailman/pipeline/to_digest.py | 2 | ||||
| -rw-r--r-- | mailman/queue/archive.py | 3 | ||||
| -rw-r--r-- | mailman/queue/docs/outgoing.txt | 96 | ||||
| -rw-r--r-- | mailman/queue/outgoing.py | 1 | ||||
| -rw-r--r-- | mailman/tests/helpers.py | 17 | ||||
| -rw-r--r-- | mailman/tests/smtplistener.py | 7 |
16 files changed, 156 insertions, 35 deletions
diff --git a/mailman/Utils.py b/mailman/Utils.py index 714ef3da7..621a307d4 100644 --- a/mailman/Utils.py +++ b/mailman/Utils.py @@ -481,7 +481,7 @@ def findtext(templatefile, dict=None, raw=False, lang=None, mlist=None): # Calculate the locations to scan searchdirs = [] if mlist is not None: - searchdirs.append(mlist.full_path) + searchdirs.append(mlist.data_path) searchdirs.append(os.path.join(TEMPLATE_DIR, mlist.host_name)) searchdirs.append(os.path.join(TEMPLATE_DIR, 'site')) searchdirs.append(TEMPLATE_DIR) diff --git a/mailman/app/membership.py b/mailman/app/membership.py index f3487028f..8a6e571a1 100644 --- a/mailman/app/membership.py +++ b/mailman/app/membership.py @@ -45,8 +45,8 @@ def add_member(mlist, address, realname, password, delivery_mode, language, ack=None, admin_notif=None, text=''): """Add a member right now. - The member's subscription must be approved by what ever policy the - list enforces. + The member's subscription must be approved by whatever policy the list + enforces. ack is a flag that specifies whether the user should get an acknowledgement of their being subscribed. Default is to use the diff --git a/mailman/bin/arch.py b/mailman/bin/arch.py index 883ab29e1..5254179e4 100644 --- a/mailman/bin/arch.py +++ b/mailman/bin/arch.py @@ -115,7 +115,7 @@ def main(): # really don't know how long it will take. # # XXX processUnixMailbox() should refresh the lock. - lock_path = os.path.join(mlist.full_path, '.archiver.lck') + lock_path = os.path.join(mlist.data_path, '.archiver.lck') with Lock(lock_path, lifetime=int(hours(3))): # Maybe wipe the old archives if opts.wipe: diff --git a/mailman/bin/senddigests.py b/mailman/bin/senddigests.py index de424ab9b..8c5ca4f64 100644 --- a/mailman/bin/senddigests.py +++ b/mailman/bin/senddigests.py @@ -72,7 +72,7 @@ def main(): print >> sys.stderr, \ 'List: %s: problem processing %s:\n%s' % \ (listname, - os.path.join(mlist.full_path(), 'digest.mbox'), + os.path.join(mlist.data_path, 'digest.mbox'), errmsg) finally: mlist.Unlock() diff --git a/mailman/database/mailinglist.py b/mailman/database/mailinglist.py index 9895f9c2d..0b80ca0b5 100644 --- a/mailman/database/mailinglist.py +++ b/mailman/database/mailinglist.py @@ -174,11 +174,10 @@ class MailingList(Model): # 2-tuple of the date of the last autoresponse and the number of # autoresponses sent on that date. self.hold_and_cmd_autoresponses = {} - self.full_path = os.path.join(config.LIST_DATA_DIR, fqdn_listname) self.personalization = Personalization.none self.real_name = string.capwords( SPACE.join(listname.split(UNDERSCORE))) - makedirs(self.full_path) + makedirs(self.data_path) # XXX FIXME def _restore(self): @@ -192,20 +191,25 @@ class MailingList(Model): @property def fqdn_listname(self): - """See IMailingListIdentity.""" + """See `IMailingList`.""" return fqdn_listname(self.list_name, self.host_name) @property def web_host(self): - """See IMailingListWeb.""" + """See `IMailingList`.""" return config.domains[self.host_name] def script_url(self, target, context=None): - """See IMailingListWeb.""" + """See `IMailingList`.""" # XXX Handle the case for when context is not None; those would be # relative URLs. return self.web_page_url + target + '/' + self.fqdn_listname + @property + def data_path(self): + """See `IMailingList`.""" + return os.path.join(config.LIST_DATA_DIR, self.fqdn_listname) + # IMailingListAddresses @property diff --git a/mailman/interfaces/mailinglist.py b/mailman/interfaces/mailinglist.py index de63e84e5..a5f6a9e9a 100644 --- a/mailman/interfaces/mailinglist.py +++ b/mailman/interfaces/mailinglist.py @@ -260,3 +260,10 @@ class IMailingList(Interface): Every mailing list has a processing pipeline that messages flow through once they've been accepted. """) + + data_path = Attribute( + """The file system path to list-specific data. + + An example of list-specific data is the temporary digest mbox file + that gets created to accumlate messages for the digest. + """) diff --git a/mailman/pipeline/decorate.py b/mailman/pipeline/decorate.py index 100c1cb6e..387685f12 100644 --- a/mailman/pipeline/decorate.py +++ b/mailman/pipeline/decorate.py @@ -48,21 +48,21 @@ def process(mlist, msg, msgdata): if msgdata.get('personalize'): # Calculate the extra personalization dictionary. Note that the # length of the recips list better be exactly 1. - recips = msgdata.get('recips') - assert isinstance(recips, list) and len(recips) == 1, ( + recips = msgdata.get('recips', []) + assert len(recips) == 1, ( 'The number of intended recipients must be exactly 1') - member = recips[0].lower() - d['user_address'] = member - try: - d['user_delivered_to'] = mlist.getMemberCPAddress(member) + recipient = recips[0].lower() + user = config.db.user_manager.get_user(recipient) + member = mlist.members.get_member(recipient) + d['user_address'] = recipient + if user is not None and member is not None: + d['user_delivered_to'] = member.address.original_address # BAW: Hmm, should we allow this? - d['user_password'] = mlist.getMemberPassword(member) - d['user_language'] = mlist.getMemberLanguage(member) - username = mlist.getMemberName(member) or None - d['user_name'] = username or d['user_delivered_to'] - d['user_optionsurl'] = mlist.GetOptionsURL(member) - except Errors.NotAMemberError: - pass + d['user_password'] = user.password + d['user_language'] = member.preferred_language + d['user_name'] = (user.real_name if user.real_name + else member.address.original_address) + d['user_optionsurl'] = member.options_url # These strings are descriptive for the log file and shouldn't be i18n'd d.update(msgdata.get('decoration-data', {})) header = decorate(mlist, mlist.msg_header, d) diff --git a/mailman/pipeline/docs/file-recips.txt b/mailman/pipeline/docs/file-recips.txt index 03328f97e..e93bba9aa 100644 --- a/mailman/pipeline/docs/file-recips.txt +++ b/mailman/pipeline/docs/file-recips.txt @@ -40,7 +40,7 @@ members.txt. If the file doesn't exist, the list of recipients will be empty. >>> import os - >>> file_path = os.path.join(mlist.full_path, 'members.txt') + >>> file_path = os.path.join(mlist.data_path, 'members.txt') >>> open(file_path) Traceback (most recent call last): ... diff --git a/mailman/pipeline/file_recipients.py b/mailman/pipeline/file_recipients.py index 8d97500fe..44ced925d 100644 --- a/mailman/pipeline/file_recipients.py +++ b/mailman/pipeline/file_recipients.py @@ -46,7 +46,7 @@ class FileRecipients: """See `IHandler`.""" if 'recips' in msgdata: return - filename = os.path.join(mlist.full_path, 'members.txt') + filename = os.path.join(mlist.data_path, 'members.txt') try: with open(filename) as fp: addrs = set(line.strip() for line in fp) diff --git a/mailman/pipeline/smtp_direct.py b/mailman/pipeline/smtp_direct.py index d79510cb0..74eaa5aad 100644 --- a/mailman/pipeline/smtp_direct.py +++ b/mailman/pipeline/smtp_direct.py @@ -110,7 +110,7 @@ def process(mlist, msg, msgdata): envsender = msgdata.get('envsender') if envsender is None: if mlist: - envsender = mlist.GetBouncesEmail() + envsender = mlist.bounces_address else: envsender = Utils.get_site_noreply() # Time to split up the recipient list. If we're personalizing or VERPing @@ -183,7 +183,7 @@ def process(mlist, msg, msgdata): 'size' : len(msg.as_string()), '#recips' : len(recips), '#refused': len(refused), - 'listname': mlist.internal_name(), + 'listname': mlist.fqdn_listname, 'sender' : origsender, }) # We have to use the copy() method because extended call syntax requires a diff --git a/mailman/pipeline/to_digest.py b/mailman/pipeline/to_digest.py index 191e3a0f1..05b1dc3d5 100644 --- a/mailman/pipeline/to_digest.py +++ b/mailman/pipeline/to_digest.py @@ -74,7 +74,7 @@ def process(mlist, msg, msgdata): # Short circuit non-digestable lists. if not mlist.digestable or msgdata.get('isdigest'): return - mboxfile = os.path.join(mlist.full_path, 'digest.mbox') + mboxfile = os.path.join(mlist.data_path, 'digest.mbox') mboxfp = open(mboxfile, 'a+') mbox = Mailbox(mboxfp) mbox.AppendMessage(msg) diff --git a/mailman/queue/archive.py b/mailman/queue/archive.py index f17e6b751..47627a04e 100644 --- a/mailman/queue/archive.py +++ b/mailman/queue/archive.py @@ -19,6 +19,7 @@ from __future__ import with_statement +import os import time from email.Utils import parsedate_tz, mktime_tz, formatdate @@ -68,5 +69,5 @@ class ArchiveRunner(Runner): # Always put an indication of when we received the message. msg['X-List-Received-Date'] = receivedtime # While a list archiving lock is acquired, archive the message. - with Lock(os.path.join(mlist.full_path, 'archive.lck')): + with Lock(os.path.join(mlist.data_path, 'archive.lck')): mlist.ArchiveMail(msg) diff --git a/mailman/queue/docs/outgoing.txt b/mailman/queue/docs/outgoing.txt new file mode 100644 index 000000000..fc9161320 --- /dev/null +++ b/mailman/queue/docs/outgoing.txt @@ -0,0 +1,96 @@ +Outgoing queue runner +===================== + +The outgoing queue runner is the process that delivers messages to the +directly upstream SMTP server. It is this external SMTP server that performs +final delivery to the intended recipients. + +Messages that appear in the outgoing queue are processed individually through +a 'delivery module', essentially a pluggable interface for determining how the +recipient set will be batched, whether messages will be personalized and +VERP'd, etc. The outgoing runner doesn't itself support retrying but it can +move messages to the 'retry queue' for handling delivery failures. + + >>> from mailman.app.lifecycle import create_list + >>> mlist = create_list(u'test@example.com') + + >>> from mailman.app.membership import add_member + >>> from mailman.interfaces import DeliveryMode + >>> add_member(mlist, u'aperson@example.com', u'Anne Person', + ... u'password', DeliveryMode.regular, u'en', + ... ack=False, admin_notif=False) + >>> add_member(mlist, u'bperson@example.com', u'Bart Person', + ... u'password', DeliveryMode.regular, u'en', + ... ack=False, admin_notif=False) + >>> add_member(mlist, u'cperson@example.com', u'Cris Person', + ... u'password', DeliveryMode.regular, u'en', + ... ack=False, admin_notif=False) + + >>> from mailman.tests.helpers import SMTPServer + >>> smtpd = SMTPServer() + >>> smtpd.start() + >>> from mailman.configuration import config + >>> old_host = config.SMTPHOST + >>> old_port = config.SMTPPORT + >>> config.SMTPHOST = smtpd.host + >>> config.SMTPPORT = smtpd.port + +By setting the mailing list to personalize messages, each recipient will get a +unique copy of the message, with certain headers tailored for that recipient. + + >>> from mailman.interfaces import Personalization + >>> mlist.personalize = Personalization.individual + >>> config.db.commit() + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: test@example.com + ... Subject: My first post + ... Message-ID: <first> + ... + ... First post! + ... """) + +Normally, messages would show up in the outgoing queue after the message has +been processed by the rule set and pipeline. But we can simulate that here by +injecting a message directly into the outgoing queue. + + >>> msgdata = {} + >>> handler = config.handlers['calculate-recipients'] + >>> handler.process(mlist, msg, msgdata) + + >>> from mailman.queue import Switchboard + >>> outgoing_queue = Switchboard(config.OUTQUEUE_DIR) + >>> ignore = outgoing_queue.enqueue( + ... msg, msgdata, + ... verp=True, listname=mlist.fqdn_listname, tolist=True, + ... _plaintext=True) + +Running the outgoing queue runner processes the message, delivering it to the +upstream SMTP, which happens to be our test server. + + >>> from mailman.queue.outgoing import OutgoingRunner + >>> from mailman.tests.helpers import make_testable_runner + >>> outgoing = make_testable_runner(OutgoingRunner) + >>> outgoing.run() + +Three messages have been delivered to our SMTP server, one for each recipient. + + >>> from operator import itemgetter + >>> messages = sorted(smtpd.messages, key=itemgetter('sender')) + >>> len(messages) + 3 + + >>> for message in messages: + ... print message['sender'] + test-bounces+aperson=example.com@example.com + test-bounces+bperson=example.com@example.com + test-bounces+cperson=example.com@example.com + + +Clean up +-------- + + >>> smtpd.stop() + >>> config.SMTPHOST = old_host + >>> config.SMTPPORT = old_port diff --git a/mailman/queue/outgoing.py b/mailman/queue/outgoing.py index 31599e5ee..838d7d137 100644 --- a/mailman/queue/outgoing.py +++ b/mailman/queue/outgoing.py @@ -60,7 +60,6 @@ class OutgoingRunner(Runner, BounceMixin): if time.time() < deliver_after: return True # Make sure we have the most up-to-date state - mlist.Load() try: pid = os.getpid() self._func(mlist, msg, msgdata) diff --git a/mailman/tests/helpers.py b/mailman/tests/helpers.py index 71b3ff60e..d1eda9e34 100644 --- a/mailman/tests/helpers.py +++ b/mailman/tests/helpers.py @@ -34,6 +34,7 @@ import time import errno import signal import socket +import logging import mailbox import smtplib import tempfile @@ -48,6 +49,9 @@ from mailman.queue import Switchboard from mailman.tests.smtplistener import Server +log = logging.getLogger('mailman.debug') + + def make_testable_runner(runner_class): """Create a queue runner that runs until its queue is empty. @@ -97,7 +101,7 @@ def digest_mbox(mlist): :param mlist: The mailing list. :return: The mailing list's pending digest as a mailbox. """ - path = os.path.join(mlist.full_path, 'digest.mbox') + path = os.path.join(mlist.data_path, 'digest.mbox') return mailbox.mbox(path) @@ -168,6 +172,7 @@ class SMTPServer: def start(self): """Start the smtp server in a thread.""" + log.info('test SMTP server starting') self._thread.start() def stop(self): @@ -178,18 +183,22 @@ class SMTPServer: self.clear() # Wait for the thread to exit. self._thread.join() + log.info('test SMTP server stopped') @property def messages(self): """Return all the messages received by the smtp server.""" - for message in self._messages: - # See if there's anything waiting in the queue. + # Look at the thread queue and append any messages from there to our + # internal list of messages. + while True: try: message = self._queue.get_nowait() except Empty: - pass + break else: self._messages.append(message) + # Now return all the messages we know about. + for message in self._messages: yield message def clear(self): diff --git a/mailman/tests/smtplistener.py b/mailman/tests/smtplistener.py index 3a5b870b7..1dc11e3e0 100644 --- a/mailman/tests/smtplistener.py +++ b/mailman/tests/smtplistener.py @@ -18,11 +18,14 @@ """A test SMTP listener.""" import smtpd +import logging import asyncore from email import message_from_string + COMMASPACE = ', ' +log = logging.getLogger('mailman.debug') @@ -60,14 +63,16 @@ class Server(smtpd.SMTPServer): def handle_accept(self): """Handle connections by creating our own Channel object.""" conn, addr = self.accept() + log.info('accepted: %s', addr) Channel(self, conn, addr) def process_message(self, peer, mailfrom, rcpttos, data): """Process a message by adding it to the mailbox.""" message = message_from_string(data) - message['X-Peer'] = peer + message['X-Peer'] = '%s:%s' % peer message['X-MailFrom'] = mailfrom message['X-RcptTo'] = COMMASPACE.join(rcpttos) + log.info('processed message: %s', message.get('message-id', 'n/a')) self._queue.put(message) def start(self): |