diff options
Diffstat (limited to 'Mailman/queue')
| -rw-r--r-- | Mailman/queue/docs/OVERVIEW.txt | 78 | ||||
| -rw-r--r-- | Mailman/queue/docs/incoming.txt | 67 | ||||
| -rw-r--r-- | Mailman/queue/docs/news.txt | 158 | ||||
| -rw-r--r-- | Mailman/queue/docs/outgoing.txt | 155 | ||||
| -rw-r--r-- | Mailman/queue/docs/runner.txt | 70 | ||||
| -rw-r--r-- | Mailman/queue/docs/switchboard.txt | 149 | ||||
| -rw-r--r-- | Mailman/queue/incoming.py | 163 | ||||
| -rw-r--r-- | Mailman/queue/tests/__init__.py | 0 |
8 files changed, 695 insertions, 145 deletions
diff --git a/Mailman/queue/docs/OVERVIEW.txt b/Mailman/queue/docs/OVERVIEW.txt new file mode 100644 index 000000000..643fa8a5c --- /dev/null +++ b/Mailman/queue/docs/OVERVIEW.txt @@ -0,0 +1,78 @@ +Alias overview +============== + +A typical Mailman list exposes nine aliases which point to seven different +wrapped scripts. E.g. for a list named `mylist', you'd have: + + mylist-bounces -> bounces + mylist-confirm -> confirm + mylist-join -> join (-subscribe is an alias) + mylist-leave -> leave (-unsubscribe is an alias) + mylist-owner -> owner + mylist -> post + mylist-request -> request + +-request, -join, and -leave are a robot addresses; their sole purpose is to +process emailed commands, although the latter two are hardcoded to +subscription and unsubscription requests. -bounces is the automated bounce +processor, and all messages to list members have their return address set to +-bounces. If the bounce processor fails to extract a bouncing member address, +it can optionally forward the message on to the list owners. + +-owner is for reaching a human operator with minimal list interaction (i.e. no +bounce processing). -confirm is another robot address which processes replies +to VERP-like confirmation notices. + +So delivery flow of messages look like this: + + joerandom ---> mylist ---> list members + | | + | |[bounces] + | mylist-bounces <---+ <-------------------------------+ + | | | + | +--->[internal bounce processing] | + | ^ | | + | | | [bounce found] | + | [bounces *] +--->[register and discard] | + | | | | | + | | | |[*] | + | [list owners] |[no bounce found] | | + | ^ | | | + | | | | | + +-------> mylist-owner <--------+ | | + | | | + | data/owner-bounces.mbox <--[site list] <---+ | + | | + +-------> mylist-join--+ | + | | | + +------> mylist-leave--+ | + | | | + | v | + +-------> mylist-request | + | | | + | +---> [command processor] | + | | | + +-----> mylist-confirm ----> +---> joerandom | + | | + |[bounces] | + +----------------------+ + +A person can send an email to the list address (for posting), the -owner +address (to reach the human operator), or the -confirm, -join, -leave, and +-request mailbots. Message to the list address are then forwarded on to the +list membership, with bounces directed to the -bounces address. + +[*] Messages sent to the -owner address are forwarded on to the list +owner/moderators. All -owner destined messages have their bounces directed to +the site list -bounces address, regardless of whether a human sent the message +or the message was crafted internally. The intention here is that the site +owners want to be notified when one of their list owners' addresses starts +bouncing (yes, the will be automated in a future release). + +Any messages to site owners has their bounces directed to a special +"loop-killer" address, which just dumps the message into +data/owners-bounces.mbox. + +Finally, message to any of the mailbots causes the requested action to be +performed. Results notifications are sent to the author of the message, which +all bounces pointing back to the -bounces address. diff --git a/Mailman/queue/docs/incoming.txt b/Mailman/queue/docs/incoming.txt new file mode 100644 index 000000000..12ff3d3d1 --- /dev/null +++ b/Mailman/queue/docs/incoming.txt @@ -0,0 +1,67 @@ +The incoming queue runner +========================= + +This runner's sole purpose in life is to decide the disposition of the +message. It can either be accepted for delivery, rejected (i.e. bounced), +held for moderator approval, or discarded. + +The runner operates by processing chains on a message/metadata pair in the +context of a mailing list. Each mailing list may have a 'start chain' where +processing begins, with a global default. This chain is processed with the +message eventually ending up in one of the four disposition states described +above. + + >>> from Mailman.app.lifecycle import create_list + >>> mlist = create_list(u'_xtest@example.com') + >>> mlist.start_chain + u'built-in' + +We have a message that is going to be sent to the mailing list. This message +is so perfectly fine for posting that it will be accepted and forward to the +prep queue. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... Subject: My first post + ... Message-ID: <first> + ... + ... First post! + ... """) + +Normally, the upstream mail server would drop the message in the incoming +queue, but this is an effective simulation. + + >>> from Mailman.inject import inject + >>> inject(u'_xtest@example.com', msg) + +The incoming queue runner runs until it is empty. + + >>> from Mailman.queue.incoming import IncomingRunner + >>> from Mailman.tests.helpers import make_testable_runner + >>> incoming = make_testable_runner(IncomingRunner) + >>> incoming.run() + +And now the message is in the prep queue. + + >>> from Mailman.configuration import config + >>> from Mailman.queue import Switchboard + >>> prep_queue = Switchboard(config.PREPQUEUE_DIR) + >>> len(prep_queue.files) + 1 + >>> from Mailman.tests.helpers import get_queue_messages + >>> item = get_queue_messages(prep_queue)[0] + >>> print item.msg.as_string() + From: aperson@example.com + To: _xtest@example.com + Subject: My first post + Message-ID: <first> + X-Mailman-Rule-Misses: approved; emergency; loop; administrivia; + implicit-dest; + max-recipients; max-size; news-moderation; no-subject; + suspicious-header + <BLANKLINE> + First post! + <BLANKLINE> + >>> sorted(item.msgdata.items()) + [...('envsender', u'noreply@example.com')...('tolist', True)...] diff --git a/Mailman/queue/docs/news.txt b/Mailman/queue/docs/news.txt new file mode 100644 index 000000000..bc6619f50 --- /dev/null +++ b/Mailman/queue/docs/news.txt @@ -0,0 +1,158 @@ +The news runner +=============== + +The news runner is the queue runner that gateways mailing list messages to an +NNTP newsgroup. One of the most important things this runner does is prepare +the message for Usenet (yes, I know that NNTP is not Usenet, but this runner +was originally written to gate to Usenet, which has its own rules). + + >>> from Mailman.configuration import config + >>> from Mailman.queue.news import prepare_message + >>> mlist = config.db.list_manager.create(u'_xtest@example.com') + >>> mlist.linked_newsgroup = u'comp.lang.python' + +Some NNTP servers such as INN reject messages containing a set of prohibited +headers, so one of the things that the news runner does is remove these +prohibited headers. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... NNTP-Posting-Host: news.example.com + ... NNTP-Posting-Date: today + ... X-Trace: blah blah + ... X-Complaints-To: abuse@dom.ain + ... Xref: blah blah + ... Xref: blah blah + ... Date-Received: yesterday + ... Posted: tomorrow + ... Posting-Version: 99.99 + ... Relay-Version: 88.88 + ... Received: blah blah + ... + ... A message + ... """) + >>> msgdata = {} + >>> prepare_message(mlist, msg, msgdata) + >>> msgdata['prepped'] + True + >>> print msg.as_string() + From: aperson@example.com + To: _xtest@example.com + Newsgroups: comp.lang.python + Message-ID: ... + Lines: 1 + <BLANKLINE> + A message + <BLANKLINE> + +Some NNTP servers will reject messages where certain headers are duplicated, +so the news runner must collapse or move these duplicate headers to an +X-Original-* header that the news server doesn't care about. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... To: two@example.com + ... Cc: three@example.com + ... Cc: four@example.com + ... Cc: five@example.com + ... Content-Transfer-Encoding: yes + ... Content-Transfer-Encoding: no + ... Content-Transfer-Encoding: maybe + ... + ... A message + ... """) + >>> msgdata = {} + >>> prepare_message(mlist, msg, msgdata) + >>> msgdata['prepped'] + True + >>> print msg.as_string() + From: aperson@example.com + Newsgroups: comp.lang.python + Message-ID: ... + Lines: 1 + To: _xtest@example.com + X-Original-To: two@example.com + CC: three@example.com + X-Original-CC: four@example.com + X-Original-CC: five@example.com + Content-Transfer-Encoding: yes + X-Original-Content-Transfer-Encoding: no + X-Original-Content-Transfer-Encoding: maybe + <BLANKLINE> + A message + <BLANKLINE> + +But if no headers are duplicated, then the news runner doesn't need to modify +the message. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... Cc: someother@example.com + ... Content-Transfer-Encoding: yes + ... + ... A message + ... """) + >>> msgdata = {} + >>> prepare_message(mlist, msg, msgdata) + >>> msgdata['prepped'] + True + >>> print msg.as_string() + From: aperson@example.com + To: _xtest@example.com + Cc: someother@example.com + Content-Transfer-Encoding: yes + Newsgroups: comp.lang.python + Message-ID: ... + Lines: 1 + <BLANKLINE> + A message + <BLANKLINE> + + +Newsgroup moderation +-------------------- + +When the newsgroup is moderated, an Approved: header with the list's posting +address is added for the benefit of the Usenet system. + + >>> from Mailman.interfaces import NewsModeration + >>> mlist.news_moderation = NewsModeration.open_moderated + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... Approved: this gets deleted + ... + ... """) + >>> prepare_message(mlist, msg, {}) + >>> msg['approved'] + u'_xtest@example.com' + + >>> mlist.news_moderation = NewsModeration.moderated + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... Approved: this gets deleted + ... + ... """) + >>> prepare_message(mlist, msg, {}) + >>> msg['approved'] + u'_xtest@example.com' + +But if the newsgroup is not moderated, the Approved: header is not chnaged. + + >>> mlist.news_moderation = NewsModeration.none + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... Approved: this doesn't get deleted + ... + ... """) + >>> prepare_message(mlist, msg, {}) + >>> msg['approved'] + u"this doesn't get deleted" + + +XXX More of the NewsRunner should be tested. diff --git a/Mailman/queue/docs/outgoing.txt b/Mailman/queue/docs/outgoing.txt new file mode 100644 index 000000000..ba2c6430b --- /dev/null +++ b/Mailman/queue/docs/outgoing.txt @@ -0,0 +1,155 @@ +The outgoing handler +==================== + +Mailman's outgoing queue is used as the wrapper around SMTP delivery to the +upstream mail server. The ToOutgoing handler does little more than drop the +message into the outgoing queue, after calculating whether the message should +be VERP'd or not. VERP means Variable Envelope Return Path; we're using that +term somewhat incorrectly, but within the spirit of the standard, which +basically describes how to encode the recipient's address in the originator +headers for unambigous bounce processing. + + >>> from Mailman.Handlers.ToOutgoing import process + >>> from Mailman.queue import Switchboard + >>> from Mailman.configuration import config + >>> mlist = config.db.list_manager.create(u'_xtest@example.com') + >>> switchboard = Switchboard(config.OUTQUEUE_DIR) + + >>> def queue_size(): + ... size = len(switchboard.files) + ... for filebase in switchboard.files: + ... msg, msgdata = switchboard.dequeue(filebase) + ... switchboard.finish(filebase) + ... return size + +Craft a message destined for the outgoing queue. Include some random metadata +as if this message had passed through some other handlers. + + >>> msg = message_from_string("""\ + ... Subject: Here is a message + ... + ... Something of great import. + ... """) + +When certain conditions are met, the message will be VERP'd. For example, if +the message metadata already has a VERP key, this message will be VERP'd. + + >>> msgdata = dict(foo=1, bar=2, verp=True) + >>> process(mlist, msg, msgdata) + >>> print msg.as_string() + Subject: Here is a message + <BLANKLINE> + Something of great import. + >>> msgdata['verp'] + True + +While the queued message will not be changed, the queued metadata will have an +additional key set: the mailing list name. + + >>> filebase = switchboard.files[0] + >>> qmsg, qmsgdata = switchboard.dequeue(filebase) + >>> switchboard.finish(filebase) + >>> print qmsg.as_string() + Subject: Here is a message + <BLANKLINE> + Something of great import. + >>> sorted(qmsgdata.items()) + [('_parsemsg', False), + ('bar', 2), ('foo', 1), + ('listname', u'_xtest@example.com'), + ('received_time', ...), + ('verp', True), ('version', 3)] + >>> queue_size() + 0 + +If the list is set to personalize deliveries, and the global configuration +option to VERP personalized deliveries is set, then the message will be +VERP'd. + + # Save the original value for clean up. + >>> verp_personalized_delivieries = config.VERP_PERSONALIZED_DELIVERIES + >>> config.VERP_PERSONALIZED_DELIVERIES = True + >>> from Mailman.interfaces import Personalization + >>> mlist.personalize = Personalization.individual + >>> msgdata = dict(foo=1, bar=2) + >>> process(mlist, msg, msgdata) + >>> msgdata['verp'] + True + >>> queue_size() + 1 + +However, if the global configuration variable prohibits VERP'ing, even +personalized lists will not VERP. + + >>> config.VERP_PERSONALIZED_DELIVERIES = False + >>> msgdata = dict(foo=1, bar=2) + >>> process(mlist, msg, msgdata) + >>> print msgdata.get('verp') + None + >>> queue_size() + 1 + +If the list is not personalized, then the message may still be VERP'd based on +the global configuration variable VERP_DELIVERY_INTERVAL. This variable tells +Mailman how often to VERP even non-personalized mailing lists. It can be set +to zero, which means non-personalized messages will never be VERP'd. + + # Save the original value for clean up. + >>> verp_delivery_interval = config.VERP_DELIVERY_INTERVAL + >>> config.VERP_DELIVERY_INTERVAL = 0 + >>> mlist.personalize = Personalization.none + >>> msgdata = dict(foo=1, bar=2) + >>> process(mlist, msg, msgdata) + >>> print msgdata.get('verp') + None + >>> queue_size() + 1 + +If the interval is set to 1, then every message will be VERP'd. + + >>> config.VERP_DELIVERY_INTERVAL = 1 + >>> for i in range(10): + ... msgdata = dict(foo=1, bar=2) + ... process(mlist, msg, msgdata) + ... print i, msgdata['verp'] + 0 True + 1 True + 2 True + 3 True + 4 True + 5 True + 6 True + 7 True + 8 True + 9 True + >>> queue_size() + 10 + +If the interval is set to some other number, then one out of that many posts +will be VERP'd. + + >>> config.VERP_DELIVERY_INTERVAL = 3 + >>> for i in range(10): + ... mlist.post_id = i + ... msgdata = dict(foo=1, bar=2) + ... process(mlist, msg, msgdata) + ... print i, msgdata.get('verp', False) + 0 True + 1 False + 2 False + 3 True + 4 False + 5 False + 6 True + 7 False + 8 False + 9 True + >>> queue_size() + 10 + + +Clean up +======== + + >>> config.VERP_PERSONALIZED_DELIVERIES = verp_personalized_delivieries + >>> config.VERP_DELIVERY_INTERVAL = verp_delivery_interval diff --git a/Mailman/queue/docs/runner.txt b/Mailman/queue/docs/runner.txt new file mode 100644 index 000000000..5e5a88d8c --- /dev/null +++ b/Mailman/queue/docs/runner.txt @@ -0,0 +1,70 @@ +Queue runners +============= + +The queue runners (qrunner) are the processes that move messages around the +Mailman system. Each qrunner is responsible for a slice of the hash space in +a queue directory. It processes all the files in its slice, sleeps a little +while, then wakes up and runs through its queue files again. + + +Basic architecture +------------------ + +The basic architecture of qrunner is implemented in the base class that all +runners inherit from. This base class implements a .run() method that runs +continuously in a loop until the .stop() method is called. + + >>> import os + >>> from Mailman.queue import Runner, Switchboard + >>> from Mailman.configuration import config + >>> mlist = config.db.list_manager.create(u'_xtest@example.com') + >>> mlist.preferred_language = u'en' + +Here is a very simple derived qrunner class. The class attribute QDIR tells +the qrunner which queue directory it is responsible for. Derived classes +should also implement various methods to provide the special functionality. +This is about as simple as a qrunner can be. + + >>> queue_directory = os.path.join(config.QUEUE_DIR, 'test') + >>> class TestableRunner(Runner): + ... QDIR = queue_directory + ... + ... def _dispose(self, mlist, msg, msgdata): + ... self.msg = msg + ... self.msgdata = msgdata + ... return False + ... + ... def _doperiodic(self): + ... self.stop() + ... + ... def _snooze(self, filecnt): + ... return + + >>> runner = TestableRunner() + >>> switchboard = Switchboard(queue_directory) + +This qrunner doesn't do much except run once, storing the message and metadata +on instance variables. + + >>> msg = message_from_string(u"""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... + ... A test message. + ... """) + >>> filebase = switchboard.enqueue(msg, listname=mlist.fqdn_listname, + ... foo='yes', bar='no') + >>> runner.run() + >>> print runner.msg.as_string() + From: aperson@example.com + To: _xtest@example.com + <BLANKLINE> + A test message. + <BLANKLINE> + >>> sorted(runner.msgdata.items()) + [('_parsemsg', False), + ('bar', 'no'), ('foo', 'yes'), + ('lang', u'en'), ('listname', u'_xtest@example.com'), + ('received_time', ...), ('version', 3)] + +XXX More of the Runner API should be tested. diff --git a/Mailman/queue/docs/switchboard.txt b/Mailman/queue/docs/switchboard.txt new file mode 100644 index 000000000..299aba499 --- /dev/null +++ b/Mailman/queue/docs/switchboard.txt @@ -0,0 +1,149 @@ +The switchboard +=============== + +The switchboard is subsystem that moves messages between queues. Each +instance of a switchboard is responsible for one queue directory. + + >>> msg = message_from_string(u"""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... + ... A test message. + ... """) + +Create a switchboard by giving its queue directory. + + >>> import os + >>> from Mailman.configuration import config + >>> queue_directory = os.path.join(config.QUEUE_DIR, 'test') + >>> from Mailman.queue import Switchboard + >>> switchboard = Switchboard(queue_directory) + >>> switchboard.queue_directory == queue_directory + True + +Here's a helper function for ensuring things work correctly. + + >>> def check_qfiles(): + ... files = {} + ... for qfile in os.listdir(queue_directory): + ... root, ext = os.path.splitext(qfile) + ... files[ext] = files.get(ext, 0) + 1 + ... return sorted(files.items()) + + +Enqueing and dequeing +--------------------- + +The message can be enqueued with metadata specified in the passed in +dictionary. + + >>> filebase = switchboard.enqueue(msg) + >>> check_qfiles() + [('.pck', 1)] + +To read the contents of a queue file, dequeue it. + + >>> msg, msgdata = switchboard.dequeue(filebase) + >>> print msg.as_string() + From: aperson@example.com + To: _xtest@example.com + <BLANKLINE> + A test message. + <BLANKLINE> + >>> sorted(msgdata.items()) + [('_parsemsg', False), ('received_time', ...), ('version', 3)] + >>> check_qfiles() + [('.bak', 1)] + +To complete the dequeing process, removing all traces of the message file, +finish it (without preservation). + + >>> switchboard.finish(filebase) + >>> check_qfiles() + [] + +When enqueing a file, you can provide additional metadata keys by using +keyword arguments. + + >>> filebase = switchboard.enqueue(msg, {'foo': 1}, bar=2) + >>> msg, msgdata = switchboard.dequeue(filebase) + >>> switchboard.finish(filebase) + >>> sorted(msgdata.items()) + [('_parsemsg', False), + ('bar', 2), ('foo', 1), + ('received_time', ...), ('version', 3)] + +Keyword arguments override keys from the metadata dictionary. + + >>> filebase = switchboard.enqueue(msg, {'foo': 1}, foo=2) + >>> msg, msgdata = switchboard.dequeue(filebase) + >>> switchboard.finish(filebase) + >>> sorted(msgdata.items()) + [('_parsemsg', False), + ('foo', 2), + ('received_time', ...), ('version', 3)] + + +Iterating over files +-------------------- + +There are two ways to iterate over all the files in a switchboard's queue. +Normally, queue files end in .pck (for 'pickle') and the easiest way to +iterate over just these files is to use the .files attribute. + + >>> filebase_1 = switchboard.enqueue(msg, foo=1) + >>> filebase_2 = switchboard.enqueue(msg, foo=2) + >>> filebase_3 = switchboard.enqueue(msg, foo=3) + >>> filebases = sorted((filebase_1, filebase_2, filebase_3)) + >>> sorted(switchboard.files) == filebases + True + >>> check_qfiles() + [('.pck', 3)] + +You can also use the .get_files() method if you want to iterate over all the +file bases for some other extension. + + >>> for filebase in switchboard.get_files(): + ... msg, msgdata = switchboard.dequeue(filebase) + >>> bakfiles = sorted(switchboard.get_files('.bak')) + >>> bakfiles == filebases + True + >>> check_qfiles() + [('.bak', 3)] + >>> for filebase in switchboard.get_files('.bak'): + ... switchboard.finish(filebase) + >>> check_qfiles() + [] + + +Recovering files +---------------- + +Calling .dequeue() without calling .finish() leaves .bak backup files in +place. These can be recovered when the switchboard is instantiated. + + >>> filebase_1 = switchboard.enqueue(msg, foo=1) + >>> filebase_2 = switchboard.enqueue(msg, foo=2) + >>> filebase_3 = switchboard.enqueue(msg, foo=3) + >>> for filebase in switchboard.files: + ... msg, msgdata = switchboard.dequeue(filebase) + ... # Don't call .finish() + >>> check_qfiles() + [('.bak', 3)] + >>> switchboard_2 = Switchboard(queue_directory, recover=True) + >>> check_qfiles() + [('.pck', 3)] + +Clean up + + >>> for filebase in switchboard.files: + ... msg, msgdata = switchboard.dequeue(filebase) + ... switchboard.finish(filebase) + >>> check_qfiles() + [] + + +Queue slices +------------ + +XXX Add tests for queue slices. diff --git a/Mailman/queue/incoming.py b/Mailman/queue/incoming.py index 6118a7ca0..649ce2213 100644 --- a/Mailman/queue/incoming.py +++ b/Mailman/queue/incoming.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2007 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2008 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -12,102 +12,26 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. -"""Incoming queue runner.""" - -# A typical Mailman list exposes nine aliases which point to seven different -# wrapped scripts. E.g. for a list named `mylist', you'd have: -# -# mylist-bounces -> bounces (-admin is a deprecated alias) -# mylist-confirm -> confirm -# mylist-join -> join (-subscribe is an alias) -# mylist-leave -> leave (-unsubscribe is an alias) -# mylist-owner -> owner -# mylist -> post -# mylist-request -> request -# -# -request, -join, and -leave are a robot addresses; their sole purpose is to -# process emailed commands in a Majordomo-like fashion (although the latter -# two are hardcoded to subscription and unsubscription requests). -bounces is -# the automated bounce processor, and all messages to list members have their -# return address set to -bounces. If the bounce processor fails to extract a -# bouncing member address, it can optionally forward the message on to the -# list owners. -# -# -owner is for reaching a human operator with minimal list interaction -# (i.e. no bounce processing). -confirm is another robot address which -# processes replies to VERP-like confirmation notices. -# -# So delivery flow of messages look like this: -# -# joerandom ---> mylist ---> list members -# | | -# | |[bounces] -# | mylist-bounces <---+ <-------------------------------+ -# | | | -# | +--->[internal bounce processing] | -# | ^ | | -# | | | [bounce found] | -# | [bounces *] +--->[register and discard] | -# | | | | | -# | | | |[*] | -# | [list owners] |[no bounce found] | | -# | ^ | | | -# | | | | | -# +-------> mylist-owner <--------+ | | -# | | | -# | data/owner-bounces.mbox <--[site list] <---+ | -# | | -# +-------> mylist-join--+ | -# | | | -# +------> mylist-leave--+ | -# | | | -# | v | -# +-------> mylist-request | -# | | | -# | +---> [command processor] | -# | | | -# +-----> mylist-confirm ----> +---> joerandom | -# | | -# |[bounces] | -# +----------------------+ -# -# A person can send an email to the list address (for posting), the -owner -# address (to reach the human operator), or the -confirm, -join, -leave, and -# -request mailbots. Message to the list address are then forwarded on to the -# list membership, with bounces directed to the -bounces address. -# -# [*] Messages sent to the -owner address are forwarded on to the list -# owner/moderators. All -owner destined messages have their bounces directed -# to the site list -bounces address, regardless of whether a human sent the -# message or the message was crafted internally. The intention here is that -# the site owners want to be notified when one of their list owners' addresses -# starts bouncing (yes, the will be automated in a future release). -# -# Any messages to site owners has their bounces directed to a special -# "loop-killer" address, which just dumps the message into -# data/owners-bounces.mbox. -# -# Finally, message to any of the mailbots causes the requested action to be -# performed. Results notifications are sent to the author of the message, -# which all bounces pointing back to the -bounces address. +"""Incoming queue runner. +This runner's sole purpose in life is to decide the disposition of the +message. It can either be accepted for delivery, rejected (i.e. bounced), +held for moderator approval, or discarded. - -import os -import sys -import logging +When accepted, the message is forwarded on to the `prep queue` where it is +prepared for delivery. Rejections, discards, and holds are processed +immediately. +""" -from cStringIO import StringIO -from Mailman import Errors + +from Mailman.app.chains import process from Mailman.configuration import config from Mailman.queue import Runner -log = logging.getLogger('mailman.error') -vlog = logging.getLogger('mailman.vette') - class IncomingRunner(Runner): @@ -115,59 +39,8 @@ class IncomingRunner(Runner): def _dispose(self, mlist, msg, msgdata): if msgdata.get('envsender') is None: - msg['envsender'] = mlist.no_reply_address - # Process the message through a handler pipeline. The handler - # pipeline can actually come from one of three places: the message - # metadata, the mlist, or the global pipeline. - # - # If a message was requeued due to an uncaught exception, its metadata - # will contain the retry pipeline. Use this above all else. - # Otherwise, if the mlist has a `pipeline' attribute, it should be - # used. Final fallback is the global pipeline. - pipeline = self._get_pipeline(mlist, msg, msgdata) - msgdata['pipeline'] = pipeline - more = self._dopipeline(mlist, msg, msgdata, pipeline) - if not more: - del msgdata['pipeline'] - config.db.commit() - return more - - # Overridable - def _get_pipeline(self, mlist, msg, msgdata): - # We must return a copy of the list, otherwise, the first message that - # flows through the pipeline will empty it out! - return msgdata.get('pipeline', - getattr(mlist, 'pipeline', - config.GLOBAL_PIPELINE))[:] - - def _dopipeline(self, mlist, msg, msgdata, pipeline): - while pipeline: - handler = pipeline.pop(0) - modname = 'Mailman.Handlers.' + handler - __import__(modname) - try: - pid = os.getpid() - sys.modules[modname].process(mlist, msg, msgdata) - # Failsafe -- a child may have leaked through. - if pid <> os.getpid(): - log.error('child process leaked thru: %s', modname) - os._exit(1) - except Errors.DiscardMessage: - # Throw the message away; we need do nothing else with it. - vlog.info('Message discarded, msgid: %s', - msg.get('message-id', 'n/a')) - return 0 - except Errors.HoldMessage: - # Let the approval process take it from here. The message no - # longer needs to be queued. - return 0 - except Errors.RejectMessage, e: - mlist.bounce_message(msg, e) - return 0 - except: - # Push this pipeline module back on the stack, then re-raise - # the exception. - pipeline.insert(0, handler) - raise - # We've successfully completed handling of this message - return 0 + msgdata['envsender'] = mlist.no_reply_address + # Process the message through the mailing list's start chain. + process(mlist, msg, msgdata, mlist.start_chain) + # Do not keep this message queued. + return False diff --git a/Mailman/queue/tests/__init__.py b/Mailman/queue/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/Mailman/queue/tests/__init__.py +++ /dev/null |