diff options
Diffstat (limited to '')
| -rw-r--r-- | Mailman/chains/__init__.py (renamed from Mailman/queue/tests/__init__.py) | 0 | ||||
| -rw-r--r-- | Mailman/queue/__init__.py | 6 | ||||
| -rw-r--r-- | Mailman/queue/command.py | 4 | ||||
| -rw-r--r-- | Mailman/queue/docs/OVERVIEW.txt | 78 | ||||
| -rw-r--r-- | Mailman/queue/docs/incoming.txt | 198 | ||||
| -rw-r--r-- | Mailman/queue/docs/news.txt (renamed from Mailman/docs/news-runner.txt) | 0 | ||||
| -rw-r--r-- | Mailman/queue/docs/outgoing.txt (renamed from Mailman/docs/outgoing.txt) | 0 | ||||
| -rw-r--r-- | Mailman/queue/docs/runner.txt (renamed from Mailman/docs/runner.txt) | 0 | ||||
| -rw-r--r-- | Mailman/queue/docs/switchboard.txt (renamed from Mailman/docs/switchboard.txt) | 0 | ||||
| -rw-r--r-- | Mailman/queue/incoming.py | 163 |
10 files changed, 302 insertions, 147 deletions
diff --git a/Mailman/queue/tests/__init__.py b/Mailman/chains/__init__.py index e69de29bb..e69de29bb 100644 --- a/Mailman/queue/tests/__init__.py +++ b/Mailman/chains/__init__.py diff --git a/Mailman/queue/__init__.py b/Mailman/queue/__init__.py index c415834ba..6a1873d60 100644 --- a/Mailman/queue/__init__.py +++ b/Mailman/queue/__init__.py @@ -140,7 +140,13 @@ class Switchboard: msg = cPickle.load(fp) data = cPickle.load(fp) if data.get('_parsemsg'): + # Calculate the original size of the text now so that we won't + # have to generate the message later when we do size restriction + # checking. + original_size = len(msg) msg = email.message_from_string(msg, Message.Message) + msg.original_size = original_size + data['original_size'] = original_size return msg, data def finish(self, filebase, preserve=False): diff --git a/Mailman/queue/command.py b/Mailman/queue/command.py index dfbfd1255..8628411de 100644 --- a/Mailman/queue/command.py +++ b/Mailman/queue/command.py @@ -88,8 +88,8 @@ class Results: assert isinstance(body, basestring) lines = body.splitlines() # Use no more lines than specified - self.commands.extend(lines[:config.DEFAULT_MAIL_COMMANDS_MAX_LINES]) - self.ignored.extend(lines[config.DEFAULT_MAIL_COMMANDS_MAX_LINES:]) + self.commands.extend(lines[:config.EMAIL_COMMANDS_MAX_LINES]) + self.ignored.extend(lines[config.EMAIL_COMMANDS_MAX_LINES:]) def process(self): # Now, process each line until we find an error. The first diff --git a/Mailman/queue/docs/OVERVIEW.txt b/Mailman/queue/docs/OVERVIEW.txt new file mode 100644 index 000000000..643fa8a5c --- /dev/null +++ b/Mailman/queue/docs/OVERVIEW.txt @@ -0,0 +1,78 @@ +Alias overview +============== + +A typical Mailman list exposes nine aliases which point to seven different +wrapped scripts. E.g. for a list named `mylist', you'd have: + + mylist-bounces -> bounces + mylist-confirm -> confirm + mylist-join -> join (-subscribe is an alias) + mylist-leave -> leave (-unsubscribe is an alias) + mylist-owner -> owner + mylist -> post + mylist-request -> request + +-request, -join, and -leave are a robot addresses; their sole purpose is to +process emailed commands, although the latter two are hardcoded to +subscription and unsubscription requests. -bounces is the automated bounce +processor, and all messages to list members have their return address set to +-bounces. If the bounce processor fails to extract a bouncing member address, +it can optionally forward the message on to the list owners. + +-owner is for reaching a human operator with minimal list interaction (i.e. no +bounce processing). -confirm is another robot address which processes replies +to VERP-like confirmation notices. + +So delivery flow of messages look like this: + + joerandom ---> mylist ---> list members + | | + | |[bounces] + | mylist-bounces <---+ <-------------------------------+ + | | | + | +--->[internal bounce processing] | + | ^ | | + | | | [bounce found] | + | [bounces *] +--->[register and discard] | + | | | | | + | | | |[*] | + | [list owners] |[no bounce found] | | + | ^ | | | + | | | | | + +-------> mylist-owner <--------+ | | + | | | + | data/owner-bounces.mbox <--[site list] <---+ | + | | + +-------> mylist-join--+ | + | | | + +------> mylist-leave--+ | + | | | + | v | + +-------> mylist-request | + | | | + | +---> [command processor] | + | | | + +-----> mylist-confirm ----> +---> joerandom | + | | + |[bounces] | + +----------------------+ + +A person can send an email to the list address (for posting), the -owner +address (to reach the human operator), or the -confirm, -join, -leave, and +-request mailbots. Message to the list address are then forwarded on to the +list membership, with bounces directed to the -bounces address. + +[*] Messages sent to the -owner address are forwarded on to the list +owner/moderators. All -owner destined messages have their bounces directed to +the site list -bounces address, regardless of whether a human sent the message +or the message was crafted internally. The intention here is that the site +owners want to be notified when one of their list owners' addresses starts +bouncing (yes, the will be automated in a future release). + +Any messages to site owners has their bounces directed to a special +"loop-killer" address, which just dumps the message into +data/owners-bounces.mbox. + +Finally, message to any of the mailbots causes the requested action to be +performed. Results notifications are sent to the author of the message, which +all bounces pointing back to the -bounces address. diff --git a/Mailman/queue/docs/incoming.txt b/Mailman/queue/docs/incoming.txt new file mode 100644 index 000000000..04c0cfa04 --- /dev/null +++ b/Mailman/queue/docs/incoming.txt @@ -0,0 +1,198 @@ +The incoming queue runner +========================= + +This runner's sole purpose in life is to decide the disposition of the +message. It can either be accepted for delivery, rejected (i.e. bounced), +held for moderator approval, or discarded. + +The runner operates by processing chains on a message/metadata pair in the +context of a mailing list. Each mailing list may have a 'start chain' where +processing begins, with a global default. This chain is processed with the +message eventually ending up in one of the four disposition states described +above. + + >>> from Mailman.app.lifecycle import create_list + >>> mlist = create_list(u'_xtest@example.com') + >>> mlist.start_chain + u'built-in' + + +Accepted messages +----------------- + +We have a message that is going to be sent to the mailing list. This message +is so perfectly fine for posting that it will be accepted and forward to the +prep queue. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... Subject: My first post + ... Message-ID: <first> + ... + ... First post! + ... """) + +Normally, the upstream mail server would drop the message in the incoming +queue, but this is an effective simulation. + + >>> from Mailman.inject import inject + >>> inject(u'_xtest@example.com', msg) + +The incoming queue runner runs until it is empty. + + >>> from Mailman.queue.incoming import IncomingRunner + >>> from Mailman.tests.helpers import make_testable_runner + >>> incoming = make_testable_runner(IncomingRunner) + >>> incoming.run() + +And now the message is in the prep queue. + + >>> from Mailman.configuration import config + >>> from Mailman.queue import Switchboard + >>> prep_queue = Switchboard(config.PREPQUEUE_DIR) + >>> len(prep_queue.files) + 1 + >>> incoming_queue = Switchboard(config.INQUEUE_DIR) + >>> len(incoming_queue.files) + 0 + >>> from Mailman.tests.helpers import get_queue_messages + >>> item = get_queue_messages(prep_queue)[0] + >>> print item.msg.as_string() + From: aperson@example.com + To: _xtest@example.com + Subject: My first post + Message-ID: <first> + X-Mailman-Rule-Misses: approved; emergency; loop; administrivia; + implicit-dest; + max-recipients; max-size; news-moderation; no-subject; + suspicious-header + <BLANKLINE> + First post! + <BLANKLINE> + >>> sorted(item.msgdata.items()) + [...('envsender', u'noreply@example.com')...('tolist', True)...] + + +Held messages +------------- + +The list moderator sets the emergency flag on the mailing list. The built-in +chain will now hold all posted messages, so nothing will show up in the prep +queue. + + # XXX This checks the vette log file because there is no other evidence + # that this chain has done anything. + >>> import os + >>> fp = open(os.path.join(config.LOG_DIR, 'vette')) + >>> fp.seek(0, 2) + + >>> mlist.emergency = True + >>> mlist.web_page_url = u'http://archives.example.com/' + >>> inject(u'_xtest@example.com', msg) + >>> file_pos = fp.tell() + >>> incoming.run() + >>> len(prep_queue.files) + 0 + >>> len(incoming_queue.files) + 0 + >>> fp.seek(file_pos) + >>> print 'LOG:', fp.read() + LOG: ... HOLD: _xtest@example.com post from aperson@example.com held, + message-id=<first>: n/a + <BLANKLINE> + + >>> mlist.emergency = False + + +Discarded messages +------------------ + +Another possibility is that the message would get immediately discarded. The +built-in chain does not have such a disposition by default, so let's craft a +new chain and set it as the mailing list's start chain. + + >>> from Mailman.chains.base import Chain, Link + >>> from Mailman.interfaces import LinkAction + >>> truth_rule = config.rules['truth'] + >>> discard_chain = config.chains['discard'] + >>> test_chain = Chain('always-discard', u'Testing discards') + >>> link = Link(truth_rule, LinkAction.jump, discard_chain) + >>> test_chain.append_link(link) + >>> mlist.start_chain = u'always-discard' + + >>> inject(u'_xtest@example.com', msg) + >>> file_pos = fp.tell() + >>> incoming.run() + >>> len(prep_queue.files) + 0 + >>> len(incoming_queue.files) + 0 + >>> fp.seek(file_pos) + >>> print 'LOG:', fp.read() + LOG: ... DISCARD: <first> + <BLANKLINE> + + >>> del config.chains['always-discard'] + + +Rejected messages +----------------- + +Similar to discarded messages, a message can be rejected, or bounced back to +the original sender. Again, the built-in chain doesn't support this so we'll +just create a new chain that does. + + >>> reject_chain = config.chains['reject'] + >>> test_chain = Chain('always-reject', u'Testing rejections') + >>> link = Link(truth_rule, LinkAction.jump, reject_chain) + >>> test_chain.append_link(link) + >>> mlist.start_chain = u'always-reject' + +The virgin queue needs to be cleared out due to artifacts from the previous +tests above. + + >>> virgin_queue = Switchboard(config.VIRGINQUEUE_DIR) + >>> ignore = get_queue_messages(virgin_queue) + + >>> inject(u'_xtest@example.com', msg) + >>> file_pos = fp.tell() + >>> incoming.run() + >>> len(prep_queue.files) + 0 + >>> len(incoming_queue.files) + 0 + + >>> len(virgin_queue.files) + 1 + >>> item = get_queue_messages(virgin_queue)[0] + >>> print item.msg.as_string() + Subject: My first post + From: _xtest-owner@example.com + To: aperson@example.com + ... + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + <BLANKLINE> + [No bounce details are available] + ... + Content-Type: message/rfc822 + MIME-Version: 1.0 + <BLANKLINE> + From: aperson@example.com + To: _xtest@example.com + Subject: My first post + Message-ID: <first> + <BLANKLINE> + First post! + <BLANKLINE> + ... + >>> sorted(item.msgdata.items()) + [...('recips', [u'aperson@example.com'])...] + >>> fp.seek(file_pos) + >>> print 'LOG:', fp.read() + LOG: ... REJECT: <first> + <BLANKLINE> + + >>> del config.chains['always-reject'] diff --git a/Mailman/docs/news-runner.txt b/Mailman/queue/docs/news.txt index bc6619f50..bc6619f50 100644 --- a/Mailman/docs/news-runner.txt +++ b/Mailman/queue/docs/news.txt diff --git a/Mailman/docs/outgoing.txt b/Mailman/queue/docs/outgoing.txt index ba2c6430b..ba2c6430b 100644 --- a/Mailman/docs/outgoing.txt +++ b/Mailman/queue/docs/outgoing.txt diff --git a/Mailman/docs/runner.txt b/Mailman/queue/docs/runner.txt index 5e5a88d8c..5e5a88d8c 100644 --- a/Mailman/docs/runner.txt +++ b/Mailman/queue/docs/runner.txt diff --git a/Mailman/docs/switchboard.txt b/Mailman/queue/docs/switchboard.txt index 299aba499..299aba499 100644 --- a/Mailman/docs/switchboard.txt +++ b/Mailman/queue/docs/switchboard.txt diff --git a/Mailman/queue/incoming.py b/Mailman/queue/incoming.py index 6118a7ca0..649ce2213 100644 --- a/Mailman/queue/incoming.py +++ b/Mailman/queue/incoming.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2007 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2008 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -12,102 +12,26 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. -"""Incoming queue runner.""" - -# A typical Mailman list exposes nine aliases which point to seven different -# wrapped scripts. E.g. for a list named `mylist', you'd have: -# -# mylist-bounces -> bounces (-admin is a deprecated alias) -# mylist-confirm -> confirm -# mylist-join -> join (-subscribe is an alias) -# mylist-leave -> leave (-unsubscribe is an alias) -# mylist-owner -> owner -# mylist -> post -# mylist-request -> request -# -# -request, -join, and -leave are a robot addresses; their sole purpose is to -# process emailed commands in a Majordomo-like fashion (although the latter -# two are hardcoded to subscription and unsubscription requests). -bounces is -# the automated bounce processor, and all messages to list members have their -# return address set to -bounces. If the bounce processor fails to extract a -# bouncing member address, it can optionally forward the message on to the -# list owners. -# -# -owner is for reaching a human operator with minimal list interaction -# (i.e. no bounce processing). -confirm is another robot address which -# processes replies to VERP-like confirmation notices. -# -# So delivery flow of messages look like this: -# -# joerandom ---> mylist ---> list members -# | | -# | |[bounces] -# | mylist-bounces <---+ <-------------------------------+ -# | | | -# | +--->[internal bounce processing] | -# | ^ | | -# | | | [bounce found] | -# | [bounces *] +--->[register and discard] | -# | | | | | -# | | | |[*] | -# | [list owners] |[no bounce found] | | -# | ^ | | | -# | | | | | -# +-------> mylist-owner <--------+ | | -# | | | -# | data/owner-bounces.mbox <--[site list] <---+ | -# | | -# +-------> mylist-join--+ | -# | | | -# +------> mylist-leave--+ | -# | | | -# | v | -# +-------> mylist-request | -# | | | -# | +---> [command processor] | -# | | | -# +-----> mylist-confirm ----> +---> joerandom | -# | | -# |[bounces] | -# +----------------------+ -# -# A person can send an email to the list address (for posting), the -owner -# address (to reach the human operator), or the -confirm, -join, -leave, and -# -request mailbots. Message to the list address are then forwarded on to the -# list membership, with bounces directed to the -bounces address. -# -# [*] Messages sent to the -owner address are forwarded on to the list -# owner/moderators. All -owner destined messages have their bounces directed -# to the site list -bounces address, regardless of whether a human sent the -# message or the message was crafted internally. The intention here is that -# the site owners want to be notified when one of their list owners' addresses -# starts bouncing (yes, the will be automated in a future release). -# -# Any messages to site owners has their bounces directed to a special -# "loop-killer" address, which just dumps the message into -# data/owners-bounces.mbox. -# -# Finally, message to any of the mailbots causes the requested action to be -# performed. Results notifications are sent to the author of the message, -# which all bounces pointing back to the -bounces address. +"""Incoming queue runner. +This runner's sole purpose in life is to decide the disposition of the +message. It can either be accepted for delivery, rejected (i.e. bounced), +held for moderator approval, or discarded. - -import os -import sys -import logging +When accepted, the message is forwarded on to the `prep queue` where it is +prepared for delivery. Rejections, discards, and holds are processed +immediately. +""" -from cStringIO import StringIO -from Mailman import Errors + +from Mailman.app.chains import process from Mailman.configuration import config from Mailman.queue import Runner -log = logging.getLogger('mailman.error') -vlog = logging.getLogger('mailman.vette') - class IncomingRunner(Runner): @@ -115,59 +39,8 @@ class IncomingRunner(Runner): def _dispose(self, mlist, msg, msgdata): if msgdata.get('envsender') is None: - msg['envsender'] = mlist.no_reply_address - # Process the message through a handler pipeline. The handler - # pipeline can actually come from one of three places: the message - # metadata, the mlist, or the global pipeline. - # - # If a message was requeued due to an uncaught exception, its metadata - # will contain the retry pipeline. Use this above all else. - # Otherwise, if the mlist has a `pipeline' attribute, it should be - # used. Final fallback is the global pipeline. - pipeline = self._get_pipeline(mlist, msg, msgdata) - msgdata['pipeline'] = pipeline - more = self._dopipeline(mlist, msg, msgdata, pipeline) - if not more: - del msgdata['pipeline'] - config.db.commit() - return more - - # Overridable - def _get_pipeline(self, mlist, msg, msgdata): - # We must return a copy of the list, otherwise, the first message that - # flows through the pipeline will empty it out! - return msgdata.get('pipeline', - getattr(mlist, 'pipeline', - config.GLOBAL_PIPELINE))[:] - - def _dopipeline(self, mlist, msg, msgdata, pipeline): - while pipeline: - handler = pipeline.pop(0) - modname = 'Mailman.Handlers.' + handler - __import__(modname) - try: - pid = os.getpid() - sys.modules[modname].process(mlist, msg, msgdata) - # Failsafe -- a child may have leaked through. - if pid <> os.getpid(): - log.error('child process leaked thru: %s', modname) - os._exit(1) - except Errors.DiscardMessage: - # Throw the message away; we need do nothing else with it. - vlog.info('Message discarded, msgid: %s', - msg.get('message-id', 'n/a')) - return 0 - except Errors.HoldMessage: - # Let the approval process take it from here. The message no - # longer needs to be queued. - return 0 - except Errors.RejectMessage, e: - mlist.bounce_message(msg, e) - return 0 - except: - # Push this pipeline module back on the stack, then re-raise - # the exception. - pipeline.insert(0, handler) - raise - # We've successfully completed handling of this message - return 0 + msgdata['envsender'] = mlist.no_reply_address + # Process the message through the mailing list's start chain. + process(mlist, msg, msgdata, mlist.start_chain) + # Do not keep this message queued. + return False |