diff options
| -rw-r--r-- | Mailman/Post.py | 62 | ||||
| -rw-r--r-- | Mailman/app/styles.py | 3 | ||||
| -rw-r--r-- | Mailman/bin/inject.py | 4 | ||||
| -rw-r--r-- | Mailman/database/mailinglist.py | 3 | ||||
| -rw-r--r-- | Mailman/database/mailman.sql | 1 | ||||
| -rw-r--r-- | Mailman/docs/mlist-addresses.txt | 2 | ||||
| -rw-r--r-- | Mailman/inject.py | 34 | ||||
| -rw-r--r-- | Mailman/interfaces/mailinglist.py | 2 | ||||
| -rw-r--r-- | Mailman/queue/docs/OVERVIEW.txt | 78 | ||||
| -rw-r--r-- | Mailman/queue/docs/incoming.txt | 67 | ||||
| -rw-r--r-- | Mailman/queue/docs/news.txt (renamed from Mailman/docs/news-runner.txt) | 0 | ||||
| -rw-r--r-- | Mailman/queue/docs/outgoing.txt (renamed from Mailman/docs/outgoing.txt) | 0 | ||||
| -rw-r--r-- | Mailman/queue/docs/runner.txt (renamed from Mailman/docs/runner.txt) | 0 | ||||
| -rw-r--r-- | Mailman/queue/docs/switchboard.txt (renamed from Mailman/docs/switchboard.txt) | 0 | ||||
| -rw-r--r-- | Mailman/queue/incoming.py | 163 | ||||
| -rw-r--r-- | Mailman/queue/tests/__init__.py | 0 | ||||
| -rw-r--r-- | Mailman/tests/smtplistener.py | 69 |
17 files changed, 261 insertions, 227 deletions
diff --git a/Mailman/Post.py b/Mailman/Post.py deleted file mode 100644 index 50b9628a0..000000000 --- a/Mailman/Post.py +++ /dev/null @@ -1,62 +0,0 @@ -#! /usr/bin/env python -# -# Copyright (C) 2001-2007 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -import sys - -from Mailman.configuration import config -from Mailman.queue import Switchboard - - - -def inject(listname, msg, recips=None, qdir=None): - if qdir is None: - qdir = config.INQUEUE_DIR - queue = Switchboard(qdir) - kws = {'listname' : listname, - 'tolist' : 1, - '_plaintext': 1, - } - if recips: - kws['recips'] = recips - queue.enqueue(msg, **kws) - - - -if __name__ == '__main__': - # When called as a command line script, standard input is read to get the - # list that this message is destined to, the list of explicit recipients, - # and the message to send (in its entirety). stdin must have the - # following format: - # - # line 1: the internal name of the mailing list - # line 2: the number of explicit recipients to follow. 0 means to use the - # list's membership to calculate recipients. - # line 3 - 3+recipnum: explicit recipients, one per line - # line 4+recipnum - end of file: the message in RFC 822 format (may - # include an initial Unix-from header) - listname = sys.stdin.readline().strip() - numrecips = int(sys.stdin.readline()) - if numrecips == 0: - recips = None - else: - recips = [] - for i in range(numrecips): - recips.append(sys.stdin.readline().strip()) - # If the message isn't parsable, we won't get an error here - inject(listname, sys.stdin.read(), recips) diff --git a/Mailman/app/styles.py b/Mailman/app/styles.py index 3edd88abf..1c978363e 100644 --- a/Mailman/app/styles.py +++ b/Mailman/app/styles.py @@ -225,6 +225,9 @@ class DefaultStyle: # is that they will get all messages, and they will not have an entry # in this dictionary. mlist.topics_userinterest = {} + # The processing chain that messages coming into this list get + # processed by. + mlist.start_chain = u'built-in' def match(self, mailing_list, styles): # If no other styles have matched, then the default style matches. diff --git a/Mailman/bin/inject.py b/Mailman/bin/inject.py index 60185289f..729038118 100644 --- a/Mailman/bin/inject.py +++ b/Mailman/bin/inject.py @@ -19,11 +19,11 @@ import os import sys import optparse -from Mailman import Post from Mailman import Utils from Mailman import Version from Mailman.configuration import config from Mailman.i18n import _ +from Mailman.inject import inject __i18n_templates__ = True @@ -88,7 +88,7 @@ def main(): else: msgtext = sys.stdin.read() - Post.inject(opts.listname, msgtext, qdir=qdir) + inject(opts.listname, msgtext, qdir=qdir) diff --git a/Mailman/database/mailinglist.py b/Mailman/database/mailinglist.py index 3230308eb..b3eb56003 100644 --- a/Mailman/database/mailinglist.py +++ b/Mailman/database/mailinglist.py @@ -151,6 +151,7 @@ class MailingList(Model): send_goodbye_msg = Bool() send_reminders = Bool() send_welcome_msg = Bool() + start_chain = Unicode() subject_prefix = Unicode() subscribe_auto_approval = Pickle() subscribe_policy = Int() @@ -215,7 +216,7 @@ class MailingList(Model): return self.fqdn_listname @property - def noreply_address(self): + def no_reply_address(self): return '%s@%s' % (config.NO_REPLY_ADDRESS, self.host_name) @property diff --git a/Mailman/database/mailman.sql b/Mailman/database/mailman.sql index 0af3401dd..7c53f25be 100644 --- a/Mailman/database/mailman.sql +++ b/Mailman/database/mailman.sql @@ -131,6 +131,7 @@ CREATE TABLE mailinglist ( send_goodbye_msg BOOLEAN, send_reminders BOOLEAN, send_welcome_msg BOOLEAN, + start_chain TEXT, subject_prefix TEXT, subscribe_auto_approval BLOB, subscribe_policy INTEGER, diff --git a/Mailman/docs/mlist-addresses.txt b/Mailman/docs/mlist-addresses.txt index dc2184175..4685a6eea 100644 --- a/Mailman/docs/mlist-addresses.txt +++ b/Mailman/docs/mlist-addresses.txt @@ -18,7 +18,7 @@ list. This is exactly the same as the fully qualified list name. Messages to the mailing list's 'no reply' address always get discarded without prejudice. - >>> mlist.noreply_address + >>> mlist.no_reply_address u'noreply@example.com' The mailing list's owner address reaches the human moderators. diff --git a/Mailman/inject.py b/Mailman/inject.py new file mode 100644 index 000000000..a17f8c7d1 --- /dev/null +++ b/Mailman/inject.py @@ -0,0 +1,34 @@ +# Copyright (C) 2001-2008 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +from Mailman.configuration import config +from Mailman.queue import Switchboard + + + +def inject(listname, msg, recips=None, qdir=None): + if qdir is None: + qdir = config.INQUEUE_DIR + queue = Switchboard(qdir) + kws = dict( + listname=listname, + tolist=True, + _plaintext=True, + ) + if recips is not None: + kws['recips'] = recips + queue.enqueue(msg, **kws) diff --git a/Mailman/interfaces/mailinglist.py b/Mailman/interfaces/mailinglist.py index 2d3811785..2c828a43c 100644 --- a/Mailman/interfaces/mailinglist.py +++ b/Mailman/interfaces/mailinglist.py @@ -82,7 +82,7 @@ class IMailingList(Interface): delivery is currently enabled. """) - noreply_address = Attribute( + no_reply_address = Attribute( """The address to which all messages will be immediately discarded, without prejudice or record. This address is specific to the ddomain, even though it's available on the IMailingListAddresses interface. diff --git a/Mailman/queue/docs/OVERVIEW.txt b/Mailman/queue/docs/OVERVIEW.txt new file mode 100644 index 000000000..643fa8a5c --- /dev/null +++ b/Mailman/queue/docs/OVERVIEW.txt @@ -0,0 +1,78 @@ +Alias overview +============== + +A typical Mailman list exposes nine aliases which point to seven different +wrapped scripts. E.g. for a list named `mylist', you'd have: + + mylist-bounces -> bounces + mylist-confirm -> confirm + mylist-join -> join (-subscribe is an alias) + mylist-leave -> leave (-unsubscribe is an alias) + mylist-owner -> owner + mylist -> post + mylist-request -> request + +-request, -join, and -leave are a robot addresses; their sole purpose is to +process emailed commands, although the latter two are hardcoded to +subscription and unsubscription requests. -bounces is the automated bounce +processor, and all messages to list members have their return address set to +-bounces. If the bounce processor fails to extract a bouncing member address, +it can optionally forward the message on to the list owners. + +-owner is for reaching a human operator with minimal list interaction (i.e. no +bounce processing). -confirm is another robot address which processes replies +to VERP-like confirmation notices. + +So delivery flow of messages look like this: + + joerandom ---> mylist ---> list members + | | + | |[bounces] + | mylist-bounces <---+ <-------------------------------+ + | | | + | +--->[internal bounce processing] | + | ^ | | + | | | [bounce found] | + | [bounces *] +--->[register and discard] | + | | | | | + | | | |[*] | + | [list owners] |[no bounce found] | | + | ^ | | | + | | | | | + +-------> mylist-owner <--------+ | | + | | | + | data/owner-bounces.mbox <--[site list] <---+ | + | | + +-------> mylist-join--+ | + | | | + +------> mylist-leave--+ | + | | | + | v | + +-------> mylist-request | + | | | + | +---> [command processor] | + | | | + +-----> mylist-confirm ----> +---> joerandom | + | | + |[bounces] | + +----------------------+ + +A person can send an email to the list address (for posting), the -owner +address (to reach the human operator), or the -confirm, -join, -leave, and +-request mailbots. Message to the list address are then forwarded on to the +list membership, with bounces directed to the -bounces address. + +[*] Messages sent to the -owner address are forwarded on to the list +owner/moderators. All -owner destined messages have their bounces directed to +the site list -bounces address, regardless of whether a human sent the message +or the message was crafted internally. The intention here is that the site +owners want to be notified when one of their list owners' addresses starts +bouncing (yes, the will be automated in a future release). + +Any messages to site owners has their bounces directed to a special +"loop-killer" address, which just dumps the message into +data/owners-bounces.mbox. + +Finally, message to any of the mailbots causes the requested action to be +performed. Results notifications are sent to the author of the message, which +all bounces pointing back to the -bounces address. diff --git a/Mailman/queue/docs/incoming.txt b/Mailman/queue/docs/incoming.txt new file mode 100644 index 000000000..12ff3d3d1 --- /dev/null +++ b/Mailman/queue/docs/incoming.txt @@ -0,0 +1,67 @@ +The incoming queue runner +========================= + +This runner's sole purpose in life is to decide the disposition of the +message. It can either be accepted for delivery, rejected (i.e. bounced), +held for moderator approval, or discarded. + +The runner operates by processing chains on a message/metadata pair in the +context of a mailing list. Each mailing list may have a 'start chain' where +processing begins, with a global default. This chain is processed with the +message eventually ending up in one of the four disposition states described +above. + + >>> from Mailman.app.lifecycle import create_list + >>> mlist = create_list(u'_xtest@example.com') + >>> mlist.start_chain + u'built-in' + +We have a message that is going to be sent to the mailing list. This message +is so perfectly fine for posting that it will be accepted and forward to the +prep queue. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... Subject: My first post + ... Message-ID: <first> + ... + ... First post! + ... """) + +Normally, the upstream mail server would drop the message in the incoming +queue, but this is an effective simulation. + + >>> from Mailman.inject import inject + >>> inject(u'_xtest@example.com', msg) + +The incoming queue runner runs until it is empty. + + >>> from Mailman.queue.incoming import IncomingRunner + >>> from Mailman.tests.helpers import make_testable_runner + >>> incoming = make_testable_runner(IncomingRunner) + >>> incoming.run() + +And now the message is in the prep queue. + + >>> from Mailman.configuration import config + >>> from Mailman.queue import Switchboard + >>> prep_queue = Switchboard(config.PREPQUEUE_DIR) + >>> len(prep_queue.files) + 1 + >>> from Mailman.tests.helpers import get_queue_messages + >>> item = get_queue_messages(prep_queue)[0] + >>> print item.msg.as_string() + From: aperson@example.com + To: _xtest@example.com + Subject: My first post + Message-ID: <first> + X-Mailman-Rule-Misses: approved; emergency; loop; administrivia; + implicit-dest; + max-recipients; max-size; news-moderation; no-subject; + suspicious-header + <BLANKLINE> + First post! + <BLANKLINE> + >>> sorted(item.msgdata.items()) + [...('envsender', u'noreply@example.com')...('tolist', True)...] diff --git a/Mailman/docs/news-runner.txt b/Mailman/queue/docs/news.txt index bc6619f50..bc6619f50 100644 --- a/Mailman/docs/news-runner.txt +++ b/Mailman/queue/docs/news.txt diff --git a/Mailman/docs/outgoing.txt b/Mailman/queue/docs/outgoing.txt index ba2c6430b..ba2c6430b 100644 --- a/Mailman/docs/outgoing.txt +++ b/Mailman/queue/docs/outgoing.txt diff --git a/Mailman/docs/runner.txt b/Mailman/queue/docs/runner.txt index 5e5a88d8c..5e5a88d8c 100644 --- a/Mailman/docs/runner.txt +++ b/Mailman/queue/docs/runner.txt diff --git a/Mailman/docs/switchboard.txt b/Mailman/queue/docs/switchboard.txt index 299aba499..299aba499 100644 --- a/Mailman/docs/switchboard.txt +++ b/Mailman/queue/docs/switchboard.txt diff --git a/Mailman/queue/incoming.py b/Mailman/queue/incoming.py index 6118a7ca0..649ce2213 100644 --- a/Mailman/queue/incoming.py +++ b/Mailman/queue/incoming.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2007 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2008 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -12,102 +12,26 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. -"""Incoming queue runner.""" - -# A typical Mailman list exposes nine aliases which point to seven different -# wrapped scripts. E.g. for a list named `mylist', you'd have: -# -# mylist-bounces -> bounces (-admin is a deprecated alias) -# mylist-confirm -> confirm -# mylist-join -> join (-subscribe is an alias) -# mylist-leave -> leave (-unsubscribe is an alias) -# mylist-owner -> owner -# mylist -> post -# mylist-request -> request -# -# -request, -join, and -leave are a robot addresses; their sole purpose is to -# process emailed commands in a Majordomo-like fashion (although the latter -# two are hardcoded to subscription and unsubscription requests). -bounces is -# the automated bounce processor, and all messages to list members have their -# return address set to -bounces. If the bounce processor fails to extract a -# bouncing member address, it can optionally forward the message on to the -# list owners. -# -# -owner is for reaching a human operator with minimal list interaction -# (i.e. no bounce processing). -confirm is another robot address which -# processes replies to VERP-like confirmation notices. -# -# So delivery flow of messages look like this: -# -# joerandom ---> mylist ---> list members -# | | -# | |[bounces] -# | mylist-bounces <---+ <-------------------------------+ -# | | | -# | +--->[internal bounce processing] | -# | ^ | | -# | | | [bounce found] | -# | [bounces *] +--->[register and discard] | -# | | | | | -# | | | |[*] | -# | [list owners] |[no bounce found] | | -# | ^ | | | -# | | | | | -# +-------> mylist-owner <--------+ | | -# | | | -# | data/owner-bounces.mbox <--[site list] <---+ | -# | | -# +-------> mylist-join--+ | -# | | | -# +------> mylist-leave--+ | -# | | | -# | v | -# +-------> mylist-request | -# | | | -# | +---> [command processor] | -# | | | -# +-----> mylist-confirm ----> +---> joerandom | -# | | -# |[bounces] | -# +----------------------+ -# -# A person can send an email to the list address (for posting), the -owner -# address (to reach the human operator), or the -confirm, -join, -leave, and -# -request mailbots. Message to the list address are then forwarded on to the -# list membership, with bounces directed to the -bounces address. -# -# [*] Messages sent to the -owner address are forwarded on to the list -# owner/moderators. All -owner destined messages have their bounces directed -# to the site list -bounces address, regardless of whether a human sent the -# message or the message was crafted internally. The intention here is that -# the site owners want to be notified when one of their list owners' addresses -# starts bouncing (yes, the will be automated in a future release). -# -# Any messages to site owners has their bounces directed to a special -# "loop-killer" address, which just dumps the message into -# data/owners-bounces.mbox. -# -# Finally, message to any of the mailbots causes the requested action to be -# performed. Results notifications are sent to the author of the message, -# which all bounces pointing back to the -bounces address. +"""Incoming queue runner. +This runner's sole purpose in life is to decide the disposition of the +message. It can either be accepted for delivery, rejected (i.e. bounced), +held for moderator approval, or discarded. - -import os -import sys -import logging +When accepted, the message is forwarded on to the `prep queue` where it is +prepared for delivery. Rejections, discards, and holds are processed +immediately. +""" -from cStringIO import StringIO -from Mailman import Errors + +from Mailman.app.chains import process from Mailman.configuration import config from Mailman.queue import Runner -log = logging.getLogger('mailman.error') -vlog = logging.getLogger('mailman.vette') - class IncomingRunner(Runner): @@ -115,59 +39,8 @@ class IncomingRunner(Runner): def _dispose(self, mlist, msg, msgdata): if msgdata.get('envsender') is None: - msg['envsender'] = mlist.no_reply_address - # Process the message through a handler pipeline. The handler - # pipeline can actually come from one of three places: the message - # metadata, the mlist, or the global pipeline. - # - # If a message was requeued due to an uncaught exception, its metadata - # will contain the retry pipeline. Use this above all else. - # Otherwise, if the mlist has a `pipeline' attribute, it should be - # used. Final fallback is the global pipeline. - pipeline = self._get_pipeline(mlist, msg, msgdata) - msgdata['pipeline'] = pipeline - more = self._dopipeline(mlist, msg, msgdata, pipeline) - if not more: - del msgdata['pipeline'] - config.db.commit() - return more - - # Overridable - def _get_pipeline(self, mlist, msg, msgdata): - # We must return a copy of the list, otherwise, the first message that - # flows through the pipeline will empty it out! - return msgdata.get('pipeline', - getattr(mlist, 'pipeline', - config.GLOBAL_PIPELINE))[:] - - def _dopipeline(self, mlist, msg, msgdata, pipeline): - while pipeline: - handler = pipeline.pop(0) - modname = 'Mailman.Handlers.' + handler - __import__(modname) - try: - pid = os.getpid() - sys.modules[modname].process(mlist, msg, msgdata) - # Failsafe -- a child may have leaked through. - if pid <> os.getpid(): - log.error('child process leaked thru: %s', modname) - os._exit(1) - except Errors.DiscardMessage: - # Throw the message away; we need do nothing else with it. - vlog.info('Message discarded, msgid: %s', - msg.get('message-id', 'n/a')) - return 0 - except Errors.HoldMessage: - # Let the approval process take it from here. The message no - # longer needs to be queued. - return 0 - except Errors.RejectMessage, e: - mlist.bounce_message(msg, e) - return 0 - except: - # Push this pipeline module back on the stack, then re-raise - # the exception. - pipeline.insert(0, handler) - raise - # We've successfully completed handling of this message - return 0 + msgdata['envsender'] = mlist.no_reply_address + # Process the message through the mailing list's start chain. + process(mlist, msg, msgdata, mlist.start_chain) + # Do not keep this message queued. + return False diff --git a/Mailman/queue/tests/__init__.py b/Mailman/queue/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/Mailman/queue/tests/__init__.py +++ /dev/null diff --git a/Mailman/tests/smtplistener.py b/Mailman/tests/smtplistener.py index 565772b1d..977726247 100644 --- a/Mailman/tests/smtplistener.py +++ b/Mailman/tests/smtplistener.py @@ -1,4 +1,4 @@ -# Copyright (C) 2007 by the Free Software Foundation, Inc. +# Copyright (C) 2007-2008 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -15,8 +15,11 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, # USA. +"""A test SMTP listener.""" + import sys import smtpd +import signal import mailbox import asyncore import optparse @@ -29,34 +32,68 @@ DEFAULT_PORT = 9025 class Channel(smtpd.SMTPChannel): - def smtp_EXIT(self, arg): - raise asyncore.ExitNow + """A channel that can reset the mailbox.""" + + def __init__(self, server, conn, addr): + smtpd.SMTPChannel.__init__(self, server, conn, addr) + # Stash this here since the subclass uses private attributes. :( + self._server = server + + def smtp_RSET(self, arg): + """Respond to RSET and clear the mailbox.""" + self._server.clear_mailbox() + smtpd.SMTPChannel.smtp_RSET(self, arg) + def send(self, data): + """Silence the bloody asynchat/asyncore broken pipe errors!""" + try: + return smtpd.SMTPChannel.send(self, data) + except socket.error: + # Nothing here can affect the outcome, and these messages are just + # plain annoying! So ignore them. + pass + + class Server(smtpd.SMTPServer): - def __init__(self, localaddr, mboxfile): + """An SMTP server that stores messages to a mailbox.""" + + def __init__(self, localaddr, mailbox_path): smtpd.SMTPServer.__init__(self, localaddr, None) - self._mbox = mailbox.mbox(mboxfile) + self._mailbox = mailbox.Maildir(mailbox_path) def handle_accept(self): + """Handle connections by creating our own Channel object.""" conn, addr = self.accept() Channel(self, conn, addr) def process_message(self, peer, mailfrom, rcpttos, data): + """Process a message by adding it to the mailbox.""" msg = message_from_string(data) msg['X-Peer'] = peer msg['X-MailFrom'] = mailfrom msg['X-RcptTo'] = COMMASPACE.join(rcpttos) - self._mbox.add(msg) + self._mailbox.add(msg) + self._mailbox.clean() - def close(self): - self._mbox.flush() - self._mbox.close() + + +def handle_signal(*ignore): + """Handle signal sent by parent to kill the process.""" + asyncore.socket_map.clear() def main(): - parser = optparse.OptionParser(usage='%prog mboxfile') + parser = optparse.OptionParser(usage="""\ +%prog [options] mboxfile + +This starts a process listening on a specified host and port (by default +localhost:9025) for SMTP conversations. All messages this process receives +are stored in a specified mbox file for the parent process to investigate. + +This SMTP server responds to RSET commands by clearing the mbox file. +""") parser.add_option('-a', '--address', type='string', default=None, help='host:port to listen on') @@ -77,12 +114,14 @@ def main(): host, port = opts.address.split(':', 1) port = int(port) + # Catch the parent's exit signal, and also C-c. + signal.signal(signal.SIGTERM, handle_signal) + signal.signal(signal.SIGINT, handle_signal) + server = Server((host, port), mboxfile) - try: - asyncore.loop() - except asyncore.ExitNow: - asyncore.close_all() - server.close() + asyncore.loop() + asyncore.close_all() + server.close() return 0 |