From c70909dbe5cf6b32ddc72963fd02eda0b5bce6d2 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Sat, 29 Dec 2007 11:15:10 -0500 Subject: Convert the administrivia check from the Hold handler to the administrivia rule. Add doctest as appropriate. DEFAULT_MAIL_COMMANDS_MAX_LINES -> EMAIL_COMMANDS_MAX_LINES --- Mailman/queue/command.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Mailman/queue') diff --git a/Mailman/queue/command.py b/Mailman/queue/command.py index dfbfd1255..8628411de 100644 --- a/Mailman/queue/command.py +++ b/Mailman/queue/command.py @@ -88,8 +88,8 @@ class Results: assert isinstance(body, basestring) lines = body.splitlines() # Use no more lines than specified - self.commands.extend(lines[:config.DEFAULT_MAIL_COMMANDS_MAX_LINES]) - self.ignored.extend(lines[config.DEFAULT_MAIL_COMMANDS_MAX_LINES:]) + self.commands.extend(lines[:config.EMAIL_COMMANDS_MAX_LINES]) + self.ignored.extend(lines[config.EMAIL_COMMANDS_MAX_LINES:]) def process(self): # Now, process each line until we find an error. The first -- cgit v1.2.3-70-g09d2 From 86f00a6cec71753952d1290bdadd836fdba5fdc1 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Sat, 29 Dec 2007 19:53:23 -0500 Subject: Port the maximum message size check to a rule. Add doctest. Rename the implicit.txt doctest. specialized_message_from_string(): Mimic the way the text->message parser will include the size of the original text as an attribute on the message object. The maximum message size rule will use this information. --- Mailman/Handlers/Hold.py | 16 -------- Mailman/docs/hold.txt | 22 ----------- Mailman/docs/implicit-dest.txt | 77 +++++++++++++++++++++++++++++++++++++ Mailman/docs/implicit.txt | 77 ------------------------------------- Mailman/docs/max-size.txt | 41 ++++++++++++++++++++ Mailman/queue/__init__.py | 6 +++ Mailman/rules/implicit_dest.py | 1 - Mailman/rules/max_size.py | 49 +++++++++++++++++++++++ Mailman/tests/test_documentation.py | 14 ++++++- 9 files changed, 186 insertions(+), 117 deletions(-) create mode 100644 Mailman/docs/implicit-dest.txt delete mode 100644 Mailman/docs/implicit.txt create mode 100644 Mailman/docs/max-size.txt create mode 100644 Mailman/rules/max_size.py (limited to 'Mailman/queue') diff --git a/Mailman/Handlers/Hold.py b/Mailman/Handlers/Hold.py index 12bab8a8b..81dab3189 100644 --- a/Mailman/Handlers/Hold.py +++ b/Mailman/Handlers/Hold.py @@ -165,22 +165,6 @@ def process(mlist, msg, msgdata): # message because the info would also go to the sender hold_for_approval(mlist, msg, msgdata, SuspiciousHeaders) # no return - # - # Is the message too big? - if mlist.max_message_size > 0: - bodylen = 0 - for line in email.Iterators.body_line_iterator(msg): - bodylen += len(line) - for part in msg.walk(): - if part.preamble: - bodylen += len(part.preamble) - if part.epilogue: - bodylen += len(part.epilogue) - if bodylen/1024.0 > mlist.max_message_size: - hold_for_approval(mlist, msg, msgdata, - MessageTooBig(bodylen, mlist.max_message_size)) - # no return - # # Are we gatewaying to a moderated newsgroup and is this list the # moderator's address for the group? if mlist.news_moderation == 2: diff --git a/Mailman/docs/hold.txt b/Mailman/docs/hold.txt index ed3fabdf0..16948331f 100644 --- a/Mailman/docs/hold.txt +++ b/Mailman/docs/hold.txt @@ -96,28 +96,6 @@ Just a bit of clean up. >>> mlist.bounce_matching_headers = None -Message size ------------- - -Mailman can hold messages that are bigger than a given size. Generally this -is used to prevent huge attachments from getting posted to the list. This -value is calculated in terms of KB (1024 bytes). - - >>> mlist.max_message_size = 1 - >>> one_line = 'x' * 79 - >>> big_body = '\n'.join([one_line] * 15) - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: _xtest@example.com - ... - ... """ + big_body) - >>> process(mlist, msg, {}) - Traceback (most recent call last): - ... - MessageTooBig - >>> clear() - - X Hold Notifications X ------------------ X diff --git a/Mailman/docs/implicit-dest.txt b/Mailman/docs/implicit-dest.txt new file mode 100644 index 000000000..b6fed2769 --- /dev/null +++ b/Mailman/docs/implicit-dest.txt @@ -0,0 +1,77 @@ +Implicit destination +==================== + +The 'implicit-dest' rule matches when the mailing list's posting address is +not explicitly mentioned in the set of message recipients. + + >>> from Mailman.configuration import config + >>> mlist = config.db.list_manager.create(u'_xtest@example.com') + >>> from Mailman.app.rules import find_rule + >>> rule = find_rule('implicit-dest') + >>> rule.name + 'implicit-dest' + +This rule matches messages that have implicit destination, meaning that the +mailing list's posting address isn't included in the explicit recipients. + + >>> mlist.require_explicit_destination = True + >>> mlist.acceptable_aliases = u'' + >>> msg = message_from_string(u"""\ + ... From: aperson@example.org + ... Subject: An implicit message + ... + ... """) + >>> rule.check(mlist, msg, {}) + True + +You can disable implicit destination checks for the mailing list. + + >>> mlist.require_explicit_destination = False + >>> rule.check(mlist, msg, {}) + False + +Even with some recipients, if the posting address is not included, the rule +will match. + + >>> mlist.require_explicit_destination = True + >>> msg['To'] = 'myfriend@example.com' + >>> rule.check(mlist, msg, {}) + True + +Add the posting address as a recipient and the rule will no longer match. + + >>> msg['Cc'] = '_xtest@example.com' + >>> rule.check(mlist, msg, {}) + False + +Alternatively, if one of the acceptable aliases is in the recipients list, +then the rule will not match. + + >>> del msg['cc'] + >>> rule.check(mlist, msg, {}) + True + >>> mlist.acceptable_aliases = u'myfriend@example.com' + >>> rule.check(mlist, msg, {}) + False + +A message gated from NNTP will obviously have an implicit destination. Such +gated messages will not be held for implicit destination because it's assumed +that Mailman pulled it from the appropriate news group. + + >>> rule.check(mlist, msg, dict(fromusenet=True)) + False + + +Alias patterns +-------------- + +It's also possible to specify an alias pattern, i.e. a regular expression to +match against the recipients. For example, we can say that if there is a +recipient in the example.net domain, then the rule does not match. + + >>> mlist.acceptable_aliases = u'^.*@example.net' + >>> rule.check(mlist, msg, {}) + True + >>> msg['To'] = 'you@example.net' + >>> rule.check(mlist, msg, {}) + False diff --git a/Mailman/docs/implicit.txt b/Mailman/docs/implicit.txt deleted file mode 100644 index 967ddc8c7..000000000 --- a/Mailman/docs/implicit.txt +++ /dev/null @@ -1,77 +0,0 @@ -Implicit destination -==================== - -The 'implicit-dest' rule matches when the mailing list's posting address is -not explicitly mentioned in the set of message recipients. - - >>> from Mailman.configuration import config - >>> mlist = config.db.list_manager.create(u'_xtest@example.com') - >>> from Mailman.app.rules import find_rule - >>> rule = find_rule('implicit-dest') - >>> rule.name - 'implicit-dest' - -Mailman will hold messages that have implicit destination, meaning that the -mailing list's posting address isn't included in the explicit recipients. - - >>> mlist.require_explicit_destination = True - >>> mlist.acceptable_aliases = u'' - >>> msg = message_from_string(u"""\ - ... From: aperson@example.org - ... Subject: An implicit message - ... - ... """) - >>> rule.check(mlist, msg, {}) - True - -You can disable implicit destination checks for the mailing list. - - >>> mlist.require_explicit_destination = False - >>> rule.check(mlist, msg, {}) - False - -Even with some recipients, if the posting address is not included, the rule -will match. - - >>> mlist.require_explicit_destination = True - >>> msg['To'] = 'myfriend@example.com' - >>> rule.check(mlist, msg, {}) - True - -Add the posting address as a recipient and the rule will no longer match. - - >>> msg['Cc'] = '_xtest@example.com' - >>> rule.check(mlist, msg, {}) - False - -Alternatively, if one of the acceptable aliases is in the recipients list, -then the rule will not match. - - >>> del msg['cc'] - >>> rule.check(mlist, msg, {}) - True - >>> mlist.acceptable_aliases = u'myfriend@example.com' - >>> rule.check(mlist, msg, {}) - False - -A message gated from NNTP will obviously have an implicit destination. Such -gated messages will not be held for implicit destination because it's assumed -that Mailman pulled it from the appropriate news group. - - >>> rule.check(mlist, msg, dict(fromusenet=True)) - False - - -Alias patterns --------------- - -It's also possible to specify an alias pattern, i.e. a regular expression to -match against the recipients. For example, we can say that if there is a -recipient in the example.net domain, then the rule does not match. - - >>> mlist.acceptable_aliases = u'^.*@example.net' - >>> rule.check(mlist, msg, {}) - True - >>> msg['To'] = 'you@example.net' - >>> rule.check(mlist, msg, {}) - False diff --git a/Mailman/docs/max-size.txt b/Mailman/docs/max-size.txt new file mode 100644 index 000000000..b477ecd2b --- /dev/null +++ b/Mailman/docs/max-size.txt @@ -0,0 +1,41 @@ +Message size +============ + +The 'message-size' rule matches when the posted message is bigger than a +specified maximum. Generally this is used to prevent huge attachments from +getting posted to the list. This value is calculated in terms of KB (1024 +bytes). + + >>> from Mailman.configuration import config + >>> mlist = config.db.list_manager.create(u'_xtest@example.com') + >>> from Mailman.app.rules import find_rule + >>> rule = find_rule('max-size') + >>> rule.name + 'max-size' + +For example, setting the maximum message size to 1 means that any message +bigger than that will match the rule. + + >>> mlist.max_message_size = 1 # 1024 bytes + >>> one_line = u'x' * 79 + >>> big_body = u'\n'.join([one_line] * 15) + >>> msg = message_from_string(u"""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... + ... """ + big_body) + >>> rule.check(mlist, msg, {}) + True + +Setting the maximum message size to zero means no size check is performed. + + >>> mlist.max_message_size = 0 + >>> rule.check(mlist, msg, {}) + False + +Of course, if the maximum size is larger than the message's size, then it's +still okay. + + >>> mlist.max_message_size = msg.original_size/1024.0 + 1 + >>> rule.check(mlist, msg, {}) + False diff --git a/Mailman/queue/__init__.py b/Mailman/queue/__init__.py index c415834ba..6a1873d60 100644 --- a/Mailman/queue/__init__.py +++ b/Mailman/queue/__init__.py @@ -140,7 +140,13 @@ class Switchboard: msg = cPickle.load(fp) data = cPickle.load(fp) if data.get('_parsemsg'): + # Calculate the original size of the text now so that we won't + # have to generate the message later when we do size restriction + # checking. + original_size = len(msg) msg = email.message_from_string(msg, Message.Message) + msg.original_size = original_size + data['original_size'] = original_size return msg, data def finish(self, filebase, preserve=False): diff --git a/Mailman/rules/implicit_dest.py b/Mailman/rules/implicit_dest.py index 8c9ba8899..e056a2e9e 100644 --- a/Mailman/rules/implicit_dest.py +++ b/Mailman/rules/implicit_dest.py @@ -23,7 +23,6 @@ __metaclass__ = type import re from email.utils import getaddresses -from itertools import chain from zope.interface import implements from Mailman.i18n import _ diff --git a/Mailman/rules/max_size.py b/Mailman/rules/max_size.py new file mode 100644 index 000000000..1e62b2ad8 --- /dev/null +++ b/Mailman/rules/max_size.py @@ -0,0 +1,49 @@ +# Copyright (C) 2007 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +"""The maximum message size rule.""" + +__all__ = ['max_size'] +__metaclass__ = type + + +from zope.interface import implements + +from Mailman.i18n import _ +from Mailman.interfaces import IRule + + + +class MaximumSize: + """The implicit destination rule.""" + implements(IRule) + + name = 'max-size' + description = _('Catch messages that are bigger than a specified maximum.') + + def check(self, mlist, msg, msgdata): + """See `IRule`.""" + if mlist.max_message_size == 0: + return False + assert hasattr(msg, 'original_size'), ( + 'Message was not sized on initial parsing.') + # The maximum size is specified in 1024 bytes. + return msg.original_size / 1024.0 > mlist.max_message_size + + + +max_size = MaximumSize() diff --git a/Mailman/tests/test_documentation.py b/Mailman/tests/test_documentation.py index 390ba6a66..36b3c7ecb 100644 --- a/Mailman/tests/test_documentation.py +++ b/Mailman/tests/test_documentation.py @@ -36,7 +36,19 @@ COMMASPACE = ', ' def specialized_message_from_string(text): - return message_from_string(text, Message) + """Parse text into a message object. + + This is specialized in the sense that an instance of Mailman's own Message + object is returned, and this message object has an attribute + `original_size` which is the pre-calculated size in bytes of the message's + text representation. + """ + # This mimic what Switchboard.dequeue() does when parsing a message from + # text into a Message instance. + original_size = len(text) + message = message_from_string(text, Message) + message.original_size = original_size + return message def setup(testobj): -- cgit v1.2.3-70-g09d2 From d865604398932718dab761f3fb4f56c3a18d25b8 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Sat, 2 Feb 2008 11:18:22 -0500 Subject: Convert IncomingRunner to use the new chains disposition architecture. move the big explanatory text at the beginning of incoming.py to a doctest called OVERVIEW.tt (which doesn't actually contain any tests yet -- it's documentation though). Added a doctest for the incoming runner, though this will be fleshed out in more detail next. Mailman.Post renamed to Mailman.inject, and simplified. We don't need its command line script behavior because that is now handled by bin/inject. Add a 'start_chain' attribute to mailing lists. This names the chain that processing of messages for that list begins with. We were inconsistent in the use of the 'no reply' address attribute. It's now always 'no_reply_address'. Update the smtplistener helper with lessons learned about how to suppress bogus asyncore error messages. Also, switch to using a maildir mailbox instead of an mbox mailbox. --- Mailman/Post.py | 62 -------------- Mailman/app/styles.py | 3 + Mailman/bin/inject.py | 4 +- Mailman/database/mailinglist.py | 3 +- Mailman/database/mailman.sql | 1 + Mailman/docs/mlist-addresses.txt | 2 +- Mailman/docs/news-runner.txt | 158 ----------------------------------- Mailman/docs/outgoing.txt | 155 ----------------------------------- Mailman/docs/runner.txt | 70 ---------------- Mailman/docs/switchboard.txt | 149 --------------------------------- Mailman/inject.py | 34 ++++++++ Mailman/interfaces/mailinglist.py | 2 +- Mailman/queue/docs/OVERVIEW.txt | 78 ++++++++++++++++++ Mailman/queue/docs/incoming.txt | 67 +++++++++++++++ Mailman/queue/docs/news.txt | 158 +++++++++++++++++++++++++++++++++++ Mailman/queue/docs/outgoing.txt | 155 +++++++++++++++++++++++++++++++++++ Mailman/queue/docs/runner.txt | 70 ++++++++++++++++ Mailman/queue/docs/switchboard.txt | 149 +++++++++++++++++++++++++++++++++ Mailman/queue/incoming.py | 163 ++++--------------------------------- Mailman/queue/tests/__init__.py | 0 Mailman/tests/smtplistener.py | 69 ++++++++++++---- 21 files changed, 793 insertions(+), 759 deletions(-) delete mode 100644 Mailman/Post.py delete mode 100644 Mailman/docs/news-runner.txt delete mode 100644 Mailman/docs/outgoing.txt delete mode 100644 Mailman/docs/runner.txt delete mode 100644 Mailman/docs/switchboard.txt create mode 100644 Mailman/inject.py create mode 100644 Mailman/queue/docs/OVERVIEW.txt create mode 100644 Mailman/queue/docs/incoming.txt create mode 100644 Mailman/queue/docs/news.txt create mode 100644 Mailman/queue/docs/outgoing.txt create mode 100644 Mailman/queue/docs/runner.txt create mode 100644 Mailman/queue/docs/switchboard.txt delete mode 100644 Mailman/queue/tests/__init__.py (limited to 'Mailman/queue') diff --git a/Mailman/Post.py b/Mailman/Post.py deleted file mode 100644 index 50b9628a0..000000000 --- a/Mailman/Post.py +++ /dev/null @@ -1,62 +0,0 @@ -#! /usr/bin/env python -# -# Copyright (C) 2001-2007 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -import sys - -from Mailman.configuration import config -from Mailman.queue import Switchboard - - - -def inject(listname, msg, recips=None, qdir=None): - if qdir is None: - qdir = config.INQUEUE_DIR - queue = Switchboard(qdir) - kws = {'listname' : listname, - 'tolist' : 1, - '_plaintext': 1, - } - if recips: - kws['recips'] = recips - queue.enqueue(msg, **kws) - - - -if __name__ == '__main__': - # When called as a command line script, standard input is read to get the - # list that this message is destined to, the list of explicit recipients, - # and the message to send (in its entirety). stdin must have the - # following format: - # - # line 1: the internal name of the mailing list - # line 2: the number of explicit recipients to follow. 0 means to use the - # list's membership to calculate recipients. - # line 3 - 3+recipnum: explicit recipients, one per line - # line 4+recipnum - end of file: the message in RFC 822 format (may - # include an initial Unix-from header) - listname = sys.stdin.readline().strip() - numrecips = int(sys.stdin.readline()) - if numrecips == 0: - recips = None - else: - recips = [] - for i in range(numrecips): - recips.append(sys.stdin.readline().strip()) - # If the message isn't parsable, we won't get an error here - inject(listname, sys.stdin.read(), recips) diff --git a/Mailman/app/styles.py b/Mailman/app/styles.py index 3edd88abf..1c978363e 100644 --- a/Mailman/app/styles.py +++ b/Mailman/app/styles.py @@ -225,6 +225,9 @@ class DefaultStyle: # is that they will get all messages, and they will not have an entry # in this dictionary. mlist.topics_userinterest = {} + # The processing chain that messages coming into this list get + # processed by. + mlist.start_chain = u'built-in' def match(self, mailing_list, styles): # If no other styles have matched, then the default style matches. diff --git a/Mailman/bin/inject.py b/Mailman/bin/inject.py index 60185289f..729038118 100644 --- a/Mailman/bin/inject.py +++ b/Mailman/bin/inject.py @@ -19,11 +19,11 @@ import os import sys import optparse -from Mailman import Post from Mailman import Utils from Mailman import Version from Mailman.configuration import config from Mailman.i18n import _ +from Mailman.inject import inject __i18n_templates__ = True @@ -88,7 +88,7 @@ def main(): else: msgtext = sys.stdin.read() - Post.inject(opts.listname, msgtext, qdir=qdir) + inject(opts.listname, msgtext, qdir=qdir) diff --git a/Mailman/database/mailinglist.py b/Mailman/database/mailinglist.py index 3230308eb..b3eb56003 100644 --- a/Mailman/database/mailinglist.py +++ b/Mailman/database/mailinglist.py @@ -151,6 +151,7 @@ class MailingList(Model): send_goodbye_msg = Bool() send_reminders = Bool() send_welcome_msg = Bool() + start_chain = Unicode() subject_prefix = Unicode() subscribe_auto_approval = Pickle() subscribe_policy = Int() @@ -215,7 +216,7 @@ class MailingList(Model): return self.fqdn_listname @property - def noreply_address(self): + def no_reply_address(self): return '%s@%s' % (config.NO_REPLY_ADDRESS, self.host_name) @property diff --git a/Mailman/database/mailman.sql b/Mailman/database/mailman.sql index 0af3401dd..7c53f25be 100644 --- a/Mailman/database/mailman.sql +++ b/Mailman/database/mailman.sql @@ -131,6 +131,7 @@ CREATE TABLE mailinglist ( send_goodbye_msg BOOLEAN, send_reminders BOOLEAN, send_welcome_msg BOOLEAN, + start_chain TEXT, subject_prefix TEXT, subscribe_auto_approval BLOB, subscribe_policy INTEGER, diff --git a/Mailman/docs/mlist-addresses.txt b/Mailman/docs/mlist-addresses.txt index dc2184175..4685a6eea 100644 --- a/Mailman/docs/mlist-addresses.txt +++ b/Mailman/docs/mlist-addresses.txt @@ -18,7 +18,7 @@ list. This is exactly the same as the fully qualified list name. Messages to the mailing list's 'no reply' address always get discarded without prejudice. - >>> mlist.noreply_address + >>> mlist.no_reply_address u'noreply@example.com' The mailing list's owner address reaches the human moderators. diff --git a/Mailman/docs/news-runner.txt b/Mailman/docs/news-runner.txt deleted file mode 100644 index bc6619f50..000000000 --- a/Mailman/docs/news-runner.txt +++ /dev/null @@ -1,158 +0,0 @@ -The news runner -=============== - -The news runner is the queue runner that gateways mailing list messages to an -NNTP newsgroup. One of the most important things this runner does is prepare -the message for Usenet (yes, I know that NNTP is not Usenet, but this runner -was originally written to gate to Usenet, which has its own rules). - - >>> from Mailman.configuration import config - >>> from Mailman.queue.news import prepare_message - >>> mlist = config.db.list_manager.create(u'_xtest@example.com') - >>> mlist.linked_newsgroup = u'comp.lang.python' - -Some NNTP servers such as INN reject messages containing a set of prohibited -headers, so one of the things that the news runner does is remove these -prohibited headers. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: _xtest@example.com - ... NNTP-Posting-Host: news.example.com - ... NNTP-Posting-Date: today - ... X-Trace: blah blah - ... X-Complaints-To: abuse@dom.ain - ... Xref: blah blah - ... Xref: blah blah - ... Date-Received: yesterday - ... Posted: tomorrow - ... Posting-Version: 99.99 - ... Relay-Version: 88.88 - ... Received: blah blah - ... - ... A message - ... """) - >>> msgdata = {} - >>> prepare_message(mlist, msg, msgdata) - >>> msgdata['prepped'] - True - >>> print msg.as_string() - From: aperson@example.com - To: _xtest@example.com - Newsgroups: comp.lang.python - Message-ID: ... - Lines: 1 - - A message - - -Some NNTP servers will reject messages where certain headers are duplicated, -so the news runner must collapse or move these duplicate headers to an -X-Original-* header that the news server doesn't care about. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: _xtest@example.com - ... To: two@example.com - ... Cc: three@example.com - ... Cc: four@example.com - ... Cc: five@example.com - ... Content-Transfer-Encoding: yes - ... Content-Transfer-Encoding: no - ... Content-Transfer-Encoding: maybe - ... - ... A message - ... """) - >>> msgdata = {} - >>> prepare_message(mlist, msg, msgdata) - >>> msgdata['prepped'] - True - >>> print msg.as_string() - From: aperson@example.com - Newsgroups: comp.lang.python - Message-ID: ... - Lines: 1 - To: _xtest@example.com - X-Original-To: two@example.com - CC: three@example.com - X-Original-CC: four@example.com - X-Original-CC: five@example.com - Content-Transfer-Encoding: yes - X-Original-Content-Transfer-Encoding: no - X-Original-Content-Transfer-Encoding: maybe - - A message - - -But if no headers are duplicated, then the news runner doesn't need to modify -the message. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: _xtest@example.com - ... Cc: someother@example.com - ... Content-Transfer-Encoding: yes - ... - ... A message - ... """) - >>> msgdata = {} - >>> prepare_message(mlist, msg, msgdata) - >>> msgdata['prepped'] - True - >>> print msg.as_string() - From: aperson@example.com - To: _xtest@example.com - Cc: someother@example.com - Content-Transfer-Encoding: yes - Newsgroups: comp.lang.python - Message-ID: ... - Lines: 1 - - A message - - - -Newsgroup moderation --------------------- - -When the newsgroup is moderated, an Approved: header with the list's posting -address is added for the benefit of the Usenet system. - - >>> from Mailman.interfaces import NewsModeration - >>> mlist.news_moderation = NewsModeration.open_moderated - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: _xtest@example.com - ... Approved: this gets deleted - ... - ... """) - >>> prepare_message(mlist, msg, {}) - >>> msg['approved'] - u'_xtest@example.com' - - >>> mlist.news_moderation = NewsModeration.moderated - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: _xtest@example.com - ... Approved: this gets deleted - ... - ... """) - >>> prepare_message(mlist, msg, {}) - >>> msg['approved'] - u'_xtest@example.com' - -But if the newsgroup is not moderated, the Approved: header is not chnaged. - - >>> mlist.news_moderation = NewsModeration.none - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: _xtest@example.com - ... Approved: this doesn't get deleted - ... - ... """) - >>> prepare_message(mlist, msg, {}) - >>> msg['approved'] - u"this doesn't get deleted" - - -XXX More of the NewsRunner should be tested. diff --git a/Mailman/docs/outgoing.txt b/Mailman/docs/outgoing.txt deleted file mode 100644 index ba2c6430b..000000000 --- a/Mailman/docs/outgoing.txt +++ /dev/null @@ -1,155 +0,0 @@ -The outgoing handler -==================== - -Mailman's outgoing queue is used as the wrapper around SMTP delivery to the -upstream mail server. The ToOutgoing handler does little more than drop the -message into the outgoing queue, after calculating whether the message should -be VERP'd or not. VERP means Variable Envelope Return Path; we're using that -term somewhat incorrectly, but within the spirit of the standard, which -basically describes how to encode the recipient's address in the originator -headers for unambigous bounce processing. - - >>> from Mailman.Handlers.ToOutgoing import process - >>> from Mailman.queue import Switchboard - >>> from Mailman.configuration import config - >>> mlist = config.db.list_manager.create(u'_xtest@example.com') - >>> switchboard = Switchboard(config.OUTQUEUE_DIR) - - >>> def queue_size(): - ... size = len(switchboard.files) - ... for filebase in switchboard.files: - ... msg, msgdata = switchboard.dequeue(filebase) - ... switchboard.finish(filebase) - ... return size - -Craft a message destined for the outgoing queue. Include some random metadata -as if this message had passed through some other handlers. - - >>> msg = message_from_string("""\ - ... Subject: Here is a message - ... - ... Something of great import. - ... """) - -When certain conditions are met, the message will be VERP'd. For example, if -the message metadata already has a VERP key, this message will be VERP'd. - - >>> msgdata = dict(foo=1, bar=2, verp=True) - >>> process(mlist, msg, msgdata) - >>> print msg.as_string() - Subject: Here is a message - - Something of great import. - >>> msgdata['verp'] - True - -While the queued message will not be changed, the queued metadata will have an -additional key set: the mailing list name. - - >>> filebase = switchboard.files[0] - >>> qmsg, qmsgdata = switchboard.dequeue(filebase) - >>> switchboard.finish(filebase) - >>> print qmsg.as_string() - Subject: Here is a message - - Something of great import. - >>> sorted(qmsgdata.items()) - [('_parsemsg', False), - ('bar', 2), ('foo', 1), - ('listname', u'_xtest@example.com'), - ('received_time', ...), - ('verp', True), ('version', 3)] - >>> queue_size() - 0 - -If the list is set to personalize deliveries, and the global configuration -option to VERP personalized deliveries is set, then the message will be -VERP'd. - - # Save the original value for clean up. - >>> verp_personalized_delivieries = config.VERP_PERSONALIZED_DELIVERIES - >>> config.VERP_PERSONALIZED_DELIVERIES = True - >>> from Mailman.interfaces import Personalization - >>> mlist.personalize = Personalization.individual - >>> msgdata = dict(foo=1, bar=2) - >>> process(mlist, msg, msgdata) - >>> msgdata['verp'] - True - >>> queue_size() - 1 - -However, if the global configuration variable prohibits VERP'ing, even -personalized lists will not VERP. - - >>> config.VERP_PERSONALIZED_DELIVERIES = False - >>> msgdata = dict(foo=1, bar=2) - >>> process(mlist, msg, msgdata) - >>> print msgdata.get('verp') - None - >>> queue_size() - 1 - -If the list is not personalized, then the message may still be VERP'd based on -the global configuration variable VERP_DELIVERY_INTERVAL. This variable tells -Mailman how often to VERP even non-personalized mailing lists. It can be set -to zero, which means non-personalized messages will never be VERP'd. - - # Save the original value for clean up. - >>> verp_delivery_interval = config.VERP_DELIVERY_INTERVAL - >>> config.VERP_DELIVERY_INTERVAL = 0 - >>> mlist.personalize = Personalization.none - >>> msgdata = dict(foo=1, bar=2) - >>> process(mlist, msg, msgdata) - >>> print msgdata.get('verp') - None - >>> queue_size() - 1 - -If the interval is set to 1, then every message will be VERP'd. - - >>> config.VERP_DELIVERY_INTERVAL = 1 - >>> for i in range(10): - ... msgdata = dict(foo=1, bar=2) - ... process(mlist, msg, msgdata) - ... print i, msgdata['verp'] - 0 True - 1 True - 2 True - 3 True - 4 True - 5 True - 6 True - 7 True - 8 True - 9 True - >>> queue_size() - 10 - -If the interval is set to some other number, then one out of that many posts -will be VERP'd. - - >>> config.VERP_DELIVERY_INTERVAL = 3 - >>> for i in range(10): - ... mlist.post_id = i - ... msgdata = dict(foo=1, bar=2) - ... process(mlist, msg, msgdata) - ... print i, msgdata.get('verp', False) - 0 True - 1 False - 2 False - 3 True - 4 False - 5 False - 6 True - 7 False - 8 False - 9 True - >>> queue_size() - 10 - - -Clean up -======== - - >>> config.VERP_PERSONALIZED_DELIVERIES = verp_personalized_delivieries - >>> config.VERP_DELIVERY_INTERVAL = verp_delivery_interval diff --git a/Mailman/docs/runner.txt b/Mailman/docs/runner.txt deleted file mode 100644 index 5e5a88d8c..000000000 --- a/Mailman/docs/runner.txt +++ /dev/null @@ -1,70 +0,0 @@ -Queue runners -============= - -The queue runners (qrunner) are the processes that move messages around the -Mailman system. Each qrunner is responsible for a slice of the hash space in -a queue directory. It processes all the files in its slice, sleeps a little -while, then wakes up and runs through its queue files again. - - -Basic architecture ------------------- - -The basic architecture of qrunner is implemented in the base class that all -runners inherit from. This base class implements a .run() method that runs -continuously in a loop until the .stop() method is called. - - >>> import os - >>> from Mailman.queue import Runner, Switchboard - >>> from Mailman.configuration import config - >>> mlist = config.db.list_manager.create(u'_xtest@example.com') - >>> mlist.preferred_language = u'en' - -Here is a very simple derived qrunner class. The class attribute QDIR tells -the qrunner which queue directory it is responsible for. Derived classes -should also implement various methods to provide the special functionality. -This is about as simple as a qrunner can be. - - >>> queue_directory = os.path.join(config.QUEUE_DIR, 'test') - >>> class TestableRunner(Runner): - ... QDIR = queue_directory - ... - ... def _dispose(self, mlist, msg, msgdata): - ... self.msg = msg - ... self.msgdata = msgdata - ... return False - ... - ... def _doperiodic(self): - ... self.stop() - ... - ... def _snooze(self, filecnt): - ... return - - >>> runner = TestableRunner() - >>> switchboard = Switchboard(queue_directory) - -This qrunner doesn't do much except run once, storing the message and metadata -on instance variables. - - >>> msg = message_from_string(u"""\ - ... From: aperson@example.com - ... To: _xtest@example.com - ... - ... A test message. - ... """) - >>> filebase = switchboard.enqueue(msg, listname=mlist.fqdn_listname, - ... foo='yes', bar='no') - >>> runner.run() - >>> print runner.msg.as_string() - From: aperson@example.com - To: _xtest@example.com - - A test message. - - >>> sorted(runner.msgdata.items()) - [('_parsemsg', False), - ('bar', 'no'), ('foo', 'yes'), - ('lang', u'en'), ('listname', u'_xtest@example.com'), - ('received_time', ...), ('version', 3)] - -XXX More of the Runner API should be tested. diff --git a/Mailman/docs/switchboard.txt b/Mailman/docs/switchboard.txt deleted file mode 100644 index 299aba499..000000000 --- a/Mailman/docs/switchboard.txt +++ /dev/null @@ -1,149 +0,0 @@ -The switchboard -=============== - -The switchboard is subsystem that moves messages between queues. Each -instance of a switchboard is responsible for one queue directory. - - >>> msg = message_from_string(u"""\ - ... From: aperson@example.com - ... To: _xtest@example.com - ... - ... A test message. - ... """) - -Create a switchboard by giving its queue directory. - - >>> import os - >>> from Mailman.configuration import config - >>> queue_directory = os.path.join(config.QUEUE_DIR, 'test') - >>> from Mailman.queue import Switchboard - >>> switchboard = Switchboard(queue_directory) - >>> switchboard.queue_directory == queue_directory - True - -Here's a helper function for ensuring things work correctly. - - >>> def check_qfiles(): - ... files = {} - ... for qfile in os.listdir(queue_directory): - ... root, ext = os.path.splitext(qfile) - ... files[ext] = files.get(ext, 0) + 1 - ... return sorted(files.items()) - - -Enqueing and dequeing ---------------------- - -The message can be enqueued with metadata specified in the passed in -dictionary. - - >>> filebase = switchboard.enqueue(msg) - >>> check_qfiles() - [('.pck', 1)] - -To read the contents of a queue file, dequeue it. - - >>> msg, msgdata = switchboard.dequeue(filebase) - >>> print msg.as_string() - From: aperson@example.com - To: _xtest@example.com - - A test message. - - >>> sorted(msgdata.items()) - [('_parsemsg', False), ('received_time', ...), ('version', 3)] - >>> check_qfiles() - [('.bak', 1)] - -To complete the dequeing process, removing all traces of the message file, -finish it (without preservation). - - >>> switchboard.finish(filebase) - >>> check_qfiles() - [] - -When enqueing a file, you can provide additional metadata keys by using -keyword arguments. - - >>> filebase = switchboard.enqueue(msg, {'foo': 1}, bar=2) - >>> msg, msgdata = switchboard.dequeue(filebase) - >>> switchboard.finish(filebase) - >>> sorted(msgdata.items()) - [('_parsemsg', False), - ('bar', 2), ('foo', 1), - ('received_time', ...), ('version', 3)] - -Keyword arguments override keys from the metadata dictionary. - - >>> filebase = switchboard.enqueue(msg, {'foo': 1}, foo=2) - >>> msg, msgdata = switchboard.dequeue(filebase) - >>> switchboard.finish(filebase) - >>> sorted(msgdata.items()) - [('_parsemsg', False), - ('foo', 2), - ('received_time', ...), ('version', 3)] - - -Iterating over files --------------------- - -There are two ways to iterate over all the files in a switchboard's queue. -Normally, queue files end in .pck (for 'pickle') and the easiest way to -iterate over just these files is to use the .files attribute. - - >>> filebase_1 = switchboard.enqueue(msg, foo=1) - >>> filebase_2 = switchboard.enqueue(msg, foo=2) - >>> filebase_3 = switchboard.enqueue(msg, foo=3) - >>> filebases = sorted((filebase_1, filebase_2, filebase_3)) - >>> sorted(switchboard.files) == filebases - True - >>> check_qfiles() - [('.pck', 3)] - -You can also use the .get_files() method if you want to iterate over all the -file bases for some other extension. - - >>> for filebase in switchboard.get_files(): - ... msg, msgdata = switchboard.dequeue(filebase) - >>> bakfiles = sorted(switchboard.get_files('.bak')) - >>> bakfiles == filebases - True - >>> check_qfiles() - [('.bak', 3)] - >>> for filebase in switchboard.get_files('.bak'): - ... switchboard.finish(filebase) - >>> check_qfiles() - [] - - -Recovering files ----------------- - -Calling .dequeue() without calling .finish() leaves .bak backup files in -place. These can be recovered when the switchboard is instantiated. - - >>> filebase_1 = switchboard.enqueue(msg, foo=1) - >>> filebase_2 = switchboard.enqueue(msg, foo=2) - >>> filebase_3 = switchboard.enqueue(msg, foo=3) - >>> for filebase in switchboard.files: - ... msg, msgdata = switchboard.dequeue(filebase) - ... # Don't call .finish() - >>> check_qfiles() - [('.bak', 3)] - >>> switchboard_2 = Switchboard(queue_directory, recover=True) - >>> check_qfiles() - [('.pck', 3)] - -Clean up - - >>> for filebase in switchboard.files: - ... msg, msgdata = switchboard.dequeue(filebase) - ... switchboard.finish(filebase) - >>> check_qfiles() - [] - - -Queue slices ------------- - -XXX Add tests for queue slices. diff --git a/Mailman/inject.py b/Mailman/inject.py new file mode 100644 index 000000000..a17f8c7d1 --- /dev/null +++ b/Mailman/inject.py @@ -0,0 +1,34 @@ +# Copyright (C) 2001-2008 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +from Mailman.configuration import config +from Mailman.queue import Switchboard + + + +def inject(listname, msg, recips=None, qdir=None): + if qdir is None: + qdir = config.INQUEUE_DIR + queue = Switchboard(qdir) + kws = dict( + listname=listname, + tolist=True, + _plaintext=True, + ) + if recips is not None: + kws['recips'] = recips + queue.enqueue(msg, **kws) diff --git a/Mailman/interfaces/mailinglist.py b/Mailman/interfaces/mailinglist.py index 2d3811785..2c828a43c 100644 --- a/Mailman/interfaces/mailinglist.py +++ b/Mailman/interfaces/mailinglist.py @@ -82,7 +82,7 @@ class IMailingList(Interface): delivery is currently enabled. """) - noreply_address = Attribute( + no_reply_address = Attribute( """The address to which all messages will be immediately discarded, without prejudice or record. This address is specific to the ddomain, even though it's available on the IMailingListAddresses interface. diff --git a/Mailman/queue/docs/OVERVIEW.txt b/Mailman/queue/docs/OVERVIEW.txt new file mode 100644 index 000000000..643fa8a5c --- /dev/null +++ b/Mailman/queue/docs/OVERVIEW.txt @@ -0,0 +1,78 @@ +Alias overview +============== + +A typical Mailman list exposes nine aliases which point to seven different +wrapped scripts. E.g. for a list named `mylist', you'd have: + + mylist-bounces -> bounces + mylist-confirm -> confirm + mylist-join -> join (-subscribe is an alias) + mylist-leave -> leave (-unsubscribe is an alias) + mylist-owner -> owner + mylist -> post + mylist-request -> request + +-request, -join, and -leave are a robot addresses; their sole purpose is to +process emailed commands, although the latter two are hardcoded to +subscription and unsubscription requests. -bounces is the automated bounce +processor, and all messages to list members have their return address set to +-bounces. If the bounce processor fails to extract a bouncing member address, +it can optionally forward the message on to the list owners. + +-owner is for reaching a human operator with minimal list interaction (i.e. no +bounce processing). -confirm is another robot address which processes replies +to VERP-like confirmation notices. + +So delivery flow of messages look like this: + + joerandom ---> mylist ---> list members + | | + | |[bounces] + | mylist-bounces <---+ <-------------------------------+ + | | | + | +--->[internal bounce processing] | + | ^ | | + | | | [bounce found] | + | [bounces *] +--->[register and discard] | + | | | | | + | | | |[*] | + | [list owners] |[no bounce found] | | + | ^ | | | + | | | | | + +-------> mylist-owner <--------+ | | + | | | + | data/owner-bounces.mbox <--[site list] <---+ | + | | + +-------> mylist-join--+ | + | | | + +------> mylist-leave--+ | + | | | + | v | + +-------> mylist-request | + | | | + | +---> [command processor] | + | | | + +-----> mylist-confirm ----> +---> joerandom | + | | + |[bounces] | + +----------------------+ + +A person can send an email to the list address (for posting), the -owner +address (to reach the human operator), or the -confirm, -join, -leave, and +-request mailbots. Message to the list address are then forwarded on to the +list membership, with bounces directed to the -bounces address. + +[*] Messages sent to the -owner address are forwarded on to the list +owner/moderators. All -owner destined messages have their bounces directed to +the site list -bounces address, regardless of whether a human sent the message +or the message was crafted internally. The intention here is that the site +owners want to be notified when one of their list owners' addresses starts +bouncing (yes, the will be automated in a future release). + +Any messages to site owners has their bounces directed to a special +"loop-killer" address, which just dumps the message into +data/owners-bounces.mbox. + +Finally, message to any of the mailbots causes the requested action to be +performed. Results notifications are sent to the author of the message, which +all bounces pointing back to the -bounces address. diff --git a/Mailman/queue/docs/incoming.txt b/Mailman/queue/docs/incoming.txt new file mode 100644 index 000000000..12ff3d3d1 --- /dev/null +++ b/Mailman/queue/docs/incoming.txt @@ -0,0 +1,67 @@ +The incoming queue runner +========================= + +This runner's sole purpose in life is to decide the disposition of the +message. It can either be accepted for delivery, rejected (i.e. bounced), +held for moderator approval, or discarded. + +The runner operates by processing chains on a message/metadata pair in the +context of a mailing list. Each mailing list may have a 'start chain' where +processing begins, with a global default. This chain is processed with the +message eventually ending up in one of the four disposition states described +above. + + >>> from Mailman.app.lifecycle import create_list + >>> mlist = create_list(u'_xtest@example.com') + >>> mlist.start_chain + u'built-in' + +We have a message that is going to be sent to the mailing list. This message +is so perfectly fine for posting that it will be accepted and forward to the +prep queue. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... Subject: My first post + ... Message-ID: + ... + ... First post! + ... """) + +Normally, the upstream mail server would drop the message in the incoming +queue, but this is an effective simulation. + + >>> from Mailman.inject import inject + >>> inject(u'_xtest@example.com', msg) + +The incoming queue runner runs until it is empty. + + >>> from Mailman.queue.incoming import IncomingRunner + >>> from Mailman.tests.helpers import make_testable_runner + >>> incoming = make_testable_runner(IncomingRunner) + >>> incoming.run() + +And now the message is in the prep queue. + + >>> from Mailman.configuration import config + >>> from Mailman.queue import Switchboard + >>> prep_queue = Switchboard(config.PREPQUEUE_DIR) + >>> len(prep_queue.files) + 1 + >>> from Mailman.tests.helpers import get_queue_messages + >>> item = get_queue_messages(prep_queue)[0] + >>> print item.msg.as_string() + From: aperson@example.com + To: _xtest@example.com + Subject: My first post + Message-ID: + X-Mailman-Rule-Misses: approved; emergency; loop; administrivia; + implicit-dest; + max-recipients; max-size; news-moderation; no-subject; + suspicious-header + + First post! + + >>> sorted(item.msgdata.items()) + [...('envsender', u'noreply@example.com')...('tolist', True)...] diff --git a/Mailman/queue/docs/news.txt b/Mailman/queue/docs/news.txt new file mode 100644 index 000000000..bc6619f50 --- /dev/null +++ b/Mailman/queue/docs/news.txt @@ -0,0 +1,158 @@ +The news runner +=============== + +The news runner is the queue runner that gateways mailing list messages to an +NNTP newsgroup. One of the most important things this runner does is prepare +the message for Usenet (yes, I know that NNTP is not Usenet, but this runner +was originally written to gate to Usenet, which has its own rules). + + >>> from Mailman.configuration import config + >>> from Mailman.queue.news import prepare_message + >>> mlist = config.db.list_manager.create(u'_xtest@example.com') + >>> mlist.linked_newsgroup = u'comp.lang.python' + +Some NNTP servers such as INN reject messages containing a set of prohibited +headers, so one of the things that the news runner does is remove these +prohibited headers. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... NNTP-Posting-Host: news.example.com + ... NNTP-Posting-Date: today + ... X-Trace: blah blah + ... X-Complaints-To: abuse@dom.ain + ... Xref: blah blah + ... Xref: blah blah + ... Date-Received: yesterday + ... Posted: tomorrow + ... Posting-Version: 99.99 + ... Relay-Version: 88.88 + ... Received: blah blah + ... + ... A message + ... """) + >>> msgdata = {} + >>> prepare_message(mlist, msg, msgdata) + >>> msgdata['prepped'] + True + >>> print msg.as_string() + From: aperson@example.com + To: _xtest@example.com + Newsgroups: comp.lang.python + Message-ID: ... + Lines: 1 + + A message + + +Some NNTP servers will reject messages where certain headers are duplicated, +so the news runner must collapse or move these duplicate headers to an +X-Original-* header that the news server doesn't care about. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... To: two@example.com + ... Cc: three@example.com + ... Cc: four@example.com + ... Cc: five@example.com + ... Content-Transfer-Encoding: yes + ... Content-Transfer-Encoding: no + ... Content-Transfer-Encoding: maybe + ... + ... A message + ... """) + >>> msgdata = {} + >>> prepare_message(mlist, msg, msgdata) + >>> msgdata['prepped'] + True + >>> print msg.as_string() + From: aperson@example.com + Newsgroups: comp.lang.python + Message-ID: ... + Lines: 1 + To: _xtest@example.com + X-Original-To: two@example.com + CC: three@example.com + X-Original-CC: four@example.com + X-Original-CC: five@example.com + Content-Transfer-Encoding: yes + X-Original-Content-Transfer-Encoding: no + X-Original-Content-Transfer-Encoding: maybe + + A message + + +But if no headers are duplicated, then the news runner doesn't need to modify +the message. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... Cc: someother@example.com + ... Content-Transfer-Encoding: yes + ... + ... A message + ... """) + >>> msgdata = {} + >>> prepare_message(mlist, msg, msgdata) + >>> msgdata['prepped'] + True + >>> print msg.as_string() + From: aperson@example.com + To: _xtest@example.com + Cc: someother@example.com + Content-Transfer-Encoding: yes + Newsgroups: comp.lang.python + Message-ID: ... + Lines: 1 + + A message + + + +Newsgroup moderation +-------------------- + +When the newsgroup is moderated, an Approved: header with the list's posting +address is added for the benefit of the Usenet system. + + >>> from Mailman.interfaces import NewsModeration + >>> mlist.news_moderation = NewsModeration.open_moderated + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... Approved: this gets deleted + ... + ... """) + >>> prepare_message(mlist, msg, {}) + >>> msg['approved'] + u'_xtest@example.com' + + >>> mlist.news_moderation = NewsModeration.moderated + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... Approved: this gets deleted + ... + ... """) + >>> prepare_message(mlist, msg, {}) + >>> msg['approved'] + u'_xtest@example.com' + +But if the newsgroup is not moderated, the Approved: header is not chnaged. + + >>> mlist.news_moderation = NewsModeration.none + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... Approved: this doesn't get deleted + ... + ... """) + >>> prepare_message(mlist, msg, {}) + >>> msg['approved'] + u"this doesn't get deleted" + + +XXX More of the NewsRunner should be tested. diff --git a/Mailman/queue/docs/outgoing.txt b/Mailman/queue/docs/outgoing.txt new file mode 100644 index 000000000..ba2c6430b --- /dev/null +++ b/Mailman/queue/docs/outgoing.txt @@ -0,0 +1,155 @@ +The outgoing handler +==================== + +Mailman's outgoing queue is used as the wrapper around SMTP delivery to the +upstream mail server. The ToOutgoing handler does little more than drop the +message into the outgoing queue, after calculating whether the message should +be VERP'd or not. VERP means Variable Envelope Return Path; we're using that +term somewhat incorrectly, but within the spirit of the standard, which +basically describes how to encode the recipient's address in the originator +headers for unambigous bounce processing. + + >>> from Mailman.Handlers.ToOutgoing import process + >>> from Mailman.queue import Switchboard + >>> from Mailman.configuration import config + >>> mlist = config.db.list_manager.create(u'_xtest@example.com') + >>> switchboard = Switchboard(config.OUTQUEUE_DIR) + + >>> def queue_size(): + ... size = len(switchboard.files) + ... for filebase in switchboard.files: + ... msg, msgdata = switchboard.dequeue(filebase) + ... switchboard.finish(filebase) + ... return size + +Craft a message destined for the outgoing queue. Include some random metadata +as if this message had passed through some other handlers. + + >>> msg = message_from_string("""\ + ... Subject: Here is a message + ... + ... Something of great import. + ... """) + +When certain conditions are met, the message will be VERP'd. For example, if +the message metadata already has a VERP key, this message will be VERP'd. + + >>> msgdata = dict(foo=1, bar=2, verp=True) + >>> process(mlist, msg, msgdata) + >>> print msg.as_string() + Subject: Here is a message + + Something of great import. + >>> msgdata['verp'] + True + +While the queued message will not be changed, the queued metadata will have an +additional key set: the mailing list name. + + >>> filebase = switchboard.files[0] + >>> qmsg, qmsgdata = switchboard.dequeue(filebase) + >>> switchboard.finish(filebase) + >>> print qmsg.as_string() + Subject: Here is a message + + Something of great import. + >>> sorted(qmsgdata.items()) + [('_parsemsg', False), + ('bar', 2), ('foo', 1), + ('listname', u'_xtest@example.com'), + ('received_time', ...), + ('verp', True), ('version', 3)] + >>> queue_size() + 0 + +If the list is set to personalize deliveries, and the global configuration +option to VERP personalized deliveries is set, then the message will be +VERP'd. + + # Save the original value for clean up. + >>> verp_personalized_delivieries = config.VERP_PERSONALIZED_DELIVERIES + >>> config.VERP_PERSONALIZED_DELIVERIES = True + >>> from Mailman.interfaces import Personalization + >>> mlist.personalize = Personalization.individual + >>> msgdata = dict(foo=1, bar=2) + >>> process(mlist, msg, msgdata) + >>> msgdata['verp'] + True + >>> queue_size() + 1 + +However, if the global configuration variable prohibits VERP'ing, even +personalized lists will not VERP. + + >>> config.VERP_PERSONALIZED_DELIVERIES = False + >>> msgdata = dict(foo=1, bar=2) + >>> process(mlist, msg, msgdata) + >>> print msgdata.get('verp') + None + >>> queue_size() + 1 + +If the list is not personalized, then the message may still be VERP'd based on +the global configuration variable VERP_DELIVERY_INTERVAL. This variable tells +Mailman how often to VERP even non-personalized mailing lists. It can be set +to zero, which means non-personalized messages will never be VERP'd. + + # Save the original value for clean up. + >>> verp_delivery_interval = config.VERP_DELIVERY_INTERVAL + >>> config.VERP_DELIVERY_INTERVAL = 0 + >>> mlist.personalize = Personalization.none + >>> msgdata = dict(foo=1, bar=2) + >>> process(mlist, msg, msgdata) + >>> print msgdata.get('verp') + None + >>> queue_size() + 1 + +If the interval is set to 1, then every message will be VERP'd. + + >>> config.VERP_DELIVERY_INTERVAL = 1 + >>> for i in range(10): + ... msgdata = dict(foo=1, bar=2) + ... process(mlist, msg, msgdata) + ... print i, msgdata['verp'] + 0 True + 1 True + 2 True + 3 True + 4 True + 5 True + 6 True + 7 True + 8 True + 9 True + >>> queue_size() + 10 + +If the interval is set to some other number, then one out of that many posts +will be VERP'd. + + >>> config.VERP_DELIVERY_INTERVAL = 3 + >>> for i in range(10): + ... mlist.post_id = i + ... msgdata = dict(foo=1, bar=2) + ... process(mlist, msg, msgdata) + ... print i, msgdata.get('verp', False) + 0 True + 1 False + 2 False + 3 True + 4 False + 5 False + 6 True + 7 False + 8 False + 9 True + >>> queue_size() + 10 + + +Clean up +======== + + >>> config.VERP_PERSONALIZED_DELIVERIES = verp_personalized_delivieries + >>> config.VERP_DELIVERY_INTERVAL = verp_delivery_interval diff --git a/Mailman/queue/docs/runner.txt b/Mailman/queue/docs/runner.txt new file mode 100644 index 000000000..5e5a88d8c --- /dev/null +++ b/Mailman/queue/docs/runner.txt @@ -0,0 +1,70 @@ +Queue runners +============= + +The queue runners (qrunner) are the processes that move messages around the +Mailman system. Each qrunner is responsible for a slice of the hash space in +a queue directory. It processes all the files in its slice, sleeps a little +while, then wakes up and runs through its queue files again. + + +Basic architecture +------------------ + +The basic architecture of qrunner is implemented in the base class that all +runners inherit from. This base class implements a .run() method that runs +continuously in a loop until the .stop() method is called. + + >>> import os + >>> from Mailman.queue import Runner, Switchboard + >>> from Mailman.configuration import config + >>> mlist = config.db.list_manager.create(u'_xtest@example.com') + >>> mlist.preferred_language = u'en' + +Here is a very simple derived qrunner class. The class attribute QDIR tells +the qrunner which queue directory it is responsible for. Derived classes +should also implement various methods to provide the special functionality. +This is about as simple as a qrunner can be. + + >>> queue_directory = os.path.join(config.QUEUE_DIR, 'test') + >>> class TestableRunner(Runner): + ... QDIR = queue_directory + ... + ... def _dispose(self, mlist, msg, msgdata): + ... self.msg = msg + ... self.msgdata = msgdata + ... return False + ... + ... def _doperiodic(self): + ... self.stop() + ... + ... def _snooze(self, filecnt): + ... return + + >>> runner = TestableRunner() + >>> switchboard = Switchboard(queue_directory) + +This qrunner doesn't do much except run once, storing the message and metadata +on instance variables. + + >>> msg = message_from_string(u"""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... + ... A test message. + ... """) + >>> filebase = switchboard.enqueue(msg, listname=mlist.fqdn_listname, + ... foo='yes', bar='no') + >>> runner.run() + >>> print runner.msg.as_string() + From: aperson@example.com + To: _xtest@example.com + + A test message. + + >>> sorted(runner.msgdata.items()) + [('_parsemsg', False), + ('bar', 'no'), ('foo', 'yes'), + ('lang', u'en'), ('listname', u'_xtest@example.com'), + ('received_time', ...), ('version', 3)] + +XXX More of the Runner API should be tested. diff --git a/Mailman/queue/docs/switchboard.txt b/Mailman/queue/docs/switchboard.txt new file mode 100644 index 000000000..299aba499 --- /dev/null +++ b/Mailman/queue/docs/switchboard.txt @@ -0,0 +1,149 @@ +The switchboard +=============== + +The switchboard is subsystem that moves messages between queues. Each +instance of a switchboard is responsible for one queue directory. + + >>> msg = message_from_string(u"""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... + ... A test message. + ... """) + +Create a switchboard by giving its queue directory. + + >>> import os + >>> from Mailman.configuration import config + >>> queue_directory = os.path.join(config.QUEUE_DIR, 'test') + >>> from Mailman.queue import Switchboard + >>> switchboard = Switchboard(queue_directory) + >>> switchboard.queue_directory == queue_directory + True + +Here's a helper function for ensuring things work correctly. + + >>> def check_qfiles(): + ... files = {} + ... for qfile in os.listdir(queue_directory): + ... root, ext = os.path.splitext(qfile) + ... files[ext] = files.get(ext, 0) + 1 + ... return sorted(files.items()) + + +Enqueing and dequeing +--------------------- + +The message can be enqueued with metadata specified in the passed in +dictionary. + + >>> filebase = switchboard.enqueue(msg) + >>> check_qfiles() + [('.pck', 1)] + +To read the contents of a queue file, dequeue it. + + >>> msg, msgdata = switchboard.dequeue(filebase) + >>> print msg.as_string() + From: aperson@example.com + To: _xtest@example.com + + A test message. + + >>> sorted(msgdata.items()) + [('_parsemsg', False), ('received_time', ...), ('version', 3)] + >>> check_qfiles() + [('.bak', 1)] + +To complete the dequeing process, removing all traces of the message file, +finish it (without preservation). + + >>> switchboard.finish(filebase) + >>> check_qfiles() + [] + +When enqueing a file, you can provide additional metadata keys by using +keyword arguments. + + >>> filebase = switchboard.enqueue(msg, {'foo': 1}, bar=2) + >>> msg, msgdata = switchboard.dequeue(filebase) + >>> switchboard.finish(filebase) + >>> sorted(msgdata.items()) + [('_parsemsg', False), + ('bar', 2), ('foo', 1), + ('received_time', ...), ('version', 3)] + +Keyword arguments override keys from the metadata dictionary. + + >>> filebase = switchboard.enqueue(msg, {'foo': 1}, foo=2) + >>> msg, msgdata = switchboard.dequeue(filebase) + >>> switchboard.finish(filebase) + >>> sorted(msgdata.items()) + [('_parsemsg', False), + ('foo', 2), + ('received_time', ...), ('version', 3)] + + +Iterating over files +-------------------- + +There are two ways to iterate over all the files in a switchboard's queue. +Normally, queue files end in .pck (for 'pickle') and the easiest way to +iterate over just these files is to use the .files attribute. + + >>> filebase_1 = switchboard.enqueue(msg, foo=1) + >>> filebase_2 = switchboard.enqueue(msg, foo=2) + >>> filebase_3 = switchboard.enqueue(msg, foo=3) + >>> filebases = sorted((filebase_1, filebase_2, filebase_3)) + >>> sorted(switchboard.files) == filebases + True + >>> check_qfiles() + [('.pck', 3)] + +You can also use the .get_files() method if you want to iterate over all the +file bases for some other extension. + + >>> for filebase in switchboard.get_files(): + ... msg, msgdata = switchboard.dequeue(filebase) + >>> bakfiles = sorted(switchboard.get_files('.bak')) + >>> bakfiles == filebases + True + >>> check_qfiles() + [('.bak', 3)] + >>> for filebase in switchboard.get_files('.bak'): + ... switchboard.finish(filebase) + >>> check_qfiles() + [] + + +Recovering files +---------------- + +Calling .dequeue() without calling .finish() leaves .bak backup files in +place. These can be recovered when the switchboard is instantiated. + + >>> filebase_1 = switchboard.enqueue(msg, foo=1) + >>> filebase_2 = switchboard.enqueue(msg, foo=2) + >>> filebase_3 = switchboard.enqueue(msg, foo=3) + >>> for filebase in switchboard.files: + ... msg, msgdata = switchboard.dequeue(filebase) + ... # Don't call .finish() + >>> check_qfiles() + [('.bak', 3)] + >>> switchboard_2 = Switchboard(queue_directory, recover=True) + >>> check_qfiles() + [('.pck', 3)] + +Clean up + + >>> for filebase in switchboard.files: + ... msg, msgdata = switchboard.dequeue(filebase) + ... switchboard.finish(filebase) + >>> check_qfiles() + [] + + +Queue slices +------------ + +XXX Add tests for queue slices. diff --git a/Mailman/queue/incoming.py b/Mailman/queue/incoming.py index 6118a7ca0..649ce2213 100644 --- a/Mailman/queue/incoming.py +++ b/Mailman/queue/incoming.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2007 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2008 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -12,102 +12,26 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. -"""Incoming queue runner.""" - -# A typical Mailman list exposes nine aliases which point to seven different -# wrapped scripts. E.g. for a list named `mylist', you'd have: -# -# mylist-bounces -> bounces (-admin is a deprecated alias) -# mylist-confirm -> confirm -# mylist-join -> join (-subscribe is an alias) -# mylist-leave -> leave (-unsubscribe is an alias) -# mylist-owner -> owner -# mylist -> post -# mylist-request -> request -# -# -request, -join, and -leave are a robot addresses; their sole purpose is to -# process emailed commands in a Majordomo-like fashion (although the latter -# two are hardcoded to subscription and unsubscription requests). -bounces is -# the automated bounce processor, and all messages to list members have their -# return address set to -bounces. If the bounce processor fails to extract a -# bouncing member address, it can optionally forward the message on to the -# list owners. -# -# -owner is for reaching a human operator with minimal list interaction -# (i.e. no bounce processing). -confirm is another robot address which -# processes replies to VERP-like confirmation notices. -# -# So delivery flow of messages look like this: -# -# joerandom ---> mylist ---> list members -# | | -# | |[bounces] -# | mylist-bounces <---+ <-------------------------------+ -# | | | -# | +--->[internal bounce processing] | -# | ^ | | -# | | | [bounce found] | -# | [bounces *] +--->[register and discard] | -# | | | | | -# | | | |[*] | -# | [list owners] |[no bounce found] | | -# | ^ | | | -# | | | | | -# +-------> mylist-owner <--------+ | | -# | | | -# | data/owner-bounces.mbox <--[site list] <---+ | -# | | -# +-------> mylist-join--+ | -# | | | -# +------> mylist-leave--+ | -# | | | -# | v | -# +-------> mylist-request | -# | | | -# | +---> [command processor] | -# | | | -# +-----> mylist-confirm ----> +---> joerandom | -# | | -# |[bounces] | -# +----------------------+ -# -# A person can send an email to the list address (for posting), the -owner -# address (to reach the human operator), or the -confirm, -join, -leave, and -# -request mailbots. Message to the list address are then forwarded on to the -# list membership, with bounces directed to the -bounces address. -# -# [*] Messages sent to the -owner address are forwarded on to the list -# owner/moderators. All -owner destined messages have their bounces directed -# to the site list -bounces address, regardless of whether a human sent the -# message or the message was crafted internally. The intention here is that -# the site owners want to be notified when one of their list owners' addresses -# starts bouncing (yes, the will be automated in a future release). -# -# Any messages to site owners has their bounces directed to a special -# "loop-killer" address, which just dumps the message into -# data/owners-bounces.mbox. -# -# Finally, message to any of the mailbots causes the requested action to be -# performed. Results notifications are sent to the author of the message, -# which all bounces pointing back to the -bounces address. +"""Incoming queue runner. +This runner's sole purpose in life is to decide the disposition of the +message. It can either be accepted for delivery, rejected (i.e. bounced), +held for moderator approval, or discarded. - -import os -import sys -import logging +When accepted, the message is forwarded on to the `prep queue` where it is +prepared for delivery. Rejections, discards, and holds are processed +immediately. +""" -from cStringIO import StringIO -from Mailman import Errors + +from Mailman.app.chains import process from Mailman.configuration import config from Mailman.queue import Runner -log = logging.getLogger('mailman.error') -vlog = logging.getLogger('mailman.vette') - class IncomingRunner(Runner): @@ -115,59 +39,8 @@ class IncomingRunner(Runner): def _dispose(self, mlist, msg, msgdata): if msgdata.get('envsender') is None: - msg['envsender'] = mlist.no_reply_address - # Process the message through a handler pipeline. The handler - # pipeline can actually come from one of three places: the message - # metadata, the mlist, or the global pipeline. - # - # If a message was requeued due to an uncaught exception, its metadata - # will contain the retry pipeline. Use this above all else. - # Otherwise, if the mlist has a `pipeline' attribute, it should be - # used. Final fallback is the global pipeline. - pipeline = self._get_pipeline(mlist, msg, msgdata) - msgdata['pipeline'] = pipeline - more = self._dopipeline(mlist, msg, msgdata, pipeline) - if not more: - del msgdata['pipeline'] - config.db.commit() - return more - - # Overridable - def _get_pipeline(self, mlist, msg, msgdata): - # We must return a copy of the list, otherwise, the first message that - # flows through the pipeline will empty it out! - return msgdata.get('pipeline', - getattr(mlist, 'pipeline', - config.GLOBAL_PIPELINE))[:] - - def _dopipeline(self, mlist, msg, msgdata, pipeline): - while pipeline: - handler = pipeline.pop(0) - modname = 'Mailman.Handlers.' + handler - __import__(modname) - try: - pid = os.getpid() - sys.modules[modname].process(mlist, msg, msgdata) - # Failsafe -- a child may have leaked through. - if pid <> os.getpid(): - log.error('child process leaked thru: %s', modname) - os._exit(1) - except Errors.DiscardMessage: - # Throw the message away; we need do nothing else with it. - vlog.info('Message discarded, msgid: %s', - msg.get('message-id', 'n/a')) - return 0 - except Errors.HoldMessage: - # Let the approval process take it from here. The message no - # longer needs to be queued. - return 0 - except Errors.RejectMessage, e: - mlist.bounce_message(msg, e) - return 0 - except: - # Push this pipeline module back on the stack, then re-raise - # the exception. - pipeline.insert(0, handler) - raise - # We've successfully completed handling of this message - return 0 + msgdata['envsender'] = mlist.no_reply_address + # Process the message through the mailing list's start chain. + process(mlist, msg, msgdata, mlist.start_chain) + # Do not keep this message queued. + return False diff --git a/Mailman/queue/tests/__init__.py b/Mailman/queue/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/Mailman/tests/smtplistener.py b/Mailman/tests/smtplistener.py index 565772b1d..977726247 100644 --- a/Mailman/tests/smtplistener.py +++ b/Mailman/tests/smtplistener.py @@ -1,4 +1,4 @@ -# Copyright (C) 2007 by the Free Software Foundation, Inc. +# Copyright (C) 2007-2008 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -15,8 +15,11 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, # USA. +"""A test SMTP listener.""" + import sys import smtpd +import signal import mailbox import asyncore import optparse @@ -29,34 +32,68 @@ DEFAULT_PORT = 9025 class Channel(smtpd.SMTPChannel): - def smtp_EXIT(self, arg): - raise asyncore.ExitNow + """A channel that can reset the mailbox.""" + + def __init__(self, server, conn, addr): + smtpd.SMTPChannel.__init__(self, server, conn, addr) + # Stash this here since the subclass uses private attributes. :( + self._server = server + + def smtp_RSET(self, arg): + """Respond to RSET and clear the mailbox.""" + self._server.clear_mailbox() + smtpd.SMTPChannel.smtp_RSET(self, arg) + def send(self, data): + """Silence the bloody asynchat/asyncore broken pipe errors!""" + try: + return smtpd.SMTPChannel.send(self, data) + except socket.error: + # Nothing here can affect the outcome, and these messages are just + # plain annoying! So ignore them. + pass + + class Server(smtpd.SMTPServer): - def __init__(self, localaddr, mboxfile): + """An SMTP server that stores messages to a mailbox.""" + + def __init__(self, localaddr, mailbox_path): smtpd.SMTPServer.__init__(self, localaddr, None) - self._mbox = mailbox.mbox(mboxfile) + self._mailbox = mailbox.Maildir(mailbox_path) def handle_accept(self): + """Handle connections by creating our own Channel object.""" conn, addr = self.accept() Channel(self, conn, addr) def process_message(self, peer, mailfrom, rcpttos, data): + """Process a message by adding it to the mailbox.""" msg = message_from_string(data) msg['X-Peer'] = peer msg['X-MailFrom'] = mailfrom msg['X-RcptTo'] = COMMASPACE.join(rcpttos) - self._mbox.add(msg) + self._mailbox.add(msg) + self._mailbox.clean() - def close(self): - self._mbox.flush() - self._mbox.close() + + +def handle_signal(*ignore): + """Handle signal sent by parent to kill the process.""" + asyncore.socket_map.clear() def main(): - parser = optparse.OptionParser(usage='%prog mboxfile') + parser = optparse.OptionParser(usage="""\ +%prog [options] mboxfile + +This starts a process listening on a specified host and port (by default +localhost:9025) for SMTP conversations. All messages this process receives +are stored in a specified mbox file for the parent process to investigate. + +This SMTP server responds to RSET commands by clearing the mbox file. +""") parser.add_option('-a', '--address', type='string', default=None, help='host:port to listen on') @@ -77,12 +114,14 @@ def main(): host, port = opts.address.split(':', 1) port = int(port) + # Catch the parent's exit signal, and also C-c. + signal.signal(signal.SIGTERM, handle_signal) + signal.signal(signal.SIGINT, handle_signal) + server = Server((host, port), mboxfile) - try: - asyncore.loop() - except asyncore.ExitNow: - asyncore.close_all() - server.close() + asyncore.loop() + asyncore.close_all() + server.close() return 0 -- cgit v1.2.3-70-g09d2 From 4823801716b1bf1711d63b649b0fafd6acd30821 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Sat, 2 Feb 2008 13:47:23 -0500 Subject: Fleshed out the doctest for the new incoming queue runner. Added a Mailman.tests.helpers module for some commonly used stuff (although test refactoring hasn't yet happened). Give Mailman 3.0a1 a code name. --- Mailman/queue/docs/incoming.txt | 131 ++++++++++++++++++++++++++++++++++++++++ Mailman/tests/helpers.py | 64 ++++++++++++++++++++ docs/NEWS.txt | 5 +- 3 files changed, 198 insertions(+), 2 deletions(-) create mode 100644 Mailman/tests/helpers.py (limited to 'Mailman/queue') diff --git a/Mailman/queue/docs/incoming.txt b/Mailman/queue/docs/incoming.txt index 12ff3d3d1..04c0cfa04 100644 --- a/Mailman/queue/docs/incoming.txt +++ b/Mailman/queue/docs/incoming.txt @@ -16,6 +16,10 @@ above. >>> mlist.start_chain u'built-in' + +Accepted messages +----------------- + We have a message that is going to be sent to the mailing list. This message is so perfectly fine for posting that it will be accepted and forward to the prep queue. @@ -49,6 +53,9 @@ And now the message is in the prep queue. >>> prep_queue = Switchboard(config.PREPQUEUE_DIR) >>> len(prep_queue.files) 1 + >>> incoming_queue = Switchboard(config.INQUEUE_DIR) + >>> len(incoming_queue.files) + 0 >>> from Mailman.tests.helpers import get_queue_messages >>> item = get_queue_messages(prep_queue)[0] >>> print item.msg.as_string() @@ -65,3 +72,127 @@ And now the message is in the prep queue. >>> sorted(item.msgdata.items()) [...('envsender', u'noreply@example.com')...('tolist', True)...] + + +Held messages +------------- + +The list moderator sets the emergency flag on the mailing list. The built-in +chain will now hold all posted messages, so nothing will show up in the prep +queue. + + # XXX This checks the vette log file because there is no other evidence + # that this chain has done anything. + >>> import os + >>> fp = open(os.path.join(config.LOG_DIR, 'vette')) + >>> fp.seek(0, 2) + + >>> mlist.emergency = True + >>> mlist.web_page_url = u'http://archives.example.com/' + >>> inject(u'_xtest@example.com', msg) + >>> file_pos = fp.tell() + >>> incoming.run() + >>> len(prep_queue.files) + 0 + >>> len(incoming_queue.files) + 0 + >>> fp.seek(file_pos) + >>> print 'LOG:', fp.read() + LOG: ... HOLD: _xtest@example.com post from aperson@example.com held, + message-id=: n/a + + + >>> mlist.emergency = False + + +Discarded messages +------------------ + +Another possibility is that the message would get immediately discarded. The +built-in chain does not have such a disposition by default, so let's craft a +new chain and set it as the mailing list's start chain. + + >>> from Mailman.chains.base import Chain, Link + >>> from Mailman.interfaces import LinkAction + >>> truth_rule = config.rules['truth'] + >>> discard_chain = config.chains['discard'] + >>> test_chain = Chain('always-discard', u'Testing discards') + >>> link = Link(truth_rule, LinkAction.jump, discard_chain) + >>> test_chain.append_link(link) + >>> mlist.start_chain = u'always-discard' + + >>> inject(u'_xtest@example.com', msg) + >>> file_pos = fp.tell() + >>> incoming.run() + >>> len(prep_queue.files) + 0 + >>> len(incoming_queue.files) + 0 + >>> fp.seek(file_pos) + >>> print 'LOG:', fp.read() + LOG: ... DISCARD: + + + >>> del config.chains['always-discard'] + + +Rejected messages +----------------- + +Similar to discarded messages, a message can be rejected, or bounced back to +the original sender. Again, the built-in chain doesn't support this so we'll +just create a new chain that does. + + >>> reject_chain = config.chains['reject'] + >>> test_chain = Chain('always-reject', u'Testing rejections') + >>> link = Link(truth_rule, LinkAction.jump, reject_chain) + >>> test_chain.append_link(link) + >>> mlist.start_chain = u'always-reject' + +The virgin queue needs to be cleared out due to artifacts from the previous +tests above. + + >>> virgin_queue = Switchboard(config.VIRGINQUEUE_DIR) + >>> ignore = get_queue_messages(virgin_queue) + + >>> inject(u'_xtest@example.com', msg) + >>> file_pos = fp.tell() + >>> incoming.run() + >>> len(prep_queue.files) + 0 + >>> len(incoming_queue.files) + 0 + + >>> len(virgin_queue.files) + 1 + >>> item = get_queue_messages(virgin_queue)[0] + >>> print item.msg.as_string() + Subject: My first post + From: _xtest-owner@example.com + To: aperson@example.com + ... + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + + [No bounce details are available] + ... + Content-Type: message/rfc822 + MIME-Version: 1.0 + + From: aperson@example.com + To: _xtest@example.com + Subject: My first post + Message-ID: + + First post! + + ... + >>> sorted(item.msgdata.items()) + [...('recips', [u'aperson@example.com'])...] + >>> fp.seek(file_pos) + >>> print 'LOG:', fp.read() + LOG: ... REJECT: + + + >>> del config.chains['always-reject'] diff --git a/Mailman/tests/helpers.py b/Mailman/tests/helpers.py new file mode 100644 index 000000000..1b24f11e6 --- /dev/null +++ b/Mailman/tests/helpers.py @@ -0,0 +1,64 @@ +# Copyright (C) 2008 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +"""Various test helpers.""" + +__metaclass__ = type +__all__ = [ + 'get_queue_messages', + 'make_testable_runner', + ] + + + +def make_testable_runner(runner_class): + """Create a queue runner that runs until its queue is empty. + + :param runner_class: An IRunner + :return: A runner instance. + """ + + class EmptyingRunner(runner_class): + """Stop processing when the queue is empty.""" + + def _doperiodic(self): + """Stop when the queue is empty.""" + self._stop = (len(self._switchboard.files) == 0) + + return EmptyingRunner() + + + +class _Bag: + def __init__(self, **kws): + for key, value in kws.items(): + setattr(self, key, value) + + +def get_queue_messages(queue): + """Return and clear all the messages in the given queue. + + :param queue: An ISwitchboard + :return: A list of 2-tuples where each item contains the message and + message metadata. + """ + messages = [] + for filebase in queue.files: + msg, msgdata = queue.dequeue(filebase) + messages.append(_Bag(msg=msg, msgdata=msgdata)) + queue.finish(filebase) + return messages diff --git a/docs/NEWS.txt b/docs/NEWS.txt index 2cc28963e..d0ebf99d7 100644 --- a/docs/NEWS.txt +++ b/docs/NEWS.txt @@ -1,10 +1,11 @@ Mailman - The GNU Mailing List Management System -Copyright (C) 1998-2007 by the Free Software Foundation, Inc. +Copyright (C) 1998-2008 by the Free Software Foundation, Inc. 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA Here is a history of user visible changes to Mailman. -3.0 alpha 1 (XX-XXX-200X) +3.0 alpha 1 -- "Leave That Thing Alone" +(XX-XXX-200X) User visible changes -- cgit v1.2.3-70-g09d2