diff options
| -rw-r--r-- | Mailman/Defaults.py.in | 8 | ||||
| -rw-r--r-- | Mailman/Queue/Runner.py | 29 | ||||
| -rw-r--r-- | Mailman/Queue/Switchboard.py | 3 | ||||
| -rw-r--r-- | Mailman/Queue/tests/test_runners.py | 163 | ||||
| -rw-r--r-- | Mailman/docs/acknowledge.txt | 24 | ||||
| -rw-r--r-- | Mailman/docs/news-runner.txt | 119 | ||||
| -rw-r--r-- | Mailman/docs/replybot.txt | 34 | ||||
| -rw-r--r-- | Mailman/docs/runner.txt | 76 | ||||
| -rw-r--r-- | Mailman/docs/switchboard.txt | 8 |
9 files changed, 236 insertions, 228 deletions
diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in index 705191b2f..7c6242521 100644 --- a/Mailman/Defaults.py.in +++ b/Mailman/Defaults.py.in @@ -469,10 +469,10 @@ NNTP_REMOVE_HEADERS = ['nntp-posting-host', 'nntp-posting-date', 'x-trace', # original message. Any second and subsequent headers are rewritten to the # second named header (case preserved). NNTP_REWRITE_DUPLICATE_HEADERS = [ - ('to', 'X-Original-To'), - ('cc', 'X-Original-Cc'), - ('content-transfer-encoding', 'X-Original-Content-Transfer-Encoding'), - ('mime-version', 'X-MIME-Version'), + ('To', 'X-Original-To'), + ('CC', 'X-Original-CC'), + ('Content-Transfer-Encoding', 'X-Original-Content-Transfer-Encoding'), + ('MIME-Version', 'X-MIME-Version'), ] # Some list posts and mail to the -owner address may contain DomainKey or diff --git a/Mailman/Queue/Runner.py b/Mailman/Queue/Runner.py index 953a201de..33d47ba47 100644 --- a/Mailman/Queue/Runner.py +++ b/Mailman/Queue/Runner.py @@ -86,7 +86,7 @@ class Runner: # Switchboard.files() is guaranteed to hand us the files in FIFO # order. Return an integer count of the number of files that were # available for this qrunner to process. - files = self._switchboard.files() + files = self._switchboard.files for filebase in files: try: # Ask the switchboard for the message and metadata objects @@ -147,7 +147,7 @@ class Runner: # # Find out which mailing list this message is destined for. listname = msgdata.get('listname') - mlist = self._open_list(listname) + mlist = config.list_manager.get(listname) if not mlist: log.error('Dequeuing message destined for missing list: %s', listname) @@ -164,10 +164,11 @@ class Runner: # approach, but I can't think of anything better right now. otranslation = i18n.get_translation() sender = msg.get_sender() - if mlist: - lang = mlist.getMemberLanguage(sender) + member = mlist.members.get_member(sender) + if member: + lang = member.preferred_language else: - lang = config.DEFAULT_SERVER_LANGUAGE + lang = mlist.preferred_language i18n.set_language(lang) msgdata['lang'] = lang try: @@ -181,24 +182,6 @@ class Runner: if keepqueued: self._switchboard.enqueue(msg, msgdata) - # Mapping of listnames to MailList instances as a weak value dictionary. - _listcache = weakref.WeakValueDictionary() - - def _open_list(self, listname): - # Cache the open list so that any use of the list within this process - # uses the same object. We use a WeakValueDictionary so that when the - # list is no longer necessary, its memory is freed. - mlist = self._listcache.get(listname) - if not mlist: - try: - mlist = MailList.MailList(listname, lock=False) - except Errors.MMListError, e: - log.error('error opening list: %s\n%s', listname, e) - return None - else: - self._listcache[listname] = mlist - return mlist - def _log(self, exc): log.error('Uncaught runner exception: %s', exc) s = StringIO() diff --git a/Mailman/Queue/Switchboard.py b/Mailman/Queue/Switchboard.py index 91dfad8c0..2c8672f57 100644 --- a/Mailman/Queue/Switchboard.py +++ b/Mailman/Queue/Switchboard.py @@ -178,8 +178,7 @@ class Switchboard: key += DELTA times[key] = filebase # FIFO sort - for key in sorted(times): - yield times[key] + return [times[key] for key in sorted(times)] def recover_backup_files(self): # Move all .bak files in our slice to .pck. It's impossible for both diff --git a/Mailman/Queue/tests/test_runners.py b/Mailman/Queue/tests/test_runners.py deleted file mode 100644 index 27269909f..000000000 --- a/Mailman/Queue/tests/test_runners.py +++ /dev/null @@ -1,163 +0,0 @@ -# Copyright (C) 2001-2007 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""Unit tests for the various Mailman qrunner modules.""" - -import os -import email -import shutil -import tempfile -import unittest - -from Mailman.Message import Message -from Mailman.Queue.NewsRunner import prepare_message -from Mailman.Queue.Runner import Runner -from Mailman.Queue.Switchboard import Switchboard -from Mailman.testing.base import TestBase - - - -class TestPrepMessage(TestBase): - def test_remove_unacceptables(self): - eq = self.assertEqual - msg = email.message_from_string("""\ -From: aperson@dom.ain -To: _xtest@dom.ain -NNTP-Posting-Host: news.dom.ain -NNTP-Posting-Date: today -X-Trace: blah blah -X-Complaints-To: abuse@dom.ain -Xref: blah blah -Xref: blah blah -Date-Received: yesterday -Posted: tomorrow -Posting-Version: 99.99 -Relay-Version: 88.88 -Received: blah blah - -A message -""") - msgdata = {} - prepare_message(self._mlist, msg, msgdata) - eq(msgdata.get('prepped'), 1) - eq(msg['from'], 'aperson@dom.ain') - eq(msg['to'], '_xtest@dom.ain') - eq(msg['nntp-posting-host'], None) - eq(msg['nntp-posting-date'], None) - eq(msg['x-trace'], None) - eq(msg['x-complaints-to'], None) - eq(msg['xref'], None) - eq(msg['date-received'], None) - eq(msg['posted'], None) - eq(msg['posting-version'], None) - eq(msg['relay-version'], None) - eq(msg['received'], None) - - def test_munge_duplicates_no_duplicates(self): - eq = self.assertEqual - msg = email.message_from_string("""\ -From: aperson@dom.ain -To: _xtest@dom.ain -Cc: someother@dom.ain -Content-Transfer-Encoding: yes - -A message -""") - msgdata = {} - prepare_message(self._mlist, msg, msgdata) - eq(msgdata.get('prepped'), 1) - eq(msg['from'], 'aperson@dom.ain') - eq(msg['to'], '_xtest@dom.ain') - eq(msg['cc'], 'someother@dom.ain') - eq(msg['content-transfer-encoding'], 'yes') - - def test_munge_duplicates(self): - eq = self.assertEqual - msg = email.message_from_string("""\ -From: aperson@dom.ain -To: _xtest@dom.ain -To: two@dom.ain -Cc: three@dom.ain -Cc: four@dom.ain -Cc: five@dom.ain -Content-Transfer-Encoding: yes -Content-Transfer-Encoding: no -Content-Transfer-Encoding: maybe - -A message -""") - msgdata = {} - prepare_message(self._mlist, msg, msgdata) - eq(msgdata.get('prepped'), 1) - eq(msg.get_all('from'), ['aperson@dom.ain']) - eq(msg.get_all('to'), ['_xtest@dom.ain']) - eq(msg.get_all('cc'), ['three@dom.ain']) - eq(msg.get_all('content-transfer-encoding'), ['yes']) - eq(msg.get_all('x-original-to'), ['two@dom.ain']) - eq(msg.get_all('x-original-cc'), ['four@dom.ain', 'five@dom.ain']) - eq(msg.get_all('x-original-content-transfer-encoding'), - ['no', 'maybe']) - - - -class TestableRunner(Runner): - def _dispose(self, mlist, msg, msgdata): - self.msg = msg - self.data = msgdata - return False - - def _doperiodic(self): - self.stop() - - def _snooze(self, filecnt): - return - - -class TestRunner(TestBase): - def setUp(self): - TestBase.setUp(self) - self._tmpdir = tempfile.mkdtemp() - self._msg = email.message_from_string("""\ -From: aperson@dom.ain -To: _xtest@dom.ain - -A test message. -""", Message) - class MyRunner(TestableRunner): - QDIR = self._tmpdir - self._runner = MyRunner() - - def tearDown(self): - shutil.rmtree(self._tmpdir, True) - TestBase.tearDown(self) - - def test_run_loop(self): - eq = self.assertEqual - sb = Switchboard(self._tmpdir) - sb.enqueue(self._msg, listname='_xtest@example.com', foo='yes') - self._runner.run() - eq(self._runner.msg['from'], self._msg['from']) - eq(self._runner.msg['to'], self._msg['to']) - eq(self._runner.data['foo'], 'yes') - - - -def test_suite(): - suite = unittest.TestSuite() - suite.addTest(unittest.makeSuite(TestPrepMessage)) - suite.addTest(unittest.makeSuite(TestRunner)) - return suite diff --git a/Mailman/docs/acknowledge.txt b/Mailman/docs/acknowledge.txt index 6271262ef..82fdd3fd3 100644 --- a/Mailman/docs/acknowledge.txt +++ b/Mailman/docs/acknowledge.txt @@ -21,7 +21,7 @@ acknowledgment. >>> # for new auto-response messages. >>> from Mailman.Queue.sbcache import get_switchboard >>> virginq = get_switchboard(config.VIRGINQUEUE_DIR) - >>> list(virginq.files) + >>> virginq.files [] Subscribe a user to the mailing list. @@ -44,7 +44,7 @@ Non-members can't get acknowledgments of their posts to the mailing list. ... ... """, Message) >>> process(mlist, msg, {}) - >>> list(virginq.files) + >>> virginq.files [] We can also specify the original sender in the message's metadata. If that @@ -55,7 +55,7 @@ person is also not a member, no acknowledgment will be sent either. ... ... """, Message) >>> process(mlist, msg, dict(original_sender='cperson@example.com')) - >>> list(virginq.files) + >>> virginq.files [] @@ -69,7 +69,7 @@ Unless the user has requested acknowledgments, they will not get one. ... ... """, Message) >>> process(mlist, msg, {}) - >>> list(virginq.files) + >>> virginq.files [] Similarly if the original sender is specified in the message metadata, and @@ -83,7 +83,7 @@ will be sent. >>> flush() >>> process(mlist, msg, dict(original_sender='dperson@example.com')) - >>> list(virginq.files) + >>> virginq.files [] @@ -104,11 +104,10 @@ The receipt will include the original message's subject in the response body, ... ... """, Message) >>> process(mlist, msg, {}) - >>> files = list(virginq.files) - >>> len(files) + >>> len(virginq.files) 1 - >>> qmsg, qdata = virginq.dequeue(files[0]) - >>> list(virginq.files) + >>> qmsg, qdata = virginq.dequeue(virginq.files[0]) + >>> virginq.files [] >>> # Print only some of the meta data. The rest is uninteresting. >>> qdata['listname'] @@ -143,11 +142,10 @@ If there is no subject, then the receipt will use a generic message. ... ... """, Message) >>> process(mlist, msg, {}) - >>> files = list(virginq.files) - >>> len(files) + >>> len(virginq.files) 1 - >>> qmsg, qdata = virginq.dequeue(files[0]) - >>> list(virginq.files) + >>> qmsg, qdata = virginq.dequeue(virginq.files[0]) + >>> virginq.files [] >>> # Print only some of the meta data. The rest is uninteresting. >>> qdata['listname'] diff --git a/Mailman/docs/news-runner.txt b/Mailman/docs/news-runner.txt new file mode 100644 index 000000000..f5189c0e0 --- /dev/null +++ b/Mailman/docs/news-runner.txt @@ -0,0 +1,119 @@ +The news runner +=============== + +The news runner is the queue runner that gateways mailing list messages to an +NNTP newsgroup. One of the most important things this runner does is prepare +the message for Usenet (yes, I know that NNTP is not Usenet, but this runner +was originally written to gate to Usenet, which has its own rules). + + >>> from email import message_from_string + >>> from Mailman.Message import Message + >>> from Mailman.configuration import config + >>> from Mailman.database import flush + >>> from Mailman.Queue.NewsRunner import prepare_message + >>> mlist = config.list_manager.create('_xtest@example.com') + >>> mlist.linked_newsgroup = 'comp.lang.python' + >>> flush() + +Some NNTP servers such as INN reject messages containing a set of prohibited +headers, so one of the things that the news runner does is remove these +prohibited headers. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... NNTP-Posting-Host: news.example.com + ... NNTP-Posting-Date: today + ... X-Trace: blah blah + ... X-Complaints-To: abuse@dom.ain + ... Xref: blah blah + ... Xref: blah blah + ... Date-Received: yesterday + ... Posted: tomorrow + ... Posting-Version: 99.99 + ... Relay-Version: 88.88 + ... Received: blah blah + ... + ... A message + ... """, Message) + >>> msgdata = {} + >>> prepare_message(mlist, msg, msgdata) + >>> msgdata['prepped'] + True + >>> print msg.as_string() + From: aperson@example.com + To: _xtest@example.com + Newsgroups: comp.lang.python + Message-ID: <mailman..._xtest@example.com> + Lines: 1 + <BLANKLINE> + A message + <BLANKLINE> + +Some NNTP servers will reject messages where certain headers are duplicated, +so the news runner must collapse or move these duplicate headers to an +X-Original-* header that the news server doesn't care about. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... To: two@example.com + ... Cc: three@example.com + ... Cc: four@example.com + ... Cc: five@example.com + ... Content-Transfer-Encoding: yes + ... Content-Transfer-Encoding: no + ... Content-Transfer-Encoding: maybe + ... + ... A message + ... """, Message) + >>> msgdata = {} + >>> prepare_message(mlist, msg, msgdata) + >>> msgdata['prepped'] + True + >>> print msg.as_string() + From: aperson@example.com + Newsgroups: comp.lang.python + Message-ID: <mailman..._xtest@example.com> + Lines: 1 + To: _xtest@example.com + X-Original-To: two@example.com + CC: three@example.com + X-Original-CC: four@example.com + X-Original-CC: five@example.com + Content-Transfer-Encoding: yes + X-Original-Content-Transfer-Encoding: no + X-Original-Content-Transfer-Encoding: maybe + <BLANKLINE> + A message + <BLANKLINE> + +But if no headers are duplicated, then the news runner doesn't need to modify +the message. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... Cc: someother@example.com + ... Content-Transfer-Encoding: yes + ... + ... A message + ... """, Message) + >>> msgdata = {} + >>> prepare_message(mlist, msg, msgdata) + >>> msgdata['prepped'] + True + >>> print msg.as_string() + From: aperson@example.com + To: _xtest@example.com + Cc: someother@example.com + Content-Transfer-Encoding: yes + Newsgroups: comp.lang.python + Message-ID: <mailman..._xtest@example.com> + Lines: 1 + <BLANKLINE> + A message + <BLANKLINE> + + +XXX More of the NewsRunner should be tested. diff --git a/Mailman/docs/replybot.txt b/Mailman/docs/replybot.txt index b8a67b6d3..e0c44180f 100644 --- a/Mailman/docs/replybot.txt +++ b/Mailman/docs/replybot.txt @@ -19,7 +19,7 @@ message or the amount of time since the last auto-response. >>> # for new auto-response messages. >>> from Mailman.Queue.sbcache import get_switchboard >>> virginq = get_switchboard(config.VIRGINQUEUE_DIR) - >>> list(virginq.files) + >>> virginq.files [] @@ -43,10 +43,9 @@ will be sent, with 0 meaning "there is no grace period". ... help ... """, Message) >>> process(mlist, msg, dict(toowner=True)) - >>> files = list(virginq.files) - >>> len(files) + >>> len(virginq.files) 1 - >>> qmsg, qdata = virginq.dequeue(files[0]) + >>> qmsg, qdata = virginq.dequeue(virginq.files[0]) >>> # Print only some of the meta data. The rest is uninteresting. >>> qdata['listname'] '_xtest@example.com' @@ -67,7 +66,7 @@ will be sent, with 0 meaning "there is no grace period". Precedence: bulk <BLANKLINE> admin autoresponse text - >>> list(virginq.files) + >>> virginq.files [] @@ -85,7 +84,7 @@ no auto-response is sent. ... help me ... """, Message) >>> process(mlist, msg, dict(toowner=True)) - >>> list(virginq.files) + >>> virginq.files [] Mailman itself can suppress autoresponses for certain types of internally @@ -97,7 +96,7 @@ crafted messages, by setting the 'noack' metadata key. ... help for you ... """, Message) >>> process(mlist, msg, dict(noack=True, toowner=True)) - >>> list(virginq.files) + >>> virginq.files [] If there is a Precedence: header with any of the values 'bulk', 'junk', or @@ -110,16 +109,16 @@ If there is a Precedence: header with any of the values 'bulk', 'junk', or ... hey! ... """, Message) >>> process(mlist, msg, dict(toowner=True)) - >>> list(virginq.files) + >>> virginq.files [] >>> msg.replace_header('precedence', 'junk') >>> process(mlist, msg, dict(toowner=True)) - >>> list(virginq.files) + >>> virginq.files [] >>> msg.replace_header('precedence', 'list') >>> process(mlist, msg, dict(toowner=True)) - >>> list(virginq.files) + >>> virginq.files [] Unless the X-Ack: header has a value of "yes", in which case, the Precedence @@ -127,10 +126,9 @@ header is ignored. >>> msg['X-Ack'] = 'yes' >>> process(mlist, msg, dict(toowner=True)) - >>> files = list(virginq.files) - >>> len(files) + >>> len(virginq.files) 1 - >>> qmsg, qdata = virginq.dequeue(files[0]) + >>> qmsg, qdata = virginq.dequeue(virginq.files[0]) >>> del qmsg['message-id'] >>> del qmsg['date'] >>> print qmsg.as_string() @@ -164,10 +162,9 @@ auto-responses: those sent to the -request address... ... help me ... """, Message) >>> process(mlist, msg, dict(torequest=True)) - >>> files = list(virginq.files) - >>> len(files) + >>> len(virginq.files) 1 - >>> qmsg, qdata = virginq.dequeue(files[0]) + >>> qmsg, qdata = virginq.dequeue(virginq.files[0]) >>> del qmsg['message-id'] >>> del qmsg['date'] >>> print qmsg.as_string() @@ -195,10 +192,9 @@ auto-responses: those sent to the -request address... ... help me ... """, Message) >>> process(mlist, msg, {}) - >>> files = list(virginq.files) - >>> len(files) + >>> len(virginq.files) 1 - >>> qmsg, qdata = virginq.dequeue(files[0]) + >>> qmsg, qdata = virginq.dequeue(virginq.files[0]) >>> del qmsg['message-id'] >>> del qmsg['date'] >>> print qmsg.as_string() diff --git a/Mailman/docs/runner.txt b/Mailman/docs/runner.txt new file mode 100644 index 000000000..26fbed405 --- /dev/null +++ b/Mailman/docs/runner.txt @@ -0,0 +1,76 @@ +Queue runners +============= + +The queue runners (qrunner) are the processes that move messages around the +Mailman system. Each qrunner is responsible for a slice of the hash space in +a queue directory. It processes all the files in its slice, sleeps a little +while, then wakes up and runs through its queue files again. + + +Basic architecture +------------------ + +The basic architecture of qrunner is implemented in the base class that all +runners inherit from. This base class implements a .run() method that runs +continuously in a loop until the .stop() method is called. + + >>> import os + >>> from email import message_from_string + >>> from Mailman.Message import Message + >>> from Mailman.Queue.Runner import Runner + >>> from Mailman.Queue.Switchboard import Switchboard + >>> from Mailman.configuration import config + >>> from Mailman.database import flush + >>> mlist = config.list_manager.create('_xtest@example.com') + >>> mlist.preferred_language = 'en' + >>> flush() + +Here is a very simple derived qrunner class. The class attribute QDIR tells +the qrunner which queue directory it is responsible for. Derived classes +should also implement various methods to provide the special functionality. +This is about as simple as a qrunner can be. + + >>> queue_directory = os.path.join(config.QUEUE_DIR, 'test') + >>> class TestableRunner(Runner): + ... QDIR = queue_directory + ... + ... def _dispose(self, mlist, msg, msgdata): + ... self.msg = msg + ... self.msgdata = msgdata + ... return False + ... + ... def _doperiodic(self): + ... self.stop() + ... + ... def _snooze(self, filecnt): + ... return + + >>> runner = TestableRunner() + >>> switchboard = Switchboard(queue_directory) + +This qrunner doesn't do much except run once, storing the message and metadata +on instance variables. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... + ... A test message. + ... """, Message) + >>> filebase = switchboard.enqueue(msg, listname=mlist.fqdn_listname, + ... foo='yes', bar='no') + >>> runner.run() + >>> print runner.msg.as_string() + From: aperson@example.com + To: _xtest@example.com + <BLANKLINE> + A test message. + <BLANKLINE> + >>> sorted(runner.msgdata.items()) + [('_parsemsg', False), + ('bar', 'no'), ('foo', 'yes'), + ('lang', 'en'), ('listname', '_xtest@example.com'), + ('received_time', ...), ('version', 3)] + + +XXX More of the Runner API should be tested. diff --git a/Mailman/docs/switchboard.txt b/Mailman/docs/switchboard.txt index 19a437d0c..267eeffe3 100644 --- a/Mailman/docs/switchboard.txt +++ b/Mailman/docs/switchboard.txt @@ -72,8 +72,8 @@ keyword arguments. >>> switchboard.finish(filebase) >>> sorted(msgdata.items()) [('_parsemsg', False), - ('bar', 2), ('foo', 1), - ('received_time', ...), ('version', 3)] + ('bar', 2), ('foo', 1), + ('received_time', ...), ('version', 3)] Keyword arguments override keys from the metadata dictionary. @@ -82,8 +82,8 @@ Keyword arguments override keys from the metadata dictionary. >>> switchboard.finish(filebase) >>> sorted(msgdata.items()) [('_parsemsg', False), - ('foo', 2), - ('received_time', ...), ('version', 3)] + ('foo', 2), + ('received_time', ...), ('version', 3)] Iterating over files |