From 5b4bb22feca4d520afef44d1c472807e020d17b5 Mon Sep 17 00:00:00 2001
From: Barry Warsaw
Date: Sun, 30 Dec 2007 02:47:45 -0500
Subject: Add three new rules and their associated doctests.

- A rule that checks to see if the sender is a moderated member.
- A rule that checks to see if the sender is a non-member.
- A rule that checks to see if the message has no (or an empty) Subject.

Give IMembers (and the associated database implementation) an `is_moderated`
flag.
---
 Mailman/database/mailman.sql | 1 +
 1 file changed, 1 insertion(+)

(limited to 'Mailman/database/mailman.sql')

diff --git a/Mailman/database/mailman.sql b/Mailman/database/mailman.sql
index cff4daba0..c511e6180 100644
--- a/Mailman/database/mailman.sql
+++ b/Mailman/database/mailman.sql
@@ -145,6 +145,7 @@ CREATE TABLE member (
         id INTEGER NOT NULL,
         role TEXT,
         mailing_list TEXT,
+        is_moderated BOOLEAN,
         address_id INTEGER,
         preferences_id INTEGER,
         PRIMARY KEY (id),
-- 
cgit v1.2.3-70-g09d2


From b6f3ba4c9ebe821dd2c4676d7397fe5312b72a36 Mon Sep 17 00:00:00 2001
From: Barry Warsaw
Date: Fri, 1 Feb 2008 22:21:05 -0500
Subject: SpamDetect is gone, so the chains/rules implementation experiment is
 deemed a success and will now be merged into the trunk.

Move the Truth rule into the built-in rules package in a separate module, and
add a test.

Modify IChainLink so that the rule and chain attributes are not names but
indeed the actual ILink or IChain object directly.  Update the
chains.process() function accordingly.  Remove the IChain.get_rule() method.

Don't derive BuiltInChain from Chain and don't make it an IMutableChain.  It's
now just an IChain, and is implemented concretely.

Refactor the HeaderMatchChain and friends so that it can be used with both the
global HEADER_MATCHES variable and the list-specific header_matches variable,
which has exactly the same semantics.  Oh yeah, get rid of the list's
header_filter_rules attribute and replace it with header_matches so that the
semantics match, it's easy to explain, and it's all nice and clean.
---
 Mailman/Handlers/SpamDetect.py         | 130 ---------------------------------
 Mailman/app/chains.py                  |  15 ++--
 Mailman/app/rules.py                   |  21 +-----
 Mailman/app/styles.py                  |   2 +-
 Mailman/chains/base.py                 |  19 +----
 Mailman/chains/builtin.py              |  63 +++++++++++-----
 Mailman/chains/headers.py              |  83 +++++++++++++++------
 Mailman/database/mailinglist.py        |   2 +-
 Mailman/database/mailman.sql           |   2 +-
 Mailman/docs/chains.txt                |   3 -
 Mailman/interfaces/chain.py            |  10 ---
 Mailman/rules/docs/header-matching.txt |  56 ++++++++++++++
 Mailman/rules/docs/truth.txt           |  10 +++
 Mailman/rules/truth.py                 |  41 +++++++++++
 14 files changed, 224 insertions(+), 233 deletions(-)
 delete mode 100644 Mailman/Handlers/SpamDetect.py
 create mode 100644 Mailman/rules/docs/truth.txt
 create mode 100644 Mailman/rules/truth.py

(limited to 'Mailman/database/mailman.sql')

diff --git a/Mailman/Handlers/SpamDetect.py b/Mailman/Handlers/SpamDetect.py
deleted file mode 100644
index f45b52a32..000000000
--- a/Mailman/Handlers/SpamDetect.py
+++ /dev/null
@@ -1,130 +0,0 @@
-# Copyright (C) 1998-2007 by the Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
-# USA.
-
-"""Do more detailed spam detection.
-
-This module hard codes site wide spam detection.  By hacking the
-KNOWN_SPAMMERS variable, you can set up more regular expression matches
-against message headers.  If spam is detected the message is discarded
-immediately.
-
-TBD: This needs to be made more configurable and robust.
-"""
-
-import re
-
-from cStringIO import StringIO
-from email.Generator import Generator
-
-from Mailman import Errors
-from Mailman import i18n
-from Mailman.Handlers.Hold import hold_for_approval
-from Mailman.configuration import config
-
-# First, play footsie with _ so that the following are marked as translated,
-# but aren't actually translated until we need the text later on.
-def _(s):
-    return s
-
-
-
-class SpamDetected(Errors.DiscardMessage):
-    """The message contains known spam"""
-
-class HeaderMatchHold(Errors.HoldMessage):
-    reason = _('The message headers matched a filter rule')
-
-
-# And reset the translator
-_ = i18n._
-
-
-
-class Tee:
-    def __init__(self, outfp_a, outfp_b):
-        self._outfp_a = outfp_a
-        self._outfp_b = outfp_b
-
-    def write(self, s):
-        self._outfp_a.write(s)
-        self._outfp_b.write(s)
-
-
-# Class to capture the headers separate from the message body
-class HeaderGenerator(Generator):
-    def __init__(self, outfp, mangle_from_=True, maxheaderlen=78):
-        Generator.__init__(self, outfp, mangle_from_, maxheaderlen)
-        self._headertxt = ''
-
-    def _write_headers(self, msg):
-        sfp = StringIO()
-        oldfp = self._fp
-        self._fp = Tee(oldfp, sfp)
-        try:
-            Generator._write_headers(self, msg)
-        finally:
-            self._fp = oldfp
-        self._headertxt = sfp.getvalue()
-
-    def header_text(self):
-        return self._headertxt
-
-
-
-def process(mlist, msg, msgdata):
-    if msgdata.get('approved'):
-        return
-    # Now do header_filter_rules
-    # TK: Collect headers in sub-parts because attachment filename
-    # extension may be a clue to possible virus/spam.
-    headers = ''
-    for p in msg.walk():
-        g = HeaderGenerator(StringIO())
-        g.flatten(p)
-        headers += g.header_text()
-    # Now reshape headers (remove extra CR and connect multiline).
-    headers = re.sub('\n+', '\n', headers)
-    headers = re.sub('\n\s', ' ', headers)
-    for patterns, action, empty in mlist.header_filter_rules:
-        if action == config.DEFER:
-            continue
-        for pattern in patterns.splitlines():
-            if pattern.startswith('#'):
-                continue
-            # ignore 'empty' patterns
-            if not pattern.strip():
-                continue
-            if re.search(pattern, headers, re.IGNORECASE|re.MULTILINE):
-                if action == config.DISCARD:
-                    raise Errors.DiscardMessage
-                if action == config.REJECT:
-                    if msgdata.get('toowner'):
-                        # Don't send rejection notice if addressed to '-owner'
-                        # because it may trigger a loop of notices if the
-                        # sender address is forged.  We just discard it here.
-                        raise Errors.DiscardMessage
-                    raise Errors.RejectMessage(
-                        _('Message rejected by filter rule match'))
-                if action == config.HOLD:
-                    if msgdata.get('toowner'):
-                        # Don't hold '-owner' addressed message.  We just
-                        # pass it here but list-owner can set this to be
-                        # discarded on the GUI if he wants.
-                        return
-                    hold_for_approval(mlist, msg, msgdata, HeaderMatchHold)
-                if action == config.ACCEPT:
-                    return
diff --git a/Mailman/app/chains.py b/Mailman/app/chains.py
index fc7899cbe..2bf0b9ae1 100644
--- a/Mailman/app/chains.py
+++ b/Mailman/app/chains.py
@@ -66,20 +66,19 @@ def process(mlist, msg, msgdata, start_chain='built-in'):
             chain, chain_iter = chain_stack.pop()
             continue
         # Process this link.
-        rule = chain.get_rule(link.rule)
-        if rule.check(mlist, msg, msgdata):
-            if rule.record:
-                hits.append(link.rule)
+        if link.rule.check(mlist, msg, msgdata):
+            if link.rule.record:
+                hits.append(link.rule.name)
             # The rule matched so run its action.
             if link.action is LinkAction.jump:
-                chain = config.chains[link.chain]
+                chain = link.chain
                 chain_iter = chain.get_links(mlist, msg, msgdata)
                 continue
             elif link.action is LinkAction.detour:
                 # Push the current chain so that we can return to it when
                 # the next chain is finished.
                 chain_stack.append((chain, chain_iter))
-                chain = config.chains[link.chain]
+                chain = link.chain
                 chain_iter = chain.get_links(mlist, msg, msgdata)
                 continue
             elif link.action is LinkAction.stop:
@@ -94,8 +93,8 @@ def process(mlist, msg, msgdata, start_chain='built-in'):
                 raise AssertionError('Bad link action: %s' % link.action)
         else:
             # The rule did not match; keep going.
-            if rule.record:
-                misses.append(link.rule)
+            if link.rule.record:
+                misses.append(link.rule.name)
 
 
 
diff --git a/Mailman/app/rules.py b/Mailman/app/rules.py
index f0209c767..50bb8d281 100644
--- a/Mailman/app/rules.py
+++ b/Mailman/app/rules.py
@@ -17,10 +17,7 @@
 
 """Various rule helpers"""
 
-__all__ = [
-    'TruthRule',
-    'initialize',
-    ]
+__all__ = ['initialize']
 __metaclass__ = type
 
 
@@ -32,25 +29,9 @@ from Mailman.configuration import config
 from Mailman.interfaces import IRule
 
 
-
-class TruthRule:
-    """A rule that always matches."""
-    implements(IRule)
-
-    name = 'truth'
-    description = 'A rule which always matches.'
-    record = False
-
-    def check(self, mlist, msg, msgdata):
-        """See `IRule`."""
-        return True
-
-
 
 def initialize():
     """Find and register all rules in all plugins."""
-    # Register built in rules.
-    config.rules[TruthRule.name] = TruthRule()
     # Find rules in plugins.
     for rule_finder in get_plugins('mailman.rules'):
         for rule_class in rule_finder():
diff --git a/Mailman/app/styles.py b/Mailman/app/styles.py
index 4ca3f1b01..3edd88abf 100644
--- a/Mailman/app/styles.py
+++ b/Mailman/app/styles.py
@@ -70,7 +70,7 @@ class DefaultStyle:
         mlist.send_goodbye_msg = config.DEFAULT_SEND_GOODBYE_MSG
         mlist.bounce_matching_headers = (
             config.DEFAULT_BOUNCE_MATCHING_HEADERS)
-        mlist.header_filter_rules = []
+        mlist.header_matches = []
         mlist.anonymous_list = config.DEFAULT_ANONYMOUS_LIST
         mlist.description = u''
         mlist.info = u''
diff --git a/Mailman/chains/base.py b/Mailman/chains/base.py
index 30b66b1cf..16aa63176 100644
--- a/Mailman/chains/base.py
+++ b/Mailman/chains/base.py
@@ -58,23 +58,17 @@ class TerminalChainBase:
         """
         raise NotImplementedError
 
-    def get_rule(self, name):
-        """See `IChain`.
-
-        This always returns the globally registered named rule.
-        """
-        return config.rules[name]
-
     def get_links(self, mlist, msg, msgdata):
         """See `IChain`."""
         return iter(self)
 
     def __iter__(self):
         """See `IChainIterator`."""
+        truth = config.rules['truth']
         # First, yield a link that always runs the process method.
-        yield Link('truth', LinkAction.run, function=self._process)
+        yield Link(truth, LinkAction.run, function=self._process)
         # Now yield a rule that stops all processing.
-        yield Link('truth', LinkAction.stop)
+        yield Link(truth, LinkAction.stop)
 
 
 
@@ -98,13 +92,6 @@ class Chain:
         """See `IMutableChain`."""
         self._links = []
 
-    def get_rule(self, name):
-        """See `IChain`.
-
-        This always returns the globally registered named rule.
-        """
-        return config.rules[name]
-
     def get_links(self, mlist, msg, msgdata):
         """See `IChain`."""
         return iter(ChainIterator(self))
diff --git a/Mailman/chains/builtin.py b/Mailman/chains/builtin.py
index e5e97c3fb..d702b48b9 100644
--- a/Mailman/chains/builtin.py
+++ b/Mailman/chains/builtin.py
@@ -23,37 +23,62 @@ __metaclass__ = type
 
 import logging
 
-from Mailman.interfaces import LinkAction
-from Mailman.chains.base import Chain, Link
+from zope.interface import implements
+
+from Mailman.chains.base import Link
+from Mailman.configuration import config
 from Mailman.i18n import _
+from Mailman.interfaces import IChain, LinkAction
 
 
 log = logging.getLogger('mailman.vette')
 
 
 
-class BuiltInChain(Chain):
+class BuiltInChain:
     """Default built-in chain."""
 
-    def __init__(self):
-        super(BuiltInChain, self).__init__(
-            'built-in', _('The built-in moderation chain.'))
-        self.append_link(Link('approved', LinkAction.jump, 'accept'))
-        self.append_link(Link('emergency', LinkAction.jump, 'hold'))
-        self.append_link(Link('loop', LinkAction.jump, 'discard'))
+    implements(IChain)
+
+    name = 'built-in'
+    description = _('The built-in moderation chain.')
+
+    _link_descriptions = (
+        ('approved', LinkAction.jump, 'accept'),
+        ('emergency', LinkAction.jump, 'hold'),
+        ('loop', LinkAction.jump, 'discard'),
         # Do all of the following before deciding whether to hold the message
         # for moderation.
-        self.append_link(Link('administrivia', LinkAction.defer))
-        self.append_link(Link('implicit-dest', LinkAction.defer))
-        self.append_link(Link('max-recipients', LinkAction.defer))
-        self.append_link(Link('max-size', LinkAction.defer))
-        self.append_link(Link('news-moderation', LinkAction.defer))
-        self.append_link(Link('no-subject', LinkAction.defer))
-        self.append_link(Link('suspicious-header', LinkAction.defer))
+        ('administrivia', LinkAction.defer, None),
+        ('implicit-dest', LinkAction.defer, None),
+        ('max-recipients', LinkAction.defer, None),
+        ('max-size', LinkAction.defer, None),
+        ('news-moderation', LinkAction.defer, None),
+        ('no-subject', LinkAction.defer, None),
+        ('suspicious-header', LinkAction.defer, None),
         # Now if any of the above hit, jump to the hold chain.
-        self.append_link(Link('any', LinkAction.jump, 'hold'))
+        ('any', LinkAction.jump, 'hold'),
         # Take a detour through the self header matching chain, which we'll
         # create later.
-        self.append_link(Link('truth', LinkAction.detour, 'header-match'))
+        ('truth', LinkAction.detour, 'header-match'),
         # Finally, the builtin chain selfs to acceptance.
-        self.append_link(Link('truth', LinkAction.jump, 'accept'))
+        ('truth', LinkAction.jump, 'accept'),
+        )
+
+    def __init__(self):
+        self._cached_links = None
+
+    def get_links(self, mlist, msg, msgdata):
+        """See `IChain`."""
+        if self._cached_links is None:
+            self._cached_links = links = []
+            for rule_name, action, chain_name in self._link_descriptions:
+                # Get the named rule.
+                rule = config.rules[rule_name]
+                # Get the chain, if one is defined.
+                if chain_name is None:
+                    chain = None
+                else:
+                    chain = config.chains[chain_name]
+                links.append(Link(rule, action, chain))
+        return iter(self._cached_links)
diff --git a/Mailman/chains/headers.py b/Mailman/chains/headers.py
index a802eaab4..06dfafeda 100644
--- a/Mailman/chains/headers.py
+++ b/Mailman/chains/headers.py
@@ -23,10 +23,11 @@ __metaclass__ = type
 
 import re
 import logging
+import itertools
 
 from zope.interface import implements
 
-from Mailman.interfaces import IRule, LinkAction
+from Mailman.interfaces import IChainIterator, IRule, LinkAction
 from Mailman.chains.base import Chain, Link
 from Mailman.i18n import _
 from Mailman.configuration import config
@@ -35,6 +36,29 @@ from Mailman.configuration import config
 log = logging.getLogger('mailman.vette')
 
 
+
+def make_link(entry):
+    """Create a Link object.
+
+    :param entry: a 2- or 3-tuple describing a link.  If a 2-tuple, it is a
+        header and a pattern, and a default chain of 'hold' will be used.  If
+        a 3-tuple, the third item is the chain name to use.
+    :return: an ILink.
+    """
+    if len(entry) == 2:
+        header, pattern = entry
+        chain_name = 'hold'
+    elif len(entry) == 3:
+        header, pattern, chain_name = entry
+        # We don't assert that the chain exists here because the jump
+        # chain may not yet have been created.
+    else:
+        raise AssertionError('Bad link description: %s' % entry)
+    rule = HeaderMatchRule(header, pattern)
+    chain = config.chains[chain_name]
+    return Link(rule, LinkAction.jump, chain)
+
+
 
 class HeaderMatchRule:
     """Header matching rule used by header-match chain."""
@@ -78,23 +102,16 @@ class HeaderMatchChain(Chain):
         # The header match rules are not global, so don't register them.
         # These are the only rules that the header match chain can execute.
         self._links = []
-        self._rules = {}
         # Initialize header check rules with those from the global
         # HEADER_MATCHES variable.
         for entry in config.HEADER_MATCHES:
-            if len(entry) == 2:
-                header, pattern = entry
-                chain = 'hold'
-            elif len(entry) == 3:
-                header, pattern, chain = entry
-                # We don't assert that the chain exists here because the jump
-                # chain may not yet have been created.
-            else:
-                raise AssertionError(
-                    'Bad entry for HEADER_MATCHES: %s' % entry)
-            self.extend(header, pattern, chain)
-
-    def extend(self, header, pattern, chain='hold'):
+            self._links.append(make_link(entry))
+        # Keep track of how many global header matching rules we've seen.
+        # This is so the flush() method will only delete those that were added
+        # via extend() or append_link().
+        self._permanent_link_count = len(self._links)
+
+    def extend(self, header, pattern, chain_name='hold'):
         """Extend the existing header matches.
 
         :param header: The case-insensitive header field name.
@@ -103,14 +120,32 @@ class HeaderMatchChain(Chain):
         :param chain: Option chain to jump to if the pattern matches any of
             the named header values.  If not given, the 'hold' chain is used.
         """
-        rule = HeaderMatchRule(header, pattern)
-        self._rules[rule.name] = rule
-        link = Link(rule.name, LinkAction.jump, chain)
-        self._links.append(link)
+        self._links.append(make_link((header, pattern, chain_name)))
 
-    def get_rule(self, name):
-        """See `IChain`.
+    def flush(self):
+        """See `IMutableChain`."""
+        del self._links[self._permanent_link_count:]
 
-        Only local rules are findable by this chain.
-        """
-        return self._rules[name]
+    def get_links(self, mlist, msg, msgdata):
+        """See `IChain`."""
+        list_iterator = HeaderMatchIterator(mlist)
+        return itertools.chain(iter(self._links), iter(list_iterator))
+
+    def __iter__(self):
+        for link in self._links:
+            yield link
+
+
+
+class HeaderMatchIterator:
+    """An iterator of both the global and list-specific chain links."""
+
+    implements(IChainIterator)
+
+    def __init__(self, mlist):
+        self._mlist = mlist
+
+    def __iter__(self):
+        """See `IChainIterator`."""
+        for entry in self._mlist.header_matches:
+            yield make_link(entry)
diff --git a/Mailman/database/mailinglist.py b/Mailman/database/mailinglist.py
index 04c872aab..3230308eb 100644
--- a/Mailman/database/mailinglist.py
+++ b/Mailman/database/mailinglist.py
@@ -113,7 +113,7 @@ class MailingList(Model):
     gateway_to_news = Bool()
     generic_nonmember_action = Int()
     goodbye_msg = Unicode()
-    header_filter_rules = Pickle()
+    header_matches = Pickle()
     hold_these_nonmembers = Pickle()
     include_list_post_header = Bool()
     include_rfc2369_headers = Bool()
diff --git a/Mailman/database/mailman.sql b/Mailman/database/mailman.sql
index c511e6180..0af3401dd 100644
--- a/Mailman/database/mailman.sql
+++ b/Mailman/database/mailman.sql
@@ -93,7 +93,7 @@ CREATE TABLE mailinglist (
         gateway_to_news BOOLEAN,
         generic_nonmember_action INTEGER,
         goodbye_msg TEXT,
-        header_filter_rules BLOB,
+        header_matches BLOB,
         hold_these_nonmembers BLOB,
         include_list_post_header BOOLEAN,
         include_rfc2369_headers BOOLEAN,
diff --git a/Mailman/docs/chains.txt b/Mailman/docs/chains.txt
index e676957d8..433ee8e8e 100644
--- a/Mailman/docs/chains.txt
+++ b/Mailman/docs/chains.txt
@@ -300,9 +300,6 @@ the Hold handler from previous versions of Mailman.
     >>> chain = config.chains['built-in']
     >>> verifyObject(IChain, chain)
     True
-    >>> from Mailman.interfaces import IMutableChain
-    >>> verifyObject(IMutableChain, chain)
-    True
     >>> chain.name
     'built-in'
     >>> chain.description
diff --git a/Mailman/interfaces/chain.py b/Mailman/interfaces/chain.py
index 63d7eb7f7..04c0260c2 100644
--- a/Mailman/interfaces/chain.py
+++ b/Mailman/interfaces/chain.py
@@ -63,16 +63,6 @@ class IChain(Interface):
     name = Attribute('Chain name; must be unique.')
     description = Attribute('A brief description of the chain.')
 
-    def get_rule(name):
-        """Lookup and return the named rule.
-
-        :param name: The name of the rule to return.  This may be a globally
-            registered rule name, in which case it must be unique, or it may
-            be a rule defined locally to the chain.
-        :return: The named `IRule`.
-        :raises: KeyError if the named rule cannot be found.
-        """
-
     def get_links(mlist, msg, msgdata):
         """Get an `IChainIterator` for processing.
 
diff --git a/Mailman/rules/docs/header-matching.txt b/Mailman/rules/docs/header-matching.txt
index b32feabe5..fbd0ff65f 100644
--- a/Mailman/rules/docs/header-matching.txt
+++ b/Mailman/rules/docs/header-matching.txt
@@ -61,6 +61,7 @@ untouched.
 
 But now if the header matches, then the message gets discarded.
 
+    >>> del msg['x-spam-score']
     >>> msg['X-Spam-Score'] = '****'
     >>> del msg['subject']
     >>> msg['Subject'] = 'This is spam, but barely'
@@ -75,6 +76,7 @@ But now if the header matches, then the message gets discarded.
 
 For kicks, let's show a message that's really spammy.
 
+    >>> del msg['x-spam-score']
     >>> msg['X-Spam-Score'] = '**********'
     >>> del msg['subject']
     >>> msg['Subject'] = 'This is really spammy'
@@ -87,3 +89,57 @@ For kicks, let's show a message that's really spammy.
     LOG: ... DISCARD: <four>
     <BLANKLINE>
 
+Flush out the extended header matching rules.
+
+    >>> chain.flush()
+
+
+List-specific header matching
+-----------------------------
+
+Each mailing list can also be configured with a set of header matching regular
+expression rules.  These are used to impose list-specific header filtering
+with the same semantics as the global `HEADER_MATCHES` variable.
+
+The list administrator wants to match not on four stars, but on three plus
+signs, but only for the current mailing list.
+
+    >>> mlist.header_matches = [('x-spam-score', '[+]{3,}', 'discard')]
+
+A message with a spam score of two pluses does not match.
+
+    >>> del msg['x-spam-score']
+    >>> msg['X-Spam-Score'] = '++'
+    >>> del msg['message-id']
+    >>> msg['Message-ID'] = '<five>'
+    >>> file_pos = fp.tell()
+    >>> process(mlist, msg, {}, 'header-match')
+    >>> fp.seek(file_pos)
+    >>> print 'LOG:', fp.read()
+    LOG:
+
+A message with a spam score of three pluses does match.
+
+    >>> del msg['x-spam-score']
+    >>> msg['X-Spam-Score'] = '+++'
+    >>> del msg['message-id']
+    >>> msg['Message-ID'] = '<six>'
+    >>> file_pos = fp.tell()
+    >>> process(mlist, msg, {}, 'header-match')
+    >>> fp.seek(file_pos)
+    >>> print 'LOG:', fp.read()
+    LOG: ... DISCARD: <six>
+    <BLANKLINE>
+
+As does a message with a spam score of four pluses.
+
+    >>> del msg['x-spam-score']
+    >>> msg['X-Spam-Score'] = '+++'
+    >>> del msg['message-id']
+    >>> msg['Message-ID'] = '<seven>'
+    >>> file_pos = fp.tell()
+    >>> process(mlist, msg, {}, 'header-match')
+    >>> fp.seek(file_pos)
+    >>> print 'LOG:', fp.read()
+    LOG: ... DISCARD: <seven>
+    <BLANKLINE>
diff --git a/Mailman/rules/docs/truth.txt b/Mailman/rules/docs/truth.txt
new file mode 100644
index 000000000..baa40772a
--- /dev/null
+++ b/Mailman/rules/docs/truth.txt
@@ -0,0 +1,10 @@
+Truth
+=====
+
+The 'truth' rule always matches.  This makes it useful as a terminus rule for
+unconditionally jumping to another chain.
+
+    >>> from Mailman.configuration import config
+    >>> rule = config.rules['truth']
+    >>> rule.check(False, False, False)
+    True
diff --git a/Mailman/rules/truth.py b/Mailman/rules/truth.py
new file mode 100644
index 000000000..d3cfc30f3
--- /dev/null
+++ b/Mailman/rules/truth.py
@@ -0,0 +1,41 @@
+# Copyright (C) 2008 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+
+"""A rule which always matches."""
+
+__all__ = ['Truth']
+__metaclass__ = type
+
+
+from zope.interface import implements
+
+from Mailman.i18n import _
+from Mailman.interfaces import IRule
+
+
+
+class Truth:
+    """Look for any previous rule match."""
+    implements(IRule)
+
+    name = 'truth'
+    description = _('A rule which always matches.')
+    record = False
+
+    def check(self, mlist, msg, msgdata):
+        """See `IRule`."""
+        return True
-- 
cgit v1.2.3-70-g09d2


From d865604398932718dab761f3fb4f56c3a18d25b8 Mon Sep 17 00:00:00 2001
From: Barry Warsaw
Date: Sat, 2 Feb 2008 11:18:22 -0500
Subject: Convert IncomingRunner to use the new chains disposition
 architecture.  move the big explanatory text at the beginning of incoming.py
 to a doctest called OVERVIEW.tt (which doesn't actually contain any tests yet
 -- it's documentation though).

Added a doctest for the incoming runner, though this will be fleshed out in
more detail next.

Mailman.Post renamed to Mailman.inject, and simplified.  We don't need its
command line script behavior because that is now handled by bin/inject.

Add a 'start_chain' attribute to mailing lists.  This names the chain that
processing of messages for that list begins with.

We were inconsistent in the use of the 'no reply' address attribute.  It's now
always 'no_reply_address'.

Update the smtplistener helper with lessons learned about how to suppress
bogus asyncore error messages.  Also, switch to using a maildir mailbox
instead of an mbox mailbox.
---
 Mailman/Post.py                    |  62 --------------
 Mailman/app/styles.py              |   3 +
 Mailman/bin/inject.py              |   4 +-
 Mailman/database/mailinglist.py    |   3 +-
 Mailman/database/mailman.sql       |   1 +
 Mailman/docs/mlist-addresses.txt   |   2 +-
 Mailman/docs/news-runner.txt       | 158 -----------------------------------
 Mailman/docs/outgoing.txt          | 155 -----------------------------------
 Mailman/docs/runner.txt            |  70 ----------------
 Mailman/docs/switchboard.txt       | 149 ---------------------------------
 Mailman/inject.py                  |  34 ++++++++
 Mailman/interfaces/mailinglist.py  |   2 +-
 Mailman/queue/docs/OVERVIEW.txt    |  78 ++++++++++++++++++
 Mailman/queue/docs/incoming.txt    |  67 +++++++++++++++
 Mailman/queue/docs/news.txt        | 158 +++++++++++++++++++++++++++++++++++
 Mailman/queue/docs/outgoing.txt    | 155 +++++++++++++++++++++++++++++++++++
 Mailman/queue/docs/runner.txt      |  70 ++++++++++++++++
 Mailman/queue/docs/switchboard.txt | 149 +++++++++++++++++++++++++++++++++
 Mailman/queue/incoming.py          | 163 ++++---------------------------------
 Mailman/queue/tests/__init__.py    |   0
 Mailman/tests/smtplistener.py      |  69 ++++++++++++----
 21 files changed, 793 insertions(+), 759 deletions(-)
 delete mode 100644 Mailman/Post.py
 delete mode 100644 Mailman/docs/news-runner.txt
 delete mode 100644 Mailman/docs/outgoing.txt
 delete mode 100644 Mailman/docs/runner.txt
 delete mode 100644 Mailman/docs/switchboard.txt
 create mode 100644 Mailman/inject.py
 create mode 100644 Mailman/queue/docs/OVERVIEW.txt
 create mode 100644 Mailman/queue/docs/incoming.txt
 create mode 100644 Mailman/queue/docs/news.txt
 create mode 100644 Mailman/queue/docs/outgoing.txt
 create mode 100644 Mailman/queue/docs/runner.txt
 create mode 100644 Mailman/queue/docs/switchboard.txt
 delete mode 100644 Mailman/queue/tests/__init__.py

(limited to 'Mailman/database/mailman.sql')

diff --git a/Mailman/Post.py b/Mailman/Post.py
deleted file mode 100644
index 50b9628a0..000000000
--- a/Mailman/Post.py
+++ /dev/null
@@ -1,62 +0,0 @@
-#! /usr/bin/env python
-#
-# Copyright (C) 2001-2007 by the Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
-# USA.
-
-import sys
-
-from Mailman.configuration import config
-from Mailman.queue import Switchboard
-
-
-
-def inject(listname, msg, recips=None, qdir=None):
-    if qdir is None:
-        qdir = config.INQUEUE_DIR
-    queue = Switchboard(qdir)
-    kws = {'listname'  : listname,
-           'tolist'    : 1,
-           '_plaintext': 1,
-           }
-    if recips:
-        kws['recips'] = recips
-    queue.enqueue(msg, **kws)
-
-
-
-if __name__ == '__main__':
-    # When called as a command line script, standard input is read to get the
-    # list that this message is destined to, the list of explicit recipients,
-    # and the message to send (in its entirety).  stdin must have the
-    # following format:
-    #
-    # line 1: the internal name of the mailing list
-    # line 2: the number of explicit recipients to follow.  0 means to use the
-    #         list's membership to calculate recipients.
-    # line 3 - 3+recipnum: explicit recipients, one per line
-    # line 4+recipnum - end of file: the message in RFC 822 format (may
-    #         include an initial Unix-from header)
-    listname = sys.stdin.readline().strip()
-    numrecips = int(sys.stdin.readline())
-    if numrecips == 0:
-        recips = None
-    else:
-        recips = []
-        for i in range(numrecips):
-            recips.append(sys.stdin.readline().strip())
-    # If the message isn't parsable, we won't get an error here
-    inject(listname, sys.stdin.read(), recips)
diff --git a/Mailman/app/styles.py b/Mailman/app/styles.py
index 3edd88abf..1c978363e 100644
--- a/Mailman/app/styles.py
+++ b/Mailman/app/styles.py
@@ -225,6 +225,9 @@ class DefaultStyle:
         # is that they will get all messages, and they will not have an entry
         # in this dictionary.
         mlist.topics_userinterest = {}
+        # The processing chain that messages coming into this list get
+        # processed by.
+        mlist.start_chain = u'built-in'
 
     def match(self, mailing_list, styles):
         # If no other styles have matched, then the default style matches.
diff --git a/Mailman/bin/inject.py b/Mailman/bin/inject.py
index 60185289f..729038118 100644
--- a/Mailman/bin/inject.py
+++ b/Mailman/bin/inject.py
@@ -19,11 +19,11 @@ import os
 import sys
 import optparse
 
-from Mailman import Post
 from Mailman import Utils
 from Mailman import Version
 from Mailman.configuration import config
 from Mailman.i18n import _
+from Mailman.inject import inject
 
 __i18n_templates__ = True
 
@@ -88,7 +88,7 @@ def main():
     else:
         msgtext = sys.stdin.read()
 
-    Post.inject(opts.listname, msgtext, qdir=qdir)
+    inject(opts.listname, msgtext, qdir=qdir)
 
 
 
diff --git a/Mailman/database/mailinglist.py b/Mailman/database/mailinglist.py
index 3230308eb..b3eb56003 100644
--- a/Mailman/database/mailinglist.py
+++ b/Mailman/database/mailinglist.py
@@ -151,6 +151,7 @@ class MailingList(Model):
     send_goodbye_msg = Bool()
     send_reminders = Bool()
     send_welcome_msg = Bool()
+    start_chain = Unicode()
     subject_prefix = Unicode()
     subscribe_auto_approval = Pickle()
     subscribe_policy = Int()
@@ -215,7 +216,7 @@ class MailingList(Model):
         return self.fqdn_listname
 
     @property
-    def noreply_address(self):
+    def no_reply_address(self):
         return '%s@%s' % (config.NO_REPLY_ADDRESS, self.host_name)
 
     @property
diff --git a/Mailman/database/mailman.sql b/Mailman/database/mailman.sql
index 0af3401dd..7c53f25be 100644
--- a/Mailman/database/mailman.sql
+++ b/Mailman/database/mailman.sql
@@ -131,6 +131,7 @@ CREATE TABLE mailinglist (
         send_goodbye_msg BOOLEAN,
         send_reminders BOOLEAN,
         send_welcome_msg BOOLEAN,
+        start_chain TEXT,
         subject_prefix TEXT,
         subscribe_auto_approval BLOB,
         subscribe_policy INTEGER,
diff --git a/Mailman/docs/mlist-addresses.txt b/Mailman/docs/mlist-addresses.txt
index dc2184175..4685a6eea 100644
--- a/Mailman/docs/mlist-addresses.txt
+++ b/Mailman/docs/mlist-addresses.txt
@@ -18,7 +18,7 @@ list.  This is exactly the same as the fully qualified list name.
 Messages to the mailing list's 'no reply' address always get discarded without
 prejudice.
 
-    >>> mlist.noreply_address
+    >>> mlist.no_reply_address
     u'noreply@example.com'
 
 The mailing list's owner address reaches the human moderators.
diff --git a/Mailman/docs/news-runner.txt b/Mailman/docs/news-runner.txt
deleted file mode 100644
index bc6619f50..000000000
--- a/Mailman/docs/news-runner.txt
+++ /dev/null
@@ -1,158 +0,0 @@
-The news runner
-===============
-
-The news runner is the queue runner that gateways mailing list messages to an
-NNTP newsgroup.  One of the most important things this runner does is prepare
-the message for Usenet (yes, I know that NNTP is not Usenet, but this runner
-was originally written to gate to Usenet, which has its own rules).
-
-    >>> from Mailman.configuration import config
-    >>> from Mailman.queue.news import prepare_message
-    >>> mlist = config.db.list_manager.create(u'_xtest@example.com')
-    >>> mlist.linked_newsgroup = u'comp.lang.python'
-
-Some NNTP servers such as INN reject messages containing a set of prohibited
-headers, so one of the things that the news runner does is remove these
-prohibited headers.
-
-    >>> msg = message_from_string("""\
-    ... From: aperson@example.com
-    ... To: _xtest@example.com
-    ... NNTP-Posting-Host: news.example.com
-    ... NNTP-Posting-Date: today
-    ... X-Trace: blah blah
-    ... X-Complaints-To: abuse@dom.ain
-    ... Xref: blah blah
-    ... Xref: blah blah
-    ... Date-Received: yesterday
-    ... Posted: tomorrow
-    ... Posting-Version: 99.99
-    ... Relay-Version: 88.88
-    ... Received: blah blah
-    ...
-    ... A message
-    ... """)
-    >>> msgdata = {}
-    >>> prepare_message(mlist, msg, msgdata)
-    >>> msgdata['prepped']
-    True
-    >>> print msg.as_string()
-    From: aperson@example.com
-    To: _xtest@example.com
-    Newsgroups: comp.lang.python
-    Message-ID: ...
-    Lines: 1
-    <BLANKLINE>
-    A message
-    <BLANKLINE>
-
-Some NNTP servers will reject messages where certain headers are duplicated,
-so the news runner must collapse or move these duplicate headers to an
-X-Original-* header that the news server doesn't care about.
-
-    >>> msg = message_from_string("""\
-    ... From: aperson@example.com
-    ... To: _xtest@example.com
-    ... To: two@example.com
-    ... Cc: three@example.com
-    ... Cc: four@example.com
-    ... Cc: five@example.com
-    ... Content-Transfer-Encoding: yes
-    ... Content-Transfer-Encoding: no
-    ... Content-Transfer-Encoding: maybe
-    ...
-    ... A message
-    ... """)
-    >>> msgdata = {}
-    >>> prepare_message(mlist, msg, msgdata)
-    >>> msgdata['prepped']
-    True
-    >>> print msg.as_string()
-    From: aperson@example.com
-    Newsgroups: comp.lang.python
-    Message-ID: ...
-    Lines: 1
-    To: _xtest@example.com
-    X-Original-To: two@example.com
-    CC: three@example.com
-    X-Original-CC: four@example.com
-    X-Original-CC: five@example.com
-    Content-Transfer-Encoding: yes
-    X-Original-Content-Transfer-Encoding: no
-    X-Original-Content-Transfer-Encoding: maybe
-    <BLANKLINE>
-    A message
-    <BLANKLINE>
-
-But if no headers are duplicated, then the news runner doesn't need to modify
-the message.
-
-    >>> msg = message_from_string("""\
-    ... From: aperson@example.com
-    ... To: _xtest@example.com
-    ... Cc: someother@example.com
-    ... Content-Transfer-Encoding: yes
-    ...
-    ... A message
-    ... """)
-    >>> msgdata = {}
-    >>> prepare_message(mlist, msg, msgdata)
-    >>> msgdata['prepped']
-    True
-    >>> print msg.as_string()
-    From: aperson@example.com
-    To: _xtest@example.com
-    Cc: someother@example.com
-    Content-Transfer-Encoding: yes
-    Newsgroups: comp.lang.python
-    Message-ID: ...
-    Lines: 1
-    <BLANKLINE>
-    A message
-    <BLANKLINE>
-
-
-Newsgroup moderation
---------------------
-
-When the newsgroup is moderated, an Approved: header with the list's posting
-address is added for the benefit of the Usenet system.
-
-    >>> from Mailman.interfaces import NewsModeration
-    >>> mlist.news_moderation = NewsModeration.open_moderated
-    >>> msg = message_from_string("""\
-    ... From: aperson@example.com
-    ... To: _xtest@example.com
-    ... Approved: this gets deleted
-    ...
-    ... """)
-    >>> prepare_message(mlist, msg, {})
-    >>> msg['approved']
-    u'_xtest@example.com'
-
-    >>> mlist.news_moderation = NewsModeration.moderated
-    >>> msg = message_from_string("""\
-    ... From: aperson@example.com
-    ... To: _xtest@example.com
-    ... Approved: this gets deleted
-    ...
-    ... """)
-    >>> prepare_message(mlist, msg, {})
-    >>> msg['approved']
-    u'_xtest@example.com'
-
-But if the newsgroup is not moderated, the Approved: header is not chnaged.
-
-    >>> mlist.news_moderation = NewsModeration.none
-    >>> msg = message_from_string("""\
-    ... From: aperson@example.com
-    ... To: _xtest@example.com
-    ... Approved: this doesn't get deleted
-    ...
-    ... """)
-    >>> prepare_message(mlist, msg, {})
-    >>> msg['approved']
-    u"this doesn't get deleted"
-
-
-XXX More of the NewsRunner should be tested.
diff --git a/Mailman/docs/outgoing.txt b/Mailman/docs/outgoing.txt
deleted file mode 100644
index ba2c6430b..000000000
--- a/Mailman/docs/outgoing.txt
+++ /dev/null
@@ -1,155 +0,0 @@
-The outgoing handler
-====================
-
-Mailman's outgoing queue is used as the wrapper around SMTP delivery to the
-upstream mail server.  The ToOutgoing handler does little more than drop the
-message into the outgoing queue, after calculating whether the message should
-be VERP'd or not.  VERP means Variable Envelope Return Path; we're using that
-term somewhat incorrectly, but within the spirit of the standard, which
-basically describes how to encode the recipient's address in the originator
-headers for unambigous bounce processing.
-
-    >>> from Mailman.Handlers.ToOutgoing import process
-    >>> from Mailman.queue import Switchboard
-    >>> from Mailman.configuration import config
-    >>> mlist = config.db.list_manager.create(u'_xtest@example.com')
-    >>> switchboard = Switchboard(config.OUTQUEUE_DIR)
-
-    >>> def queue_size():
-    ...     size = len(switchboard.files)
-    ...     for filebase in switchboard.files:
-    ...         msg, msgdata = switchboard.dequeue(filebase)
-    ...         switchboard.finish(filebase)
-    ...     return size
-
-Craft a message destined for the outgoing queue.  Include some random metadata
-as if this message had passed through some other handlers.
-
-    >>> msg = message_from_string("""\
-    ... Subject: Here is a message
-    ...
-    ... Something of great import.
-    ... """)
-
-When certain conditions are met, the message will be VERP'd.  For example, if
-the message metadata already has a VERP key, this message will be VERP'd.
-
-    >>> msgdata = dict(foo=1, bar=2, verp=True)
-    >>> process(mlist, msg, msgdata)
-    >>> print msg.as_string()
-    Subject: Here is a message
-    <BLANKLINE>
-    Something of great import.
-    >>> msgdata['verp']
-    True
-
-While the queued message will not be changed, the queued metadata will have an
-additional key set: the mailing list name.
-
-    >>> filebase = switchboard.files[0]
-    >>> qmsg, qmsgdata = switchboard.dequeue(filebase)
-    >>> switchboard.finish(filebase)
-    >>> print qmsg.as_string()
-    Subject: Here is a message
-    <BLANKLINE>
-    Something of great import.
-    >>> sorted(qmsgdata.items())    
-    [('_parsemsg', False),
-     ('bar', 2), ('foo', 1),
-     ('listname', u'_xtest@example.com'),
-     ('received_time', ...),
-     ('verp', True), ('version', 3)]
-    >>> queue_size()
-    0
-
-If the list is set to personalize deliveries, and the global configuration
-option to VERP personalized deliveries is set, then the message will be
-VERP'd.
-
-    # Save the original value for clean up.
-    >>> verp_personalized_delivieries = config.VERP_PERSONALIZED_DELIVERIES
-    >>> config.VERP_PERSONALIZED_DELIVERIES = True
-    >>> from Mailman.interfaces import Personalization
-    >>> mlist.personalize = Personalization.individual
-    >>> msgdata = dict(foo=1, bar=2)
-    >>> process(mlist, msg, msgdata)
-    >>> msgdata['verp']
-    True
-    >>> queue_size()
-    1
-
-However, if the global configuration variable prohibits VERP'ing, even
-personalized lists will not VERP.
-
-    >>> config.VERP_PERSONALIZED_DELIVERIES = False
-    >>> msgdata = dict(foo=1, bar=2)
-    >>> process(mlist, msg, msgdata)
-    >>> print msgdata.get('verp')
-    None
-    >>> queue_size()
-    1
-
-If the list is not personalized, then the message may still be VERP'd based on
-the global configuration variable VERP_DELIVERY_INTERVAL.  This variable tells
-Mailman how often to VERP even non-personalized mailing lists.  It can be set
-to zero, which means non-personalized messages will never be VERP'd.
-
-    # Save the original value for clean up.
-    >>> verp_delivery_interval = config.VERP_DELIVERY_INTERVAL
-    >>> config.VERP_DELIVERY_INTERVAL = 0
-    >>> mlist.personalize = Personalization.none
-    >>> msgdata = dict(foo=1, bar=2)
-    >>> process(mlist, msg, msgdata)
-    >>> print msgdata.get('verp')
-    None
-    >>> queue_size()
-    1
-
-If the interval is set to 1, then every message will be VERP'd.
-
-    >>> config.VERP_DELIVERY_INTERVAL = 1
-    >>> for i in range(10):
-    ...     msgdata = dict(foo=1, bar=2)
-    ...     process(mlist, msg, msgdata)
-    ...     print i, msgdata['verp']
-    0 True
-    1 True
-    2 True
-    3 True
-    4 True
-    5 True
-    6 True
-    7 True
-    8 True
-    9 True
-    >>> queue_size()
-    10
-
-If the interval is set to some other number, then one out of that many posts
-will be VERP'd.
-
-    >>> config.VERP_DELIVERY_INTERVAL = 3
-    >>> for i in range(10):
-    ...     mlist.post_id = i
-    ...     msgdata = dict(foo=1, bar=2)
-    ...     process(mlist, msg, msgdata)
-    ...     print i, msgdata.get('verp', False)
-    0 True
-    1 False
-    2 False
-    3 True
-    4 False
-    5 False
-    6 True
-    7 False
-    8 False
-    9 True
-    >>> queue_size()
-    10
-
-
-Clean up
-========
-
-    >>> config.VERP_PERSONALIZED_DELIVERIES = verp_personalized_delivieries
-    >>> config.VERP_DELIVERY_INTERVAL = verp_delivery_interval
diff --git a/Mailman/docs/runner.txt b/Mailman/docs/runner.txt
deleted file mode 100644
index 5e5a88d8c..000000000
--- a/Mailman/docs/runner.txt
+++ /dev/null
@@ -1,70 +0,0 @@
-Queue runners
-=============
-
-The queue runners (qrunner) are the processes that move messages around the
-Mailman system.  Each qrunner is responsible for a slice of the hash space in
-a queue directory.  It processes all the files in its slice, sleeps a little
-while, then wakes up and runs through its queue files again.
-
-
-Basic architecture
-------------------
-
-The basic architecture of qrunner is implemented in the base class that all
-runners inherit from.  This base class implements a .run() method that runs
-continuously in a loop until the .stop() method is called.
-
-    >>> import os
-    >>> from Mailman.queue import Runner, Switchboard
-    >>> from Mailman.configuration import config
-    >>> mlist = config.db.list_manager.create(u'_xtest@example.com')
-    >>> mlist.preferred_language = u'en'
-
-Here is a very simple derived qrunner class.  The class attribute QDIR tells
-the qrunner which queue directory it is responsible for.  Derived classes
-should also implement various methods to provide the special functionality.
-This is about as simple as a qrunner can be.
-
-    >>> queue_directory = os.path.join(config.QUEUE_DIR, 'test')
-    >>> class TestableRunner(Runner):
-    ...     QDIR = queue_directory
-    ...
-    ...     def _dispose(self, mlist, msg, msgdata):
-    ...         self.msg = msg
-    ...         self.msgdata = msgdata
-    ...         return False
-    ...
-    ...     def _doperiodic(self):
-    ...         self.stop()
-    ...
-    ...     def _snooze(self, filecnt):
-    ...         return
-
-    >>> runner = TestableRunner()
-    >>> switchboard = Switchboard(queue_directory)
-
-This qrunner doesn't do much except run once, storing the message and metadata
-on instance variables.
-
-    >>> msg = message_from_string(u"""\
-    ... From: aperson@example.com
-    ... To: _xtest@example.com
-    ...
-    ... A test message.
-    ... """)
-    >>> filebase = switchboard.enqueue(msg, listname=mlist.fqdn_listname,
-    ...                                foo='yes', bar='no')
-    >>> runner.run()
-    >>> print runner.msg.as_string()
-    From: aperson@example.com
-    To: _xtest@example.com
-    <BLANKLINE>
-    A test message.
-    <BLANKLINE>
-    >>> sorted(runner.msgdata.items())
-    [('_parsemsg', False),
-     ('bar', 'no'), ('foo', 'yes'),
-     ('lang', u'en'), ('listname', u'_xtest@example.com'),
-     ('received_time', ...), ('version', 3)]
-
-XXX More of the Runner API should be tested.
diff --git a/Mailman/docs/switchboard.txt b/Mailman/docs/switchboard.txt
deleted file mode 100644
index 299aba499..000000000
--- a/Mailman/docs/switchboard.txt
+++ /dev/null
@@ -1,149 +0,0 @@
-The switchboard
-===============
-
-The switchboard is subsystem that moves messages between queues.  Each
-instance of a switchboard is responsible for one queue directory.
-
-    >>> msg = message_from_string(u"""\
-    ... From: aperson@example.com
-    ... To: _xtest@example.com
-    ...
-    ... A test message.
-    ... """)
-
-Create a switchboard by giving its queue directory.
-
-    >>> import os
-    >>> from Mailman.configuration import config
-    >>> queue_directory = os.path.join(config.QUEUE_DIR, 'test')
-    >>> from Mailman.queue import Switchboard
-    >>> switchboard = Switchboard(queue_directory)
-    >>> switchboard.queue_directory == queue_directory
-    True
-
-Here's a helper function for ensuring things work correctly.
-
-    >>> def check_qfiles():
-    ...     files = {}
-    ...     for qfile in os.listdir(queue_directory):
-    ...         root, ext = os.path.splitext(qfile)
-    ...         files[ext] = files.get(ext, 0) + 1
-    ...     return sorted(files.items())
-
-
-Enqueing and dequeing
----------------------
-
-The message can be enqueued with metadata specified in the passed in
-dictionary.
-
-    >>> filebase = switchboard.enqueue(msg)
-    >>> check_qfiles()
-    [('.pck', 1)]
-
-To read the contents of a queue file, dequeue it.
-
-    >>> msg, msgdata = switchboard.dequeue(filebase)
-    >>> print msg.as_string()
-    From: aperson@example.com
-    To: _xtest@example.com
-    <BLANKLINE>
-    A test message.
-    <BLANKLINE>
-    >>> sorted(msgdata.items())
-    [('_parsemsg', False), ('received_time', ...), ('version', 3)]
-    >>> check_qfiles()
-    [('.bak', 1)]
-
-To complete the dequeing process, removing all traces of the message file,
-finish it (without preservation).
-
-    >>> switchboard.finish(filebase)
-    >>> check_qfiles()
-    []
-
-When enqueing a file, you can provide additional metadata keys by using
-keyword arguments.
-
-    >>> filebase = switchboard.enqueue(msg, {'foo': 1}, bar=2)
-    >>> msg, msgdata = switchboard.dequeue(filebase)
-    >>> switchboard.finish(filebase)
-    >>> sorted(msgdata.items())
-    [('_parsemsg', False),
-     ('bar', 2), ('foo', 1),
-     ('received_time', ...), ('version', 3)]
-
-Keyword arguments override keys from the metadata dictionary.
-
-    >>> filebase = switchboard.enqueue(msg, {'foo': 1}, foo=2)
-    >>> msg, msgdata = switchboard.dequeue(filebase)
-    >>> switchboard.finish(filebase)
-    >>> sorted(msgdata.items())
-    [('_parsemsg', False),
-     ('foo', 2),
-     ('received_time', ...), ('version', 3)]
-
-
-Iterating over files
---------------------
-
-There are two ways to iterate over all the files in a switchboard's queue.
-Normally, queue files end in .pck (for 'pickle') and the easiest way to
-iterate over just these files is to use the .files attribute.
-
-    >>> filebase_1 = switchboard.enqueue(msg, foo=1)
-    >>> filebase_2 = switchboard.enqueue(msg, foo=2)
-    >>> filebase_3 = switchboard.enqueue(msg, foo=3)
-    >>> filebases = sorted((filebase_1, filebase_2, filebase_3))
-    >>> sorted(switchboard.files) == filebases
-    True
-    >>> check_qfiles()
-    [('.pck', 3)]
-
-You can also use the .get_files() method if you want to iterate over all the
-file bases for some other extension.
-
-    >>> for filebase in switchboard.get_files():
-    ...     msg, msgdata = switchboard.dequeue(filebase)
-    >>> bakfiles = sorted(switchboard.get_files('.bak'))
-    >>> bakfiles == filebases
-    True
-    >>> check_qfiles()
-    [('.bak', 3)]
-    >>> for filebase in switchboard.get_files('.bak'):
-    ...     switchboard.finish(filebase)
-    >>> check_qfiles()
-    []
-
-
-Recovering files
-----------------
-
-Calling .dequeue() without calling .finish() leaves .bak backup files in
-place.  These can be recovered when the switchboard is instantiated.
-
-    >>> filebase_1 = switchboard.enqueue(msg, foo=1)
-    >>> filebase_2 = switchboard.enqueue(msg, foo=2)
-    >>> filebase_3 = switchboard.enqueue(msg, foo=3)
-    >>> for filebase in switchboard.files:
-    ...     msg, msgdata = switchboard.dequeue(filebase)
-    ...     # Don't call .finish()
-    >>> check_qfiles()
-    [('.bak', 3)]
-    >>> switchboard_2 = Switchboard(queue_directory, recover=True)
-    >>> check_qfiles()
-    [('.pck', 3)]
-
-Clean up
-
-    >>> for filebase in switchboard.files:
-    ...     msg, msgdata = switchboard.dequeue(filebase)
-    ...     switchboard.finish(filebase)
-    >>> check_qfiles()
-    []
-
-
-Queue slices
-------------
-
-XXX Add tests for queue slices.
diff --git a/Mailman/inject.py b/Mailman/inject.py
new file mode 100644
index 000000000..a17f8c7d1
--- /dev/null
+++ b/Mailman/inject.py
@@ -0,0 +1,34 @@
+# Copyright (C) 2001-2008 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+
+from Mailman.configuration import config
+from Mailman.queue import Switchboard
+
+
+
+def inject(listname, msg, recips=None, qdir=None):
+    if qdir is None:
+        qdir = config.INQUEUE_DIR
+    queue = Switchboard(qdir)
+    kws = dict(
+        listname=listname,
+        tolist=True,
+        _plaintext=True,
+        )
+    if recips is not None:
+        kws['recips'] = recips
+    queue.enqueue(msg, **kws)
diff --git a/Mailman/interfaces/mailinglist.py b/Mailman/interfaces/mailinglist.py
index 2d3811785..2c828a43c 100644
--- a/Mailman/interfaces/mailinglist.py
+++ b/Mailman/interfaces/mailinglist.py
@@ -82,7 +82,7 @@ class IMailingList(Interface):
         delivery is currently enabled.
         """)
 
-    noreply_address = Attribute(
+    no_reply_address = Attribute(
         """The address to which all messages will be immediately discarded,
         without prejudice or record.  This address is specific to the ddomain,
         even though it's available on the IMailingListAddresses interface.
diff --git a/Mailman/queue/docs/OVERVIEW.txt b/Mailman/queue/docs/OVERVIEW.txt
new file mode 100644
index 000000000..643fa8a5c
--- /dev/null
+++ b/Mailman/queue/docs/OVERVIEW.txt
@@ -0,0 +1,78 @@
+Alias overview
+==============
+
+A typical Mailman list exposes nine aliases which point to seven different
+wrapped scripts.  E.g. for a list named `mylist', you'd have:
+
+    mylist-bounces -> bounces
+    mylist-confirm -> confirm
+    mylist-join    -> join    (-subscribe is an alias)
+    mylist-leave   -> leave   (-unsubscribe is an alias)
+    mylist-owner   -> owner
+    mylist         -> post
+    mylist-request -> request
+
+-request, -join, and -leave are a robot addresses; their sole purpose is to
+process emailed commands, although the latter two are hardcoded to
+subscription and unsubscription requests.  -bounces is the automated bounce
+processor, and all messages to list members have their return address set to
+-bounces.  If the bounce processor fails to extract a bouncing member address,
+it can optionally forward the message on to the list owners.
+
+-owner is for reaching a human operator with minimal list interaction (i.e. no
+bounce processing).  -confirm is another robot address which processes replies
+to VERP-like confirmation notices.
+
+So delivery flow of messages look like this:
+
+    joerandom ---> mylist ---> list members
+       |                           |
+       |                           |[bounces]
+       |        mylist-bounces <---+ <-------------------------------+
+       |              |                                              |
+       |              +--->[internal bounce processing]              |
+       |              ^                |                             |
+       |              |                |    [bounce found]           |
+       |         [bounces *]           +--->[register and discard]   |
+       |              |                |                      |      |
+       |              |                |                      |[*]   |
+       |        [list owners]          |[no bounce found]     |      |
+       |              ^                |                      |      |
+       |              |                |                      |      |
+       +-------> mylist-owner <--------+                      |      |
+       |                                                      |      |
+       |           data/owner-bounces.mbox <--[site list] <---+      |
+       |                                                             |
+       +-------> mylist-join--+                                      |
+       |                      |                                      |
+       +------> mylist-leave--+                                      |
+       |                      |                                      |
+       |                      v                                      |
+       +-------> mylist-request                                      |
+       |              |                                              |
+       |              +---> [command processor]                      |
+       |                            |                                |
+       +-----> mylist-confirm ----> +---> joerandom                  |
+                                              |                      |
+                                              |[bounces]             |
+                                              +----------------------+
+
+A person can send an email to the list address (for posting), the -owner
+address (to reach the human operator), or the -confirm, -join, -leave, and
+-request mailbots.  Message to the list address are then forwarded on to the
+list membership, with bounces directed to the -bounces address.
+
+[*] Messages sent to the -owner address are forwarded on to the list
+owner/moderators.  All -owner destined messages have their bounces directed to
+the site list -bounces address, regardless of whether a human sent the message
+or the message was crafted internally.  The intention here is that the site
+owners want to be notified when one of their list owners' addresses starts
+bouncing (yes, the will be automated in a future release).
+
+Any messages to site owners has their bounces directed to a special
+"loop-killer" address, which just dumps the message into
+data/owners-bounces.mbox.
+
+Finally, message to any of the mailbots causes the requested action to be
+performed.  Results notifications are sent to the author of the message, which
+all bounces pointing back to the -bounces address.
diff --git a/Mailman/queue/docs/incoming.txt b/Mailman/queue/docs/incoming.txt
new file mode 100644
index 000000000..12ff3d3d1
--- /dev/null
+++ b/Mailman/queue/docs/incoming.txt
@@ -0,0 +1,67 @@
+The incoming queue runner
+=========================
+
+This runner's sole purpose in life is to decide the disposition of the
+message.  It can either be accepted for delivery, rejected (i.e. bounced),
+held for moderator approval, or discarded.
+
+The runner operates by processing chains on a message/metadata pair in the
+context of a mailing list.  Each mailing list may have a 'start chain' where
+processing begins, with a global default.  This chain is processed with the
+message eventually ending up in one of the four disposition states described
+above.
+
+    >>> from Mailman.app.lifecycle import create_list
+    >>> mlist = create_list(u'_xtest@example.com')
+    >>> mlist.start_chain
+    u'built-in'
+
+We have a message that is going to be sent to the mailing list.  This message
+is so perfectly fine for posting that it will be accepted and forward to the
+prep queue.
+
+    >>> msg = message_from_string("""\
+    ... From: aperson@example.com
+    ... To: _xtest@example.com
+    ... Subject: My first post
+    ... Message-ID: <first>
+    ...
+    ... First post!
+    ... """)
+
+Normally, the upstream mail server would drop the message in the incoming
+queue, but this is an effective simulation.
+
+    >>> from Mailman.inject import inject
+    >>> inject(u'_xtest@example.com', msg)
+
+The incoming queue runner runs until it is empty.
+
+    >>> from Mailman.queue.incoming import IncomingRunner
+    >>> from Mailman.tests.helpers import make_testable_runner
+    >>> incoming = make_testable_runner(IncomingRunner)
+    >>> incoming.run()
+
+And now the message is in the prep queue.
+
+    >>> from Mailman.configuration import config
+    >>> from Mailman.queue import Switchboard
+    >>> prep_queue = Switchboard(config.PREPQUEUE_DIR)
+    >>> len(prep_queue.files)
+    1
+    >>> from Mailman.tests.helpers import get_queue_messages
+    >>> item = get_queue_messages(prep_queue)[0]
+    >>> print item.msg.as_string()
+    From: aperson@example.com
+    To: _xtest@example.com
+    Subject: My first post
+    Message-ID: <first>
+    X-Mailman-Rule-Misses: approved; emergency; loop; administrivia;
+        implicit-dest;
+        max-recipients; max-size; news-moderation; no-subject;
+        suspicious-header
+    <BLANKLINE>
+    First post!
+    <BLANKLINE>
+    >>> sorted(item.msgdata.items())
+    [...('envsender', u'noreply@example.com')...('tolist', True)...]
diff --git a/Mailman/queue/docs/news.txt b/Mailman/queue/docs/news.txt
new file mode 100644
index 000000000..bc6619f50
--- /dev/null
+++ b/Mailman/queue/docs/news.txt
@@ -0,0 +1,158 @@
+The news runner
+===============
+
+The news runner is the queue runner that gateways mailing list messages to an
+NNTP newsgroup.  One of the most important things this runner does is prepare
+the message for Usenet (yes, I know that NNTP is not Usenet, but this runner
+was originally written to gate to Usenet, which has its own rules).
+
+    >>> from Mailman.configuration import config
+    >>> from Mailman.queue.news import prepare_message
+    >>> mlist = config.db.list_manager.create(u'_xtest@example.com')
+    >>> mlist.linked_newsgroup = u'comp.lang.python'
+
+Some NNTP servers such as INN reject messages containing a set of prohibited
+headers, so one of the things that the news runner does is remove these
+prohibited headers.
+
+    >>> msg = message_from_string("""\
+    ... From: aperson@example.com
+    ... To: _xtest@example.com
+    ... NNTP-Posting-Host: news.example.com
+    ... NNTP-Posting-Date: today
+    ... X-Trace: blah blah
+    ... X-Complaints-To: abuse@dom.ain
+    ... Xref: blah blah
+    ... Xref: blah blah
+    ... Date-Received: yesterday
+    ... Posted: tomorrow
+    ... Posting-Version: 99.99
+    ... Relay-Version: 88.88
+    ... Received: blah blah
+    ...
+    ... A message
+    ... """)
+    >>> msgdata = {}
+    >>> prepare_message(mlist, msg, msgdata)
+    >>> msgdata['prepped']
+    True
+    >>> print msg.as_string()
+    From: aperson@example.com
+    To: _xtest@example.com
+    Newsgroups: comp.lang.python
+    Message-ID: ...
+    Lines: 1
+    <BLANKLINE>
+    A message
+    <BLANKLINE>
+
+Some NNTP servers will reject messages where certain headers are duplicated,
+so the news runner must collapse or move these duplicate headers to an
+X-Original-* header that the news server doesn't care about.
+
+    >>> msg = message_from_string("""\
+    ... From: aperson@example.com
+    ... To: _xtest@example.com
+    ... To: two@example.com
+    ... Cc: three@example.com
+    ... Cc: four@example.com
+    ... Cc: five@example.com
+    ... Content-Transfer-Encoding: yes
+    ... Content-Transfer-Encoding: no
+    ... Content-Transfer-Encoding: maybe
+    ...
+    ... A message
+    ... """)
+    >>> msgdata = {}
+    >>> prepare_message(mlist, msg, msgdata)
+    >>> msgdata['prepped']
+    True
+    >>> print msg.as_string()
+    From: aperson@example.com
+    Newsgroups: comp.lang.python
+    Message-ID: ...
+    Lines: 1
+    To: _xtest@example.com
+    X-Original-To: two@example.com
+    CC: three@example.com
+    X-Original-CC: four@example.com
+    X-Original-CC: five@example.com
+    Content-Transfer-Encoding: yes
+    X-Original-Content-Transfer-Encoding: no
+    X-Original-Content-Transfer-Encoding: maybe
+    <BLANKLINE>
+    A message
+    <BLANKLINE>
+
+But if no headers are duplicated, then the news runner doesn't need to modify
+the message.
+
+    >>> msg = message_from_string("""\
+    ... From: aperson@example.com
+    ... To: _xtest@example.com
+    ... Cc: someother@example.com
+    ... Content-Transfer-Encoding: yes
+    ...
+    ... A message
+    ... """)
+    >>> msgdata = {}
+    >>> prepare_message(mlist, msg, msgdata)
+    >>> msgdata['prepped']
+    True
+    >>> print msg.as_string()
+    From: aperson@example.com
+    To: _xtest@example.com
+    Cc: someother@example.com
+    Content-Transfer-Encoding: yes
+    Newsgroups: comp.lang.python
+    Message-ID: ...
+    Lines: 1
+    <BLANKLINE>
+    A message
+    <BLANKLINE>
+
+
+Newsgroup moderation
+--------------------
+
+When the newsgroup is moderated, an Approved: header with the list's posting
+address is added for the benefit of the Usenet system.
+
+    >>> from Mailman.interfaces import NewsModeration
+    >>> mlist.news_moderation = NewsModeration.open_moderated
+    >>> msg = message_from_string("""\
+    ... From: aperson@example.com
+    ... To: _xtest@example.com
+    ... Approved: this gets deleted
+    ...
+    ... """)
+    >>> prepare_message(mlist, msg, {})
+    >>> msg['approved']
+    u'_xtest@example.com'
+
+    >>> mlist.news_moderation = NewsModeration.moderated
+    >>> msg = message_from_string("""\
+    ... From: aperson@example.com
+    ... To: _xtest@example.com
+    ... Approved: this gets deleted
+    ...
+    ... """)
+    >>> prepare_message(mlist, msg, {})
+    >>> msg['approved']
+    u'_xtest@example.com'
+
+But if the newsgroup is not moderated, the Approved: header is not chnaged.
+
+    >>> mlist.news_moderation = NewsModeration.none
+    >>> msg = message_from_string("""\
+    ... From: aperson@example.com
+    ... To: _xtest@example.com
+    ... Approved: this doesn't get deleted
+    ...
+    ... """)
+    >>> prepare_message(mlist, msg, {})
+    >>> msg['approved']
+    u"this doesn't get deleted"
+
+
+XXX More of the NewsRunner should be tested.
diff --git a/Mailman/queue/docs/outgoing.txt b/Mailman/queue/docs/outgoing.txt
new file mode 100644
index 000000000..ba2c6430b
--- /dev/null
+++ b/Mailman/queue/docs/outgoing.txt
@@ -0,0 +1,155 @@
+The outgoing handler
+====================
+
+Mailman's outgoing queue is used as the wrapper around SMTP delivery to the
+upstream mail server.  The ToOutgoing handler does little more than drop the
+message into the outgoing queue, after calculating whether the message should
+be VERP'd or not.  VERP means Variable Envelope Return Path; we're using that
+term somewhat incorrectly, but within the spirit of the standard, which
+basically describes how to encode the recipient's address in the originator
+headers for unambigous bounce processing.
+
+    >>> from Mailman.Handlers.ToOutgoing import process
+    >>> from Mailman.queue import Switchboard
+    >>> from Mailman.configuration import config
+    >>> mlist = config.db.list_manager.create(u'_xtest@example.com')
+    >>> switchboard = Switchboard(config.OUTQUEUE_DIR)
+
+    >>> def queue_size():
+    ...     size = len(switchboard.files)
+    ...     for filebase in switchboard.files:
+    ...         msg, msgdata = switchboard.dequeue(filebase)
+    ...         switchboard.finish(filebase)
+    ...     return size
+
+Craft a message destined for the outgoing queue.  Include some random metadata
+as if this message had passed through some other handlers.
+
+    >>> msg = message_from_string("""\
+    ... Subject: Here is a message
+    ...
+    ... Something of great import.
+    ... """)
+
+When certain conditions are met, the message will be VERP'd.  For example, if
+the message metadata already has a VERP key, this message will be VERP'd.
+
+    >>> msgdata = dict(foo=1, bar=2, verp=True)
+    >>> process(mlist, msg, msgdata)
+    >>> print msg.as_string()
+    Subject: Here is a message
+    <BLANKLINE>
+    Something of great import.
+    >>> msgdata['verp']
+    True
+
+While the queued message will not be changed, the queued metadata will have an
+additional key set: the mailing list name.
+
+    >>> filebase = switchboard.files[0]
+    >>> qmsg, qmsgdata = switchboard.dequeue(filebase)
+    >>> switchboard.finish(filebase)
+    >>> print qmsg.as_string()
+    Subject: Here is a message
+    <BLANKLINE>
+    Something of great import.
+    >>> sorted(qmsgdata.items())    
+    [('_parsemsg', False),
+     ('bar', 2), ('foo', 1),
+     ('listname', u'_xtest@example.com'),
+     ('received_time', ...),
+     ('verp', True), ('version', 3)]
+    >>> queue_size()
+    0
+
+If the list is set to personalize deliveries, and the global configuration
+option to VERP personalized deliveries is set, then the message will be
+VERP'd.
+
+    # Save the original value for clean up.
+    >>> verp_personalized_delivieries = config.VERP_PERSONALIZED_DELIVERIES
+    >>> config.VERP_PERSONALIZED_DELIVERIES = True
+    >>> from Mailman.interfaces import Personalization
+    >>> mlist.personalize = Personalization.individual
+    >>> msgdata = dict(foo=1, bar=2)
+    >>> process(mlist, msg, msgdata)
+    >>> msgdata['verp']
+    True
+    >>> queue_size()
+    1
+
+However, if the global configuration variable prohibits VERP'ing, even
+personalized lists will not VERP.
+
+    >>> config.VERP_PERSONALIZED_DELIVERIES = False
+    >>> msgdata = dict(foo=1, bar=2)
+    >>> process(mlist, msg, msgdata)
+    >>> print msgdata.get('verp')
+    None
+    >>> queue_size()
+    1
+
+If the list is not personalized, then the message may still be VERP'd based on
+the global configuration variable VERP_DELIVERY_INTERVAL.  This variable tells
+Mailman how often to VERP even non-personalized mailing lists.  It can be set
+to zero, which means non-personalized messages will never be VERP'd.
+
+    # Save the original value for clean up.
+    >>> verp_delivery_interval = config.VERP_DELIVERY_INTERVAL
+    >>> config.VERP_DELIVERY_INTERVAL = 0
+    >>> mlist.personalize = Personalization.none
+    >>> msgdata = dict(foo=1, bar=2)
+    >>> process(mlist, msg, msgdata)
+    >>> print msgdata.get('verp')
+    None
+    >>> queue_size()
+    1
+
+If the interval is set to 1, then every message will be VERP'd.
+
+    >>> config.VERP_DELIVERY_INTERVAL = 1
+    >>> for i in range(10):
+    ...     msgdata = dict(foo=1, bar=2)
+    ...     process(mlist, msg, msgdata)
+    ...     print i, msgdata['verp']
+    0 True
+    1 True
+    2 True
+    3 True
+    4 True
+    5 True
+    6 True
+    7 True
+    8 True
+    9 True
+    >>> queue_size()
+    10
+
+If the interval is set to some other number, then one out of that many posts
+will be VERP'd.
+
+    >>> config.VERP_DELIVERY_INTERVAL = 3
+    >>> for i in range(10):
+    ...     mlist.post_id = i
+    ...     msgdata = dict(foo=1, bar=2)
+    ...     process(mlist, msg, msgdata)
+    ...     print i, msgdata.get('verp', False)
+    0 True
+    1 False
+    2 False
+    3 True
+    4 False
+    5 False
+    6 True
+    7 False
+    8 False
+    9 True
+    >>> queue_size()
+    10
+
+
+Clean up
+========
+
+    >>> config.VERP_PERSONALIZED_DELIVERIES = verp_personalized_delivieries
+    >>> config.VERP_DELIVERY_INTERVAL = verp_delivery_interval
diff --git a/Mailman/queue/docs/runner.txt b/Mailman/queue/docs/runner.txt
new file mode 100644
index 000000000..5e5a88d8c
--- /dev/null
+++ b/Mailman/queue/docs/runner.txt
@@ -0,0 +1,70 @@
+Queue runners
+=============
+
+The queue runners (qrunner) are the processes that move messages around the
+Mailman system.  Each qrunner is responsible for a slice of the hash space in
+a queue directory.  It processes all the files in its slice, sleeps a little
+while, then wakes up and runs through its queue files again.
+
+
+Basic architecture
+------------------
+
+The basic architecture of qrunner is implemented in the base class that all
+runners inherit from.  This base class implements a .run() method that runs
+continuously in a loop until the .stop() method is called.
+
+    >>> import os
+    >>> from Mailman.queue import Runner, Switchboard
+    >>> from Mailman.configuration import config
+    >>> mlist = config.db.list_manager.create(u'_xtest@example.com')
+    >>> mlist.preferred_language = u'en'
+
+Here is a very simple derived qrunner class.  The class attribute QDIR tells
+the qrunner which queue directory it is responsible for.  Derived classes
+should also implement various methods to provide the special functionality.
+This is about as simple as a qrunner can be.
+
+    >>> queue_directory = os.path.join(config.QUEUE_DIR, 'test')
+    >>> class TestableRunner(Runner):
+    ...     QDIR = queue_directory
+    ...
+    ...     def _dispose(self, mlist, msg, msgdata):
+    ...         self.msg = msg
+    ...         self.msgdata = msgdata
+    ...         return False
+    ...
+    ...     def _doperiodic(self):
+    ...         self.stop()
+    ...
+    ...     def _snooze(self, filecnt):
+    ...         return
+
+    >>> runner = TestableRunner()
+    >>> switchboard = Switchboard(queue_directory)
+
+This qrunner doesn't do much except run once, storing the message and metadata
+on instance variables.
+
+    >>> msg = message_from_string(u"""\
+    ... From: aperson@example.com
+    ... To: _xtest@example.com
+    ...
+    ... A test message.
+    ... """)
+    >>> filebase = switchboard.enqueue(msg, listname=mlist.fqdn_listname,
+    ...                                foo='yes', bar='no')
+    >>> runner.run()
+    >>> print runner.msg.as_string()
+    From: aperson@example.com
+    To: _xtest@example.com
+    <BLANKLINE>
+    A test message.
+    <BLANKLINE>
+    >>> sorted(runner.msgdata.items())
+    [('_parsemsg', False),
+     ('bar', 'no'), ('foo', 'yes'),
+     ('lang', u'en'), ('listname', u'_xtest@example.com'),
+     ('received_time', ...), ('version', 3)]
+
+XXX More of the Runner API should be tested.
diff --git a/Mailman/queue/docs/switchboard.txt b/Mailman/queue/docs/switchboard.txt
new file mode 100644
index 000000000..299aba499
--- /dev/null
+++ b/Mailman/queue/docs/switchboard.txt
@@ -0,0 +1,149 @@
+The switchboard
+===============
+
+The switchboard is subsystem that moves messages between queues.  Each
+instance of a switchboard is responsible for one queue directory.
+
+    >>> msg = message_from_string(u"""\
+    ... From: aperson@example.com
+    ... To: _xtest@example.com
+    ...
+    ... A test message.
+    ... """)
+
+Create a switchboard by giving its queue directory.
+
+    >>> import os
+    >>> from Mailman.configuration import config
+    >>> queue_directory = os.path.join(config.QUEUE_DIR, 'test')
+    >>> from Mailman.queue import Switchboard
+    >>> switchboard = Switchboard(queue_directory)
+    >>> switchboard.queue_directory == queue_directory
+    True
+
+Here's a helper function for ensuring things work correctly.
+
+    >>> def check_qfiles():
+    ...     files = {}
+    ...     for qfile in os.listdir(queue_directory):
+    ...         root, ext = os.path.splitext(qfile)
+    ...         files[ext] = files.get(ext, 0) + 1
+    ...     return sorted(files.items())
+
+
+Enqueing and dequeing
+---------------------
+
+The message can be enqueued with metadata specified in the passed in
+dictionary.
+
+    >>> filebase = switchboard.enqueue(msg)
+    >>> check_qfiles()
+    [('.pck', 1)]
+
+To read the contents of a queue file, dequeue it.
+
+    >>> msg, msgdata = switchboard.dequeue(filebase)
+    >>> print msg.as_string()
+    From: aperson@example.com
+    To: _xtest@example.com
+    <BLANKLINE>
+    A test message.
+    <BLANKLINE>
+    >>> sorted(msgdata.items())
+    [('_parsemsg', False), ('received_time', ...), ('version', 3)]
+    >>> check_qfiles()
+    [('.bak', 1)]
+
+To complete the dequeing process, removing all traces of the message file,
+finish it (without preservation).
+
+    >>> switchboard.finish(filebase)
+    >>> check_qfiles()
+    []
+
+When enqueing a file, you can provide additional metadata keys by using
+keyword arguments.
+
+    >>> filebase = switchboard.enqueue(msg, {'foo': 1}, bar=2)
+    >>> msg, msgdata = switchboard.dequeue(filebase)
+    >>> switchboard.finish(filebase)
+    >>> sorted(msgdata.items())
+    [('_parsemsg', False),
+     ('bar', 2), ('foo', 1),
+     ('received_time', ...), ('version', 3)]
+
+Keyword arguments override keys from the metadata dictionary.
+
+    >>> filebase = switchboard.enqueue(msg, {'foo': 1}, foo=2)
+    >>> msg, msgdata = switchboard.dequeue(filebase)
+    >>> switchboard.finish(filebase)
+    >>> sorted(msgdata.items())
+    [('_parsemsg', False),
+     ('foo', 2),
+     ('received_time', ...), ('version', 3)]
+
+
+Iterating over files
+--------------------
+
+There are two ways to iterate over all the files in a switchboard's queue.
+Normally, queue files end in .pck (for 'pickle') and the easiest way to
+iterate over just these files is to use the .files attribute.
+
+    >>> filebase_1 = switchboard.enqueue(msg, foo=1)
+    >>> filebase_2 = switchboard.enqueue(msg, foo=2)
+    >>> filebase_3 = switchboard.enqueue(msg, foo=3)
+    >>> filebases = sorted((filebase_1, filebase_2, filebase_3))
+    >>> sorted(switchboard.files) == filebases
+    True
+    >>> check_qfiles()
+    [('.pck', 3)]
+
+You can also use the .get_files() method if you want to iterate over all the
+file bases for some other extension.
+
+    >>> for filebase in switchboard.get_files():
+    ...     msg, msgdata = switchboard.dequeue(filebase)
+    >>> bakfiles = sorted(switchboard.get_files('.bak'))
+    >>> bakfiles == filebases
+    True
+    >>> check_qfiles()
+    [('.bak', 3)]
+    >>> for filebase in switchboard.get_files('.bak'):
+    ...     switchboard.finish(filebase)
+    >>> check_qfiles()
+    []
+
+
+Recovering files
+----------------
+
+Calling .dequeue() without calling .finish() leaves .bak backup files in
+place.  These can be recovered when the switchboard is instantiated.
+
+    >>> filebase_1 = switchboard.enqueue(msg, foo=1)
+    >>> filebase_2 = switchboard.enqueue(msg, foo=2)
+    >>> filebase_3 = switchboard.enqueue(msg, foo=3)
+    >>> for filebase in switchboard.files:
+    ...     msg, msgdata = switchboard.dequeue(filebase)
+    ...     # Don't call .finish()
+    >>> check_qfiles()
+    [('.bak', 3)]
+    >>> switchboard_2 = Switchboard(queue_directory, recover=True)
+    >>> check_qfiles()
+    [('.pck', 3)]
+
+Clean up
+
+    >>> for filebase in switchboard.files:
+    ...     msg, msgdata = switchboard.dequeue(filebase)
+    ...     switchboard.finish(filebase)
+    >>> check_qfiles()
+    []
+
+
+Queue slices
+------------
+
+XXX Add tests for queue slices.
diff --git a/Mailman/queue/incoming.py b/Mailman/queue/incoming.py
index 6118a7ca0..649ce2213 100644
--- a/Mailman/queue/incoming.py
+++ b/Mailman/queue/incoming.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2007 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2008 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -12,102 +12,26 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
 
-"""Incoming queue runner."""
-
-# A typical Mailman list exposes nine aliases which point to seven different
-# wrapped scripts.  E.g. for a list named `mylist', you'd have:
-#
-# mylist-bounces -> bounces (-admin is a deprecated alias)
-# mylist-confirm -> confirm
-# mylist-join    -> join    (-subscribe is an alias)
-# mylist-leave   -> leave   (-unsubscribe is an alias)
-# mylist-owner   -> owner
-# mylist         -> post
-# mylist-request -> request
-#
-# -request, -join, and -leave are a robot addresses; their sole purpose is to
-# process emailed commands in a Majordomo-like fashion (although the latter
-# two are hardcoded to subscription and unsubscription requests).  -bounces is
-# the automated bounce processor, and all messages to list members have their
-# return address set to -bounces.  If the bounce processor fails to extract a
-# bouncing member address, it can optionally forward the message on to the
-# list owners.
-#
-# -owner is for reaching a human operator with minimal list interaction
-# (i.e. no bounce processing).  -confirm is another robot address which
-# processes replies to VERP-like confirmation notices.
-#
-# So delivery flow of messages look like this:
-#
-# joerandom ---> mylist ---> list members
-#    |                           |
-#    |                           |[bounces]
-#    |        mylist-bounces <---+ <-------------------------------+
-#    |              |                                              |
-#    |              +--->[internal bounce processing]              |
-#    |              ^                |                             |
-#    |              |                |    [bounce found]           |
-#    |         [bounces *]           +--->[register and discard]   |
-#    |              |                |                      |      |
-#    |              |                |                      |[*]   |
-#    |        [list owners]          |[no bounce found]     |      |
-#    |              ^                |                      |      |
-#    |              |                |                      |      |
-#    +-------> mylist-owner <--------+                      |      |
-#    |                                                      |      |
-#    |           data/owner-bounces.mbox <--[site list] <---+      |
-#    |                                                             |
-#    +-------> mylist-join--+                                      |
-#    |                      |                                      |
-#    +------> mylist-leave--+                                      |
-#    |                      |                                      |
-#    |                      v                                      |
-#    +-------> mylist-request                                      |
-#    |              |                                              |
-#    |              +---> [command processor]                      |
-#    |                            |                                |
-#    +-----> mylist-confirm ----> +---> joerandom                  |
-#                                           |                      |
-#                                           |[bounces]             |
-#                                           +----------------------+
-#
-# A person can send an email to the list address (for posting), the -owner
-# address (to reach the human operator), or the -confirm, -join, -leave, and
-# -request mailbots.  Message to the list address are then forwarded on to the
-# list membership, with bounces directed to the -bounces address.
-#
-# [*] Messages sent to the -owner address are forwarded on to the list
-# owner/moderators.  All -owner destined messages have their bounces directed
-# to the site list -bounces address, regardless of whether a human sent the
-# message or the message was crafted internally.  The intention here is that
-# the site owners want to be notified when one of their list owners' addresses
-# starts bouncing (yes, the will be automated in a future release).
-#
-# Any messages to site owners has their bounces directed to a special
-# "loop-killer" address, which just dumps the message into
-# data/owners-bounces.mbox.
-#
-# Finally, message to any of the mailbots causes the requested action to be
-# performed.  Results notifications are sent to the author of the message,
-# which all bounces pointing back to the -bounces address.
+"""Incoming queue runner.
 
+This runner's sole purpose in life is to decide the disposition of the
+message.  It can either be accepted for delivery, rejected (i.e. bounced),
+held for moderator approval, or discarded.
 
-
-import os
-import sys
-import logging
+When accepted, the message is forwarded on to the `prep queue` where it is
+prepared for delivery.  Rejections, discards, and holds are processed
+immediately.
+"""
 
-from cStringIO import StringIO
 
-from Mailman import Errors
+
+from Mailman.app.chains import process
 from Mailman.configuration import config
 from Mailman.queue import Runner
 
-log     = logging.getLogger('mailman.error')
-vlog    = logging.getLogger('mailman.vette')
-
 
 
 class IncomingRunner(Runner):
@@ -115,59 +39,8 @@ class IncomingRunner(Runner):
 
     def _dispose(self, mlist, msg, msgdata):
         if msgdata.get('envsender') is None:
-            msg['envsender'] = mlist.no_reply_address
-        # Process the message through a handler pipeline.  The handler
-        # pipeline can actually come from one of three places: the message
-        # metadata, the mlist, or the global pipeline.
-        #
-        # If a message was requeued due to an uncaught exception, its metadata
-        # will contain the retry pipeline.  Use this above all else.
-        # Otherwise, if the mlist has a `pipeline' attribute, it should be
-        # used.  Final fallback is the global pipeline.
-        pipeline = self._get_pipeline(mlist, msg, msgdata)
-        msgdata['pipeline'] = pipeline
-        more = self._dopipeline(mlist, msg, msgdata, pipeline)
-        if not more:
-            del msgdata['pipeline']
-        config.db.commit()
-        return more
-
-    # Overridable
-    def _get_pipeline(self, mlist, msg, msgdata):
-        # We must return a copy of the list, otherwise, the first message that
-        # flows through the pipeline will empty it out!
-        return msgdata.get('pipeline',
-                           getattr(mlist, 'pipeline',
-                                   config.GLOBAL_PIPELINE))[:]
-
-    def _dopipeline(self, mlist, msg, msgdata, pipeline):
-        while pipeline:
-            handler = pipeline.pop(0)
-            modname = 'Mailman.Handlers.' + handler
-            __import__(modname)
-            try:
-                pid = os.getpid()
-                sys.modules[modname].process(mlist, msg, msgdata)
-                # Failsafe -- a child may have leaked through.
-                if pid <> os.getpid():
-                    log.error('child process leaked thru: %s', modname)
-                    os._exit(1)
-            except Errors.DiscardMessage:
-                # Throw the message away; we need do nothing else with it.
-                vlog.info('Message discarded, msgid: %s',
-                          msg.get('message-id', 'n/a'))
-                return 0
-            except Errors.HoldMessage:
-                # Let the approval process take it from here.  The message no
-                # longer needs to be queued.
-                return 0
-            except Errors.RejectMessage, e:
-                mlist.bounce_message(msg, e)
-                return 0
-            except:
-                # Push this pipeline module back on the stack, then re-raise
-                # the exception.
-                pipeline.insert(0, handler)
-                raise
-        # We've successfully completed handling of this message
-        return 0
+            msgdata['envsender'] = mlist.no_reply_address
+        # Process the message through the mailing list's start chain.
+        process(mlist, msg, msgdata, mlist.start_chain)
+        # Do not keep this message queued.
+        return False
diff --git a/Mailman/queue/tests/__init__.py b/Mailman/queue/tests/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/Mailman/tests/smtplistener.py b/Mailman/tests/smtplistener.py
index 565772b1d..977726247 100644
--- a/Mailman/tests/smtplistener.py
+++ b/Mailman/tests/smtplistener.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2007 by the Free Software Foundation, Inc.
+# Copyright (C) 2007-2008 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -15,8 +15,11 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
 # USA.
 
+"""A test SMTP listener."""
+
 import sys
 import smtpd
+import signal
 import mailbox
 import asyncore
 import optparse
@@ -29,34 +32,68 @@ DEFAULT_PORT = 9025
 
 
 class Channel(smtpd.SMTPChannel):
-    def smtp_EXIT(self, arg):
-        raise asyncore.ExitNow
+    """A channel that can reset the mailbox."""
+
+    def __init__(self, server, conn, addr):
+        smtpd.SMTPChannel.__init__(self, server, conn, addr)
+        # Stash this here since the subclass uses private attributes. :(
+        self._server = server
+
+    def smtp_RSET(self, arg):
+        """Respond to RSET and clear the mailbox."""
+        self._server.clear_mailbox()
+        smtpd.SMTPChannel.smtp_RSET(self, arg)
 
+    def send(self, data):
+        """Silence the bloody asynchat/asyncore broken pipe errors!"""
+        try:
+            return smtpd.SMTPChannel.send(self, data)
+        except socket.error:
+            # Nothing here can affect the outcome, and these messages are just
+            # plain annoying!  So ignore them.
+            pass
 
+
+
 class Server(smtpd.SMTPServer):
-    def __init__(self, localaddr, mboxfile):
+    """An SMTP server that stores messages to a mailbox."""
+
+    def __init__(self, localaddr, mailbox_path):
         smtpd.SMTPServer.__init__(self, localaddr, None)
-        self._mbox = mailbox.mbox(mboxfile)
+        self._mailbox = mailbox.Maildir(mailbox_path)
 
     def handle_accept(self):
+        """Handle connections by creating our own Channel object."""
         conn, addr = self.accept()
         Channel(self, conn, addr)
 
     def process_message(self, peer, mailfrom, rcpttos, data):
+        """Process a message by adding it to the mailbox."""
         msg = message_from_string(data)
         msg['X-Peer'] = peer
         msg['X-MailFrom'] = mailfrom
         msg['X-RcptTo'] = COMMASPACE.join(rcpttos)
-        self._mbox.add(msg)
+        self._mailbox.add(msg)
+        self._mailbox.clean()
 
-    def close(self):
-        self._mbox.flush()
-        self._mbox.close()
+
+
+def handle_signal(*ignore):
+    """Handle signal sent by parent to kill the process."""
+    asyncore.socket_map.clear()
 
 
 
 def main():
-    parser = optparse.OptionParser(usage='%prog mboxfile')
+    parser = optparse.OptionParser(usage="""\
+%prog [options] mboxfile
+
+This starts a process listening on a specified host and port (by default
+localhost:9025) for SMTP conversations.  All messages this process receives
+are stored in a specified mbox file for the parent process to investigate.
+
+This SMTP server responds to RSET commands by clearing the mbox file.
+""")
     parser.add_option('-a', '--address',
                       type='string', default=None,
                       help='host:port to listen on')
@@ -77,12 +114,14 @@ def main():
         host, port = opts.address.split(':', 1)
         port = int(port)
 
+    # Catch the parent's exit signal, and also C-c.
+    signal.signal(signal.SIGTERM, handle_signal)
+    signal.signal(signal.SIGINT, handle_signal)
+
     server = Server((host, port), mboxfile)
-    try:
-        asyncore.loop()
-    except asyncore.ExitNow:
-        asyncore.close_all()
-        server.close()
+    asyncore.loop()
+    asyncore.close_all()
+    server.close()
     return 0
 
 
-- 
cgit v1.2.3-70-g09d2

