4 files changed, 445 insertions, 156 deletions

diff --git a/Mailman/Handlers/Approve.py b/Mailman/Handlers/Approve.py
index 1ff58abf0..1198e6ea7 100644
--- a/Mailman/Handlers/Approve.py
+++ b/Mailman/Handlers/Approve.py
@@ -15,13 +15,7 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
 # USA.
 
-"""Determine whether the message is approved for delivery.
-
-This module only tests for definitive approvals.  IOW, this module only
-determines whether the message is definitively approved or definitively
-denied.  Situations that could hold a message for approval or confirmation are
-not tested by this module.
-"""
+"""Determine whether the message is pre-approved for delivery."""
 
 import re
 
@@ -30,7 +24,7 @@ from email.Iterators import typed_subpart_iterator
 from Mailman import Errors
 from Mailman.configuration import config
 
-NL = '\n'
+EMPTYSTRING = ''
 
 
 
@@ -38,18 +32,15 @@ def process(mlist, msg, msgdata):
     # Short circuits
     if msgdata.get('approved'):
         # Digests, Usenet postings, and some other messages come pre-approved.
-        # TBD: we may want to further filter Usenet messages, so the test
-        # above may not be entirely correct.
+        # XXX we may want to further filter Usenet messages, so the test above
+        # may not be entirely correct.
         return
     # See if the message has an Approved or Approve header with a valid
-    # list-moderator, list-admin.  Also look at the first non-whitespace line
-    # in the file to see if it looks like an Approved header.  We are
-    # specifically /not/ allowing the site admins password to work here
-    # because we want to discourage the practice of sending the site admin
-    # password through email in the clear.
-    missing = []
-    passwd = msg.get('approved', msg.get('approve', missing))
-    if passwd is missing:
+    # moderator password.  Also look at the first non-whitespace line in the
+    # file to see if it looks like an Approved header.
+    missing = object()
+    password = msg.get('approved', msg.get('approve', missing))
+    if password is missing:
         # Find the first text/plain part in the message
         part = None
         stripped = False
@@ -57,22 +48,19 @@ def process(mlist, msg, msgdata):
             break
         # XXX I'm not entirely sure why, but it is possible for the payload of
         # the part to be None, and you can't splitlines() on None.
-        if part is not None and part.get_payload() is not None:
-            lines = part.get_payload(decode=True).splitlines()
-            line = ''
-            for lineno, line in zip(range(len(lines)), lines):
+        if part and part.get_payload() is not None:
+            lines = part.get_payload(decode=True).splitlines(True)
+            for lineno, line in enumerate(lines):
                 if line.strip():
                     break
-            i = line.find(':')
-            if i >= 0:
-                name = line[:i]
-                value = line[i+1:]
-                if name.lower() in ('approve', 'approved'):
-                    passwd = value.lstrip()
+            if ':' in line:
+                header, value = line.split(':', 1)
+                if header.lower() in ('approved', 'approve'):
+                    password = value.strip()
                     # Now strip the first line from the payload so the
                     # password doesn't leak.
                     del lines[lineno]
-                    reset_payload(part, NL.join(lines))
+                    reset_payload(part, EMPTYSTRING.join(lines))
                     stripped = True
         if stripped:
             # MAS: Bug 1181161 - Now try all the text parts in case it's
@@ -84,35 +72,35 @@ def process(mlist, msg, msgdata):
             #
             # This will process all the multipart/alternative parts in the
             # message as well as all other text parts.  We shouldn't find the
-            # pattern outside the mp/a parts, but if we do, it is probably
-            # best to delete it anyway as it does contain the password.
+            # pattern outside the multipart/alternative parts, but if we do,
+            # it is probably best to delete it anyway as it does contain the
+            # password.
             #
             # Make a pattern to delete.  We can't just delete a line because
             # line of HTML or other fancy text may include additional message
             # text.  This pattern works with HTML.  It may not work with rtf
             # or whatever else is possible.
-            pattern = name + ':(\s| )*' + re.escape(passwd)
+            pattern = header + ':(\s| )*' + re.escape(password)
             for part in typed_subpart_iterator(msg, 'text'):
                 if part is not None and part.get_payload() is not None:
                     lines = part.get_payload(decode=True)
                     if re.search(pattern, lines):
                         reset_payload(part, re.sub(pattern, '', lines))
-    if passwd is not missing and mlist.Authenticate((config.AuthListModerator,
-                                                     config.AuthListAdmin),
-                                                    passwd):
+    if password is not missing and password == mlist.moderator_password:
         # BAW: should we definitely deny if the password exists but does not
         # match?  For now we'll let it percolate up for further determination.
         msgdata['approved'] = True
         # Used by the Emergency module
         msgdata['adminapproved'] = True
-    # has this message already been posted to this list?
+    # Has this message already been posted to this list?
     beentheres = [s.strip().lower() for s in msg.get_all('x-beenthere', [])]
-    if mlist.GetListEmail().lower() in beentheres:
+    if mlist.posting_address in beentheres:
         raise Errors.LoopError
 
+
 def reset_payload(part, payload):
     # Set decoded payload maintaining content-type, format and delsp.
-    # TK: Message with 'charset=' cause trouble. So, instead of
+    # TK: Messages with 'charset=' cause trouble.  So, instead of
     #     part.get_content_charset('us-ascii') ...
     cset = part.get_content_charset() or 'us-ascii'
     ctype = part.get_content_type()
diff --git a/Mailman/database/model/mailinglist.py b/Mailman/database/model/mailinglist.py
index b37bcbce1..0725910ec 100644
--- a/Mailman/database/model/mailinglist.py
+++ b/Mailman/database/model/mailinglist.py
@@ -120,7 +120,7 @@ class MailingList(Entity):
     has_field('member_moderation_action',                   Boolean),
     has_field('member_moderation_notice',                   Unicode),
     has_field('mime_is_default_digest',                     Boolean),
-    has_field('mod_password',                               Unicode),
+    has_field('moderator_password',                         Unicode),
     has_field('msg_footer',                                 Unicode),
     has_field('msg_header',                                 Unicode),
     has_field('new_member_options',                         Integer),
@@ -132,7 +132,6 @@ class MailingList(Entity):
     has_field('obscure_addresses',                          Boolean),
     has_field('pass_filename_extensions',                   PickleType),
     has_field('pass_mime_types',                            PickleType),
-    has_field('password',                                   Unicode),
     has_field('personalize',                                Integer),
     has_field('post_id',                                    Integer),
     has_field('preferred_language',                         Unicode),
diff --git a/Mailman/docs/approve.txt b/Mailman/docs/approve.txt
new file mode 100644
index 000000000..cd928e187
--- /dev/null
+++ b/Mailman/docs/approve.txt
@@ -0,0 +1,418 @@
+Pre-approved postings
+=====================
+
+Messages can contain a pre-approval, which is used to bypass the message
+approval queue.  This has several use cases:
+
+- A list administrator can send an emergency message to the mailing list from
+  an unregistered address, say if they are away from their normal email.
+
+- An automated script can be programmed to send a message to an otherwise
+  moderated list.
+  
+In order to support this, a mailing list can be given a 'moderator password'
+which is shared among all the administrators.
+
+    >>> from Mailman.Handlers.Approve import process
+    >>> from Mailman.database import flush
+    >>> from Mailman.configuration import config
+    >>> mlist = config.db.list_manager.create('_xtest@example.com')
+
+
+Short circuiting
+----------------
+
+The message may have been approved by some other means, as evident in the
+message metadata.  In this case, the handler returns immediately.
+
+    >>> from email import message_from_string
+    >>> from Mailman.Message import Message
+    >>> msg = message_from_string("""\
+    ... From: aperson@example.com
+    ...
+    ... An important message.
+    ... """, Message)
+    >>> msgdata = {'approved': True}
+    >>> process(mlist, msg, msgdata)
+    >>> print msg.as_string()
+    From: aperson@example.com
+    <BLANKLINE>
+    An important message.
+    <BLANKLINE>
+    >>> msgdata
+    {'approved': True}
+
+
+The Approved header
+-------------------
+
+If the moderator password is given in an Approved header, then the message
+gets sent through with no further posting moderation.  The Approved header is
+not stripped in this handler module, but instead in the Cleanse module.  This
+ensures that no moderator approval password in the headers will leak out.
+
+    >>> mlist.moderator_password = 'abcxyz'
+    >>> flush()
+    >>> msg['Approved'] = 'abcxyz'
+    >>> msgdata = {}
+    >>> process(mlist, msg, msgdata)
+    >>> print msg.as_string()
+    From: aperson@example.com
+    Approved: abcxyz
+    <BLANKLINE>
+    An important message.
+    <BLANKLINE>
+    >>> sorted(msgdata.items())
+    [('adminapproved', True), ('approved', True)]
+
+But if the wrong password is given, then the message is not marked as being
+approved.  The header is still removed though.
+
+    >>> del msg['Approved']
+    >>> msg['Approved'] = '123456'
+    >>> msgdata = {}
+    >>> process(mlist, msg, msgdata)
+    >>> print msg.as_string()
+    From: aperson@example.com
+    Approved: 123456
+    <BLANKLINE>
+    An important message.
+    <BLANKLINE>
+    >>> msgdata
+    {}
+
+In the spirit of being liberal in what you accept, using an Approve header is
+completely synonymous.
+
+    >>> del msg['Approved']
+    >>> msg['Approve'] = 'abcxyz'
+    >>> msgdata = {}
+    >>> process(mlist, msg, msgdata)
+    >>> print msg.as_string()
+    From: aperson@example.com
+    Approve: abcxyz
+    <BLANKLINE>
+    An important message.
+    <BLANKLINE>
+    >>> sorted(msgdata.items())
+    [('adminapproved', True), ('approved', True)]
+
+    >>> del msg['Approve']
+    >>> msg['Approve'] = '123456'
+    >>> msgdata = {}
+    >>> process(mlist, msg, msgdata)
+    >>> print msg.as_string()
+    From: aperson@example.com
+    Approve: 123456
+    <BLANKLINE>
+    An important message.
+    <BLANKLINE>
+    >>> msgdata
+    {}
+
+
+Using a pseudo-header
+---------------------
+
+Different mail user agents have varying degrees to which they support custom
+headers like Approve and Approved.  For this reason, Mailman also supports
+using a 'pseudo-header', which is really just the first non-whitespace line in
+the payload of the message of the message.  If this pseudo-header looks like a
+matching Approve or Approved header, the message is similarly allowed to pass.
+
+    >>> msg = message_from_string("""\
+    ... From: aperson@example.com
+    ...
+    ... Approved: abcxyz
+    ... An important message.
+    ... """, Message)
+    >>> msgdata = {}
+    >>> process(mlist, msg, msgdata)
+    >>> print msg.as_string()
+    From: aperson@example.com
+    Content-Transfer-Encoding: 7bit
+    MIME-Version: 1.0
+    Content-Type: text/plain; charset="us-ascii"
+    <BLANKLINE>
+    An important message.
+    <BLANKLINE>
+    >>> sorted(msgdata.items())
+    [('adminapproved', True), ('approved', True)]
+
+    >>> msg = message_from_string("""\
+    ... From: aperson@example.com
+    ...
+    ... Approve: abcxyz
+    ... An important message.
+    ... """, Message)
+    >>> msgdata = {}
+    >>> process(mlist, msg, msgdata)
+    >>> print msg.as_string()
+    From: aperson@example.com
+    Content-Transfer-Encoding: 7bit
+    MIME-Version: 1.0
+    Content-Type: text/plain; charset="us-ascii"
+    <BLANKLINE>
+    An important message.
+    <BLANKLINE>
+    >>> sorted(msgdata.items())
+    [('adminapproved', True), ('approved', True)]
+
+As before, a mismatch in the pseudo-header does not approve the message, but
+the pseudo-header line is still removed.
+
+    >>> msg = message_from_string("""\
+    ... From: aperson@example.com
+    ...
+    ... Approved: 123456
+    ... An important message.
+    ... """, Message)
+    >>> msgdata = {}
+    >>> process(mlist, msg, msgdata)
+    >>> print msg.as_string()
+    From: aperson@example.com
+    Content-Transfer-Encoding: 7bit
+    MIME-Version: 1.0
+    Content-Type: text/plain; charset="us-ascii"
+    <BLANKLINE>
+    An important message.
+    <BLANKLINE>
+    >>> msgdata
+    {}
+
+    >>> msg = message_from_string("""\
+    ... From: aperson@example.com
+    ...
+    ... Approve: 123456
+    ... An important message.
+    ... """, Message)
+    >>> msgdata = {}
+    >>> process(mlist, msg, msgdata)
+    >>> print msg.as_string()
+    From: aperson@example.com
+    Content-Transfer-Encoding: 7bit
+    MIME-Version: 1.0
+    Content-Type: text/plain; charset="us-ascii"
+    <BLANKLINE>
+    An important message.
+    <BLANKLINE>
+    >>> msgdata
+    {}
+
+
+MIME multipart support
+----------------------
+
+Mailman searches for the pseudo-header as the first non-whitespace line in the
+first text/plain message part of the message.  This allows the feature to be
+used with MIME documents.
+
+    >>> msg = message_from_string("""\
+    ... From: aperson@example.com
+    ... MIME-Version: 1.0
+    ... Content-Type: multipart/mixed; boundary="AAA"
+    ...
+    ... --AAA
+    ... Content-Type: application/x-ignore
+    ...
+    ... Approved: 123456
+    ... The above line will be ignored.
+    ...
+    ... --AAA
+    ... Content-Type: text/plain
+    ...
+    ... Approved: abcxyz
+    ... An important message.
+    ... --AAA--
+    ... """, Message)
+    >>> msgdata = {}
+    >>> process(mlist, msg, msgdata)
+    >>> print msg.as_string()
+    From: aperson@example.com
+    MIME-Version: 1.0
+    Content-Type: multipart/mixed; boundary="AAA"
+    <BLANKLINE>
+    --AAA
+    Content-Type: application/x-ignore
+    <BLANKLINE>
+    Approved: 123456
+    The above line will be ignored.
+    <BLANKLINE>
+    --AAA
+    Content-Transfer-Encoding: 7bit
+    MIME-Version: 1.0
+    Content-Type: text/plain; charset="us-ascii"
+    <BLANKLINE>
+    An important message.
+    --AAA--
+    <BLANKLINE>
+    >>> sorted(msgdata.items())
+    [('adminapproved', True), ('approved', True)]
+
+    >>> msg = message_from_string("""\
+    ... From: aperson@example.com
+    ... MIME-Version: 1.0
+    ... Content-Type: multipart/mixed; boundary="AAA"
+    ...
+    ... --AAA
+    ... Content-Type: application/x-ignore
+    ...
+    ... Approve: 123456
+    ... The above line will be ignored.
+    ...
+    ... --AAA
+    ... Content-Type: text/plain
+    ...
+    ... Approve: abcxyz
+    ... An important message.
+    ... --AAA--
+    ... """, Message)
+    >>> msgdata = {}
+    >>> process(mlist, msg, msgdata)
+    >>> print msg.as_string()
+    From: aperson@example.com
+    MIME-Version: 1.0
+    Content-Type: multipart/mixed; boundary="AAA"
+    <BLANKLINE>
+    --AAA
+    Content-Type: application/x-ignore
+    <BLANKLINE>
+    Approve: 123456
+    The above line will be ignored.
+    <BLANKLINE>
+    --AAA
+    Content-Transfer-Encoding: 7bit
+    MIME-Version: 1.0
+    Content-Type: text/plain; charset="us-ascii"
+    <BLANKLINE>
+    An important message.
+    --AAA--
+    <BLANKLINE>
+    >>> sorted(msgdata.items())
+    [('adminapproved', True), ('approved', True)]
+
+Here, the correct password is in the non-text/plain part, so it is ignored.
+
+    >>> msg = message_from_string("""\
+    ... From: aperson@example.com
+    ... MIME-Version: 1.0
+    ... Content-Type: multipart/mixed; boundary="AAA"
+    ...
+    ... --AAA
+    ... Content-Type: application/x-ignore
+    ...
+    ... Approve: abcxyz
+    ... The above line will be ignored.
+    ...
+    ... --AAA
+    ... Content-Type: text/plain
+    ...
+    ... Approve: 123456
+    ... An important message.
+    ... --AAA--
+    ... """, Message)
+    >>> msgdata = {}
+    >>> process(mlist, msg, msgdata)
+    >>> print msg.as_string()
+    From: aperson@example.com
+    MIME-Version: 1.0
+    Content-Type: multipart/mixed; boundary="AAA"
+    <BLANKLINE>
+    --AAA
+    Content-Type: application/x-ignore
+    <BLANKLINE>
+    Approve: abcxyz
+    The above line will be ignored.
+    <BLANKLINE>
+    --AAA
+    Content-Transfer-Encoding: 7bit
+    MIME-Version: 1.0
+    Content-Type: text/plain; charset="us-ascii"
+    <BLANKLINE>
+    An important message.
+    --AAA--
+    >>> msgdata
+    {}
+
+    >>> msg = message_from_string("""\
+    ... From: aperson@example.com
+    ... MIME-Version: 1.0
+    ... Content-Type: multipart/mixed; boundary="AAA"
+    ...
+    ... --AAA
+    ... Content-Type: application/x-ignore
+    ...
+    ... Approve: abcxyz
+    ... The above line will be ignored.
+    ...
+    ... --AAA
+    ... Content-Type: text/plain
+    ...
+    ... Approve: 123456
+    ... An important message.
+    ... --AAA--
+    ... """, Message)
+    >>> msgdata = {}
+    >>> process(mlist, msg, msgdata)
+    >>> print msg.as_string()
+    From: aperson@example.com
+    MIME-Version: 1.0
+    Content-Type: multipart/mixed; boundary="AAA"
+    <BLANKLINE>
+    --AAA
+    Content-Type: application/x-ignore
+    <BLANKLINE>
+    Approve: abcxyz
+    The above line will be ignored.
+    <BLANKLINE>
+    --AAA
+    Content-Transfer-Encoding: 7bit
+    MIME-Version: 1.0
+    Content-Type: text/plain; charset="us-ascii"
+    <BLANKLINE>
+    An important message.
+    --AAA--
+    >>> msgdata
+    {}
+
+    >>> msg = message_from_string("""\
+    ... From: aperson@example.com
+    ... MIME-Version: 1.0
+    ... Content-Type: multipart/mixed; boundary="AAA"
+    ...
+    ... --AAA
+    ... Content-Type: application/x-ignore
+    ...
+    ... Approved: abcxyz
+    ... The above line will be ignored.
+    ...
+    ... --AAA
+    ... Content-Type: text/plain
+    ...
+    ... Approved: 123456
+    ... An important message.
+    ... --AAA--
+    ... """, Message)
+    >>> msgdata = {}
+    >>> process(mlist, msg, msgdata)
+    >>> print msg.as_string()
+    From: aperson@example.com
+    MIME-Version: 1.0
+    Content-Type: multipart/mixed; boundary="AAA"
+    <BLANKLINE>
+    --AAA
+    Content-Type: application/x-ignore
+    <BLANKLINE>
+    Approved: abcxyz
+    The above line will be ignored.
+    <BLANKLINE>
+    --AAA
+    Content-Transfer-Encoding: 7bit
+    MIME-Version: 1.0
+    Content-Type: text/plain; charset="us-ascii"
+    <BLANKLINE>
+    An important message.
+    --AAA--
+    <BLANKLINE>
+    >>> msgdata
+    {}
diff --git a/Mailman/tests/test_handlers.py b/Mailman/tests/test_handlers.py
deleted file mode 100644
index 9cadc1db9..000000000
--- a/Mailman/tests/test_handlers.py
+++ /dev/null
@@ -1,116 +0,0 @@
-# Copyright (C) 2001-2007 by the Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
-# USA.
-
-"""Unit tests for the various Mailman/Handlers/*.py modules."""
-
-import email
-import unittest
-
-from Mailman import Errors
-from Mailman import Message
-from Mailman import passwords
-from Mailman.configuration import config
-
-from Mailman.Handlers import Approve
-# Don't test handlers such as SMTPDirect and Sendmail here
-
-
-
-def password(cleartext):
-    return passwords.make_secret(cleartext, passwords.Schemes.ssha)
-
-
-
-class TestApprove(unittest.TestCase):
-    def test_short_circuit(self):
-        msgdata = {'approved': 1}
-        rtn = Approve.process(self._mlist, None, msgdata)
-        # Not really a great test, but there's little else to assert
-        self.assertEqual(rtn, None)
-
-    def test_approved_moderator(self):
-        mlist = self._mlist
-        mlist.mod_password = password('wazoo')
-        msg = email.message_from_string("""\
-Approved: wazoo
-
-""")
-        msgdata = {}
-        Approve.process(mlist, msg, msgdata)
-        self.failUnless(msgdata.has_key('approved'))
-        self.assertEqual(msgdata['approved'], 1)
-
-    def test_approve_moderator(self):
-        mlist = self._mlist
-        mlist.mod_password = password('wazoo')
-        msg = email.message_from_string("""\
-Approve: wazoo
-
-""")
-        msgdata = {}
-        Approve.process(mlist, msg, msgdata)
-        self.failUnless(msgdata.has_key('approved'))
-        self.assertEqual(msgdata['approved'], 1)
-
-    def test_approved_admin(self):
-        mlist = self._mlist
-        mlist.password = password('wazoo')
-        msg = email.message_from_string("""\
-Approved: wazoo
-
-""")
-        msgdata = {}
-        Approve.process(mlist, msg, msgdata)
-        self.failUnless(msgdata.has_key('approved'))
-        self.assertEqual(msgdata['approved'], 1)
-
-    def test_approve_admin(self):
-        mlist = self._mlist
-        mlist.password = password('wazoo')
-        msg = email.message_from_string("""\
-Approve: wazoo
-
-""")
-        msgdata = {}
-        Approve.process(mlist, msg, msgdata)
-        self.failUnless(msgdata.has_key('approved'))
-        self.assertEqual(msgdata['approved'], 1)
-
-    def test_unapproved(self):
-        mlist = self._mlist
-        mlist.password = password('zoowa')
-        msg = email.message_from_string("""\
-Approve: wazoo
-
-""")
-        msgdata = {}
-        Approve.process(mlist, msg, msgdata)
-        self.assertEqual(msgdata.get('approved'), None)
-
-    def test_trip_beentheres(self):
-        mlist = self._mlist
-        msg = email.message_from_string("""\
-X-BeenThere: %s
-
-""" % mlist.GetListEmail())
-        self.assertRaises(Errors.LoopError, Approve.process, mlist, msg, {})
-
-
-
-def test_suite():
-    suite = unittest.TestSuite()
-    return suite