10 files changed, 839 insertions, 83 deletions

diff --git a/Mailman/Cgi/admindb.py b/Mailman/Cgi/admindb.py
index 884b2419f..5764f03a0 100644
--- a/Mailman/Cgi/admindb.py
+++ b/Mailman/Cgi/admindb.py
@@ -37,6 +37,7 @@ from Mailman.Handlers.Moderate import ModeratedMemberPost
 from Mailman.ListAdmin import readMessage
 from Mailman.configuration import config
 from Mailman.htmlformat import *
+from Mailman.interfaces import RequestType
 
 EMPTYSTRING = ''
 NL = '\n'
@@ -54,11 +55,14 @@ log = logging.getLogger('mailman.error')
 
 
 def helds_by_sender(mlist):
-    heldmsgs = mlist.GetHeldMessageIds()
     bysender = {}
-    for id in heldmsgs:
-        sender = mlist.GetRecord(id)[1]
-        bysender.setdefault(sender, []).append(id)
+    requests = config.db.get_list_requests(mlist)
+    for request in requests.of_type(RequestType.held_message):
+        key, data = requests.get_request(request.id)
+        sender = data.get('sender')
+            assert sender is not None, (
+                'No sender for held message: %s' % request.id)
+            bysender.setdefault(sender, []).append(request.id)
     return bysender
 
 
@@ -146,7 +150,7 @@ def main():
             process_form(mlist, doc, cgidata)
         # Now print the results and we're done.  Short circuit for when there
         # are no pending requests, but be sure to save the results!
-        if not mlist.NumRequestsPending():
+        if config.db.requests.get_list_requests(mlist).count == 0:
             title = _('%(realname)s Administrative Database')
             doc.SetTitle(title)
             doc.AddItem(Header(2, title))
@@ -172,8 +176,9 @@ def main():
                 + ' <em>%s</em>' % mlist.real_name))
         if details <> 'instructions':
             form.AddItem(Center(SubmitButton('submit', _('Submit All Data'))))
-        nomessages = not mlist.GetHeldMessageIds()
-        if not (details or sender or msgid or nomessages):
+        requestsdb = config.db.get_list_requests(mlist)
+        message_count = requestsdb.count_of(RequestType.held_message)
+        if not (details or sender or msgid or message_count == 0):
             form.AddItem(Center(
                 CheckBox('discardalldefersp', 0).Format() +
                 '&nbsp;' +
@@ -257,8 +262,8 @@ def handle_no_list(msg=''):
 
 def show_pending_subs(mlist, form):
     # Add the subscription request section
-    pendingsubs = mlist.GetSubscriptionIds()
-    if not pendingsubs:
+    requestsdb = config.db.get_list_requests(mlist)
+    if requestsdb.count_of(RequestType.subscription) == 0:
         return 0
     form.AddItem('<hr>')
     form.AddItem(Center(Header(2, _('Subscription Requests'))))
@@ -269,18 +274,24 @@ def show_pending_subs(mlist, form):
                   ])
     # Alphabetical order by email address
     byaddrs = {}
-    for id in pendingsubs:
-        addr = mlist.GetRecord(id)[1]
-        byaddrs.setdefault(addr, []).append(id)
-    addrs = byaddrs.keys()
-    addrs.sort()
+    for request in requestsdb.of_type(RequestType.subscription):
+        key, data = requestsdb.get_request(requst.id)
+        addr = data['addr']
+        byaddrs.setdefault(addr, []).append(request.id)
+    addrs = sorted(byaddrs)
     num = 0
     for addr, ids in byaddrs.items():
         # Eliminate duplicates
         for id in ids[1:]:
             mlist.HandleRequest(id, config.DISCARD)
         id = ids[0]
-        time, addr, fullname, passwd, digest, lang = mlist.GetRecord(id)
+        key, data = requestsdb.get_request(id)
+        time = data['time']
+        addr = data['addr']
+        fullname = data['fullname']
+        passwd = data['passwd']
+        digest = data['digest']
+        lang = data['lang']
         fullname = Utils.uncanonstr(fullname, mlist.preferred_language)
         radio = RadioButtonArray(id, (_('Defer'),
                                       _('Approve'),
@@ -310,8 +321,8 @@ def show_pending_subs(mlist, form):
 def show_pending_unsubs(mlist, form):
     # Add the pending unsubscription request section
     lang = mlist.preferred_language
-    pendingunsubs = mlist.GetUnsubscriptionIds()
-    if not pendingunsubs:
+    requestsdb = config.db.get_list_requests(mlist)
+    if requestsdb.count_of(RequestType.unsubscription) == 0:
         return 0
     table = Table(border=2)
     table.AddRow([Center(Bold(_('User address/name'))),
@@ -320,18 +331,19 @@ def show_pending_unsubs(mlist, form):
                   ])
     # Alphabetical order by email address
     byaddrs = {}
-    for id in pendingunsubs:
-        addr = mlist.GetRecord(id)[1]
-        byaddrs.setdefault(addr, []).append(id)
-    addrs = byaddrs.keys()
-    addrs.sort()
+    for request in requestsdb.of_type(RequestType.unsubscription):
+        key, data = requestsdb.get_request(request.id)
+        addr = data['addr']
+        byaddrs.setdefault(addr, []).append(request.id)
+    addrs = sorted(byaddrs)
     num = 0
     for addr, ids in byaddrs.items():
         # Eliminate duplicates
         for id in ids[1:]:
             mlist.HandleRequest(id, config.DISCARD)
         id = ids[0]
-        addr = mlist.GetRecord(id)
+        key, data = requestsdb.get_record(id)
+        addr = data['addr']
         try:
             fullname = Utils.uncanonstr(mlist.getMemberName(addr), lang)
         except Errors.NotAMemberError:
@@ -458,8 +470,13 @@ def show_helds_overview(mlist, form):
         right.AddRow(['&nbsp;', '&nbsp;'])
         counter = 1
         for id in bysender[sender]:
-            info = mlist.GetRecord(id)
-            ptime, sender, subject, reason, filename, msgdata = info
+            key, data = requestsdb.get_record(id)
+            ptime = data['ptime']
+            sender = data['sender']
+            subject = data['subject']
+            reason = data['reason']
+            filename = data['filename']
+            msgdata = data['msgdata']
             # BAW: This is really the size of the message pickle, which should
             # be close, but won't be exact.  Sigh, good enough.
             try:
@@ -505,18 +522,18 @@ def show_sender_requests(mlist, form, sender):
         # BAW: should we print an error message?
         return
     total = len(sender_ids)
-    count = 1
-    for id in sender_ids:
-        info = mlist.GetRecord(id)
-        show_post_requests(mlist, id, info, total, count, form)
-        count += 1
+    requestsdb = config.db.get_list_requests(mlist)
+    for i, id in enumerate(sender_ids):
+        key, data = requestsdb.get_record(id)
+        show_post_requests(mlist, id, data, total, count + 1, form)
 
 
 
 def show_message_requests(mlist, form, id):
+    requestdb = config.db.get_list_requests(mlist)
     try:
         id = int(id)
-        info = mlist.GetRecord(id)
+        info = requestdb.get_record(id)
     except (ValueError, KeyError):
         # BAW: print an error message?
         return
@@ -525,13 +542,12 @@ def show_message_requests(mlist, form, id):
 
 
 def show_detailed_requests(mlist, form):
-    all = mlist.GetHeldMessageIds()
-    total = len(all)
-    count = 1
-    for id in mlist.GetHeldMessageIds():
-        info = mlist.GetRecord(id)
-        show_post_requests(mlist, id, info, total, count, form)
-        count += 1
+    requestsdb = config.db.get_list_requests(mlist)
+    total = requestsdb.count_of(RequestType.held_message)
+    all = requestsdb.of_type(RequestType.held_message)
+    for i, request in enumerate(all):
+        key, data = requestdb.get_request(request.id)
+        show_post_requests(mlist, request.id, data, total, i + 1, form)
 
 
 
@@ -767,8 +783,10 @@ def process_form(mlist, doc, cgidata):
             forwardaddr = cgidata[forwardaddrkey].value
         # Should we ban this address?  Do this check before handling the
         # request id because that will evict the record.
+        requestsdb = config.db.get_list_requests(mlist)
         if cgidata.getvalue(bankey):
-            sender = mlist.GetRecord(request_id)[1]
+            key, data = requestsdb.get_record(request_id)
+            sender = data['sender']
             if sender not in mlist.ban_list:
                 mlist.ban_list.append(sender)
         # Handle the request id
@@ -782,7 +800,8 @@ def process_form(mlist, doc, cgidata):
         except Errors.MMAlreadyAMember, v:
             erroraddrs.append(v)
         except Errors.MembershipIsBanned, pattern:
-            sender = mlist.GetRecord(request_id)[1]
+            data = requestsdb.get_record(request_id)
+            sender = data['sender']
             banaddrs.append((sender, pattern))
     # save the list and print the results
     doc.AddItem(Header(2, _('Database Updated...')))
diff --git a/Mailman/Cgi/confirm.py b/Mailman/Cgi/confirm.py
index 121d775ed..e9a3355d8 100644
--- a/Mailman/Cgi/confirm.py
+++ b/Mailman/Cgi/confirm.py
@@ -623,7 +623,10 @@ def heldmsg_confirm(mlist, doc, cookie):
             # Do this in two steps so we can get the preferred language for
             # the user who posted the message.
             op, id = mlist.pend_confirm(cookie)
-            ign, sender, msgsubject, ign, ign, ign = mlist.GetRecord(id)
+            requestsdb = config.db.get_list_requests(mlist)
+            key, data = requestsdb.get_record(id)
+            sender = data['sender']
+            msgsubject = data['msgsubject']
             subject = Utils.websafe(msgsubject)
             lang = mlist.getMemberLanguage(sender)
             i18n.set_language(lang)
@@ -670,9 +673,10 @@ def heldmsg_prompt(mlist, doc, cookie, id):
     # Get the record, but watch for KeyErrors which mean the admin has already
     # disposed of this message.
     mlist.Lock()
+    requestdb = config.db.get_list_requests(mlist)
     try:
         try:
-            data = mlist.GetRecord(id)
+            key, data = requestdb.get_record(id)
         except KeyError:
             data = None
     finally:
diff --git a/Mailman/MailList.py b/Mailman/MailList.py
index 8d61f6a20..b828bf5af 100644
--- a/Mailman/MailList.py
+++ b/Mailman/MailList.py
@@ -59,7 +59,6 @@ from Mailman.Bouncer import Bouncer
 from Mailman.Deliverer import Deliverer
 from Mailman.Digester import Digester
 from Mailman.HTMLFormatter import HTMLFormatter
-from Mailman.ListAdmin import ListAdmin
 from Mailman.SecurityManager import SecurityManager
 
 # GUI components package
@@ -84,7 +83,7 @@ slog    = logging.getLogger('mailman.subscribe')
 
 
 # Use mixins here just to avoid having any one chunk be too large.
-class MailList(object, HTMLFormatter, Deliverer, ListAdmin,
+class MailList(object, HTMLFormatter, Deliverer,
                Archiver, Digester, SecurityManager, Bouncer):
 
     implements(
@@ -365,7 +364,6 @@ class MailList(object, HTMLFormatter, Deliverer, ListAdmin,
         self._lock.refresh()
         # The member adaptor may have its own save operation
         self._memberadaptor.save()
-        self.SaveRequestsDb()
         self.CheckHTMLArchiveDir()
 
     def Load(self):
diff --git a/Mailman/Utils.py b/Mailman/Utils.py
index fdc1d2c44..9d6fdf145 100644
--- a/Mailman/Utils.py
+++ b/Mailman/Utils.py
@@ -475,6 +475,7 @@ def findtext(templatefile, dict=None, raw=False, lang=None, mlist=None):
     if mlist is not None:
         languages.add(mlist.preferred_language)
     languages.add(config.DEFAULT_SERVER_LANGUAGE)
+    assert None not in languages, 'None in languages'
     # Calculate the locations to scan
     searchdirs = []
     if mlist is not None:
diff --git a/Mailman/app/moderator.py b/Mailman/app/moderator.py
new file mode 100644
index 000000000..48a4d4251
--- /dev/null
+++ b/Mailman/app/moderator.py
@@ -0,0 +1,352 @@
+# Copyright (C) 2007 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+
+"""Application support for moderators."""
+
+from __future__ import with_statement
+
+__all__ = [
+    'hold_message',
+    ]
+
+import logging
+
+from datetime import datetime
+from email.utils import formatdate, getaddresses, make_msgid
+
+from Mailman import Message
+from Mailman import Utils
+from Mailman import i18n
+from Mailman.Queue.sbcache import get_switchboard
+from Mailman.configuration import config
+from Mailman.constants import Action
+from Mailman.interfaces import RequestType
+
+_ = i18n._
+__i18n_templates__ = True
+
+log = logging.getLogger('mailman.vette')
+
+
+
+def hold_message(mlist, msg, msgdata=None, reason=None):
+    if msgdata is None:
+        msgdata = {}
+    else:
+        # Make a copy of msgdata so that subsequent changes won't corrupt the
+        # request database.  TBD: remove the `filebase' key since this will
+        # not be relevant when the message is resurrected.
+        msgdata = msgdata.copy()
+    if reason is None:
+        reason = ''
+    # Add the message to the message store.  It is required to have a
+    # Message-ID header.
+    if 'message-id' not in msg:
+        msg['Message-ID'] = make_msgid()
+    seqno = config.db.message_store.add(msg)
+    global_id = '%s/%s' % (msg['X-List-ID-Hash'], seqno)
+    # Prepare the message metadata with some extra information needed only by
+    # the moderation interface.
+    msgdata['_mod_global_id'] = global_id
+    msgdata['_mod_fqdn_listname'] = mlist.fqdn_listname
+    msgdata['_mod_sender'] = msg.get_sender()
+    msgdata['_mod_subject'] = msg.get('subject', _('(no subject)'))
+    msgdata['_mod_reason'] = reason
+    msgdata['_mod_hold_date'] = datetime.now().isoformat()
+    # Now hold this request.  We'll use the message's global ID as the key.
+    requestsdb = config.db.requests.get_list_requests(mlist)
+    request_id = requestsdb.hold_request(
+        RequestType.held_message, global_id, msgdata)
+    return request_id
+
+
+
+def handle_message(mlist, id, action,
+                   comment=None, preserve=False, forward=None):
+    requestdb = config.db.requests.get_list_requests(mlist)
+    key, msgdata = requestdb.get_request(id)
+    # Handle the action.
+    rejection = None
+    global_id = msgdata['_mod_global_id']
+    if action is Action.defer:
+        # Nothing to do, but preserve the message for later.
+        preserve = True
+    elif action is Action.discard:
+        rejection = 'Discarded'
+    elif action is Action.reject:
+        rejection = 'Refused'
+        sender = msgdata['_mod_sender']
+        subject = msgdata['_mod_subject']
+        member = mlist.members.get_member(sender)
+        if member:
+            language = member.preferred_language
+        else:
+            language = None
+        _refuse(mlist, _('Posting of your message titled "$subject"'),
+                sender, comment or _('[No reason given]'), language)
+    elif action is Action.accept:
+        # Start by getting the message from the message store.
+        msg = config.db.message_store.get_message(global_id)
+        # Delete moderation-specific entries from the message metadata.
+        for key in msgdata.keys():
+            if key.startswith('_mod_'):
+                del msgdata[key]
+        # Add some metadata to indicate this message has now been approved.
+        # XXX 'adminapproved' is used for backward compatibility, but it
+        # should really be called 'moderator_approved'.
+        msgdata['approved'] = True
+        msgdata['adminapproved'] = True
+        # Calculate a new filebase for the approved message, otherwise
+        # delivery errors will cause duplicates.
+        if 'filebase' in msgdata:
+            del msgdata['filebase']
+        # Queue the file for delivery by qrunner.  Trying to deliver the
+        # message directly here can lead to a huge delay in web turnaround.
+        # Log the moderation and add a header.
+        msg['X-Mailman-Approved-At'] = formatdate(localtime=True)
+        log.info('held message approved, message-id: %s',
+                 msg.get('message-id', 'n/a'))
+        # Stick the message back in the incoming queue for further
+        # processing.
+        inq = get_switchboard(config.INQUEUE_DIR)
+        inq.enqueue(msg, _metadata=msgdata)
+    else:
+        raise AssertionError('Unexpected action: %s' % action)
+    # Forward the message.
+    if forward:
+        # Get a copy of the original message from the message store.
+        msg = config.db.message_store.get_message(global_id)
+        # It's possible the forwarding address list is a comma separated list
+        # of realname/address pairs.
+        addresses = [addr[1] for addr in getaddresses(forward)]
+        language = mlist.preferred_language
+        if len(addresses) == 1:
+            # If the address getting the forwarded message is a member of
+            # the list, we want the headers of the outer message to be
+            # encoded in their language.  Otherwise it'll be the preferred
+            # language of the mailing list.  This is better than sending a
+            # separate message per recipient.
+            member = mlist.members.get_member(addresses[0])
+            if member:
+                language = member.preferred_language
+        otrans = i18n.get_translation()
+        i18n.set_language(language)
+        try:
+            fmsg = Message.UserNotification(
+                addr, mlist.bounces_address,
+                _('Forward of moderated message'),
+                lang=language)
+        finally:
+            i18n.set_translation(otrans)
+        fmsg.set_type('message/rfc822')
+        fmsg.attach(msg)
+        fmsg.send(mlist)
+    # Delete the message from the message store if it is not being preserved.
+    if not preserve:
+        config.db.message_store.delete_message(global_id)
+        requestdb.delete_request(id)
+    # Log the rejection
+    if rejection:
+        note = """$listname: $rejection posting:
+\tFrom: $sender
+\tSubject: $subject"""
+        if comment:
+            note += '\n\tReason: ' + comment
+        log.info(note)
+
+
+def HoldSubscription(self, addr, fullname, password, digest, lang):
+    # Assure that the database is open for writing
+    self._opendb()
+    # Get the next unique id
+    id = self._next_id
+    # Save the information to the request database. for held subscription
+    # entries, each record in the database will be one of the following
+    # format:
+    #
+    # the time the subscription request was received
+    # the subscriber's address
+    # the subscriber's selected password (TBD: is this safe???)
+    # the digest flag
+    # the user's preferred language
+    data = time.time(), addr, fullname, password, digest, lang
+    self._db[id] = (SUBSCRIPTION, data)
+    #
+    # TBD: this really shouldn't go here but I'm not sure where else is
+    # appropriate.
+    log.info('%s: held subscription request from %s',
+             self.internal_name(), addr)
+    # Possibly notify the administrator in default list language
+    if self.admin_immed_notify:
+        realname = self.real_name
+        subject = _(
+            'New subscription request to list %(realname)s from %(addr)s')
+        text = Utils.maketext(
+            'subauth.txt',
+            {'username'   : addr,
+             'listname'   : self.internal_name(),
+             'hostname'   : self.host_name,
+             'admindb_url': self.GetScriptURL('admindb', absolute=1),
+             }, mlist=self)
+        # This message should appear to come from the <list>-owner so as
+        # to avoid any useless bounce processing.
+        owneraddr = self.GetOwnerEmail()
+        msg = Message.UserNotification(owneraddr, owneraddr, subject, text,
+                                       self.preferred_language)
+        msg.send(self, **{'tomoderators': 1})
+
+def __handlesubscription(self, record, value, comment):
+    stime, addr, fullname, password, digest, lang = record
+    if value == config.DEFER:
+        return DEFER
+    elif value == config.DISCARD:
+        pass
+    elif value == config.REJECT:
+        self._refuse(_('Subscription request'), addr,
+                      comment or _('[No reason given]'),
+                      lang=lang)
+    else:
+        # subscribe
+        assert value == config.SUBSCRIBE
+        try:
+            userdesc = UserDesc(addr, fullname, password, digest, lang)
+            self.ApprovedAddMember(userdesc, whence='via admin approval')
+        except Errors.MMAlreadyAMember:
+            # User has already been subscribed, after sending the request
+            pass
+        # TBD: disgusting hack: ApprovedAddMember() can end up closing
+        # the request database.
+        self._opendb()
+    return REMOVE
+
+def HoldUnsubscription(self, addr):
+    # Assure the database is open for writing
+    self._opendb()
+    # Get the next unique id
+    id = self._next_id
+    # All we need to do is save the unsubscribing address
+    self._db[id] = (UNSUBSCRIPTION, addr)
+    log.info('%s: held unsubscription request from %s',
+             self.internal_name(), addr)
+    # Possibly notify the administrator of the hold
+    if self.admin_immed_notify:
+        realname = self.real_name
+        subject = _(
+            'New unsubscription request from %(realname)s by %(addr)s')
+        text = Utils.maketext(
+            'unsubauth.txt',
+            {'username'   : addr,
+             'listname'   : self.internal_name(),
+             'hostname'   : self.host_name,
+             'admindb_url': self.GetScriptURL('admindb', absolute=1),
+             }, mlist=self)
+        # This message should appear to come from the <list>-owner so as
+        # to avoid any useless bounce processing.
+        owneraddr = self.GetOwnerEmail()
+        msg = Message.UserNotification(owneraddr, owneraddr, subject, text,
+                                       self.preferred_language)
+        msg.send(self, **{'tomoderators': 1})
+
+def _handleunsubscription(self, record, value, comment):
+    addr = record
+    if value == config.DEFER:
+        return DEFER
+    elif value == config.DISCARD:
+        pass
+    elif value == config.REJECT:
+        self._refuse(_('Unsubscription request'), addr, comment)
+    else:
+        assert value == config.UNSUBSCRIBE
+        try:
+            self.ApprovedDeleteMember(addr)
+        except Errors.NotAMemberError:
+            # User has already been unsubscribed
+            pass
+    return REMOVE
+
+
+
+def _refuse(mlist, request, recip, comment, origmsg=None, lang=None):
+    # As this message is going to the requester, try to set the language to
+    # his/her language choice, if they are a member.  Otherwise use the list's
+    # preferred language.
+    realname = mlist.real_name
+    if lang is None:
+        member = mlist.members.get_member(recip)
+        if member:
+            lang = member.preferred_language
+    text = Utils.maketext(
+        'refuse.txt',
+        {'listname' : mlist.fqdn_listname,
+         'request'  : request,
+         'reason'   : comment,
+         'adminaddr': mlist.owner_address,
+        }, lang=lang, mlist=mlist)
+    otrans = i18n.get_translation()
+    i18n.set_language(lang)
+    try:
+        # add in original message, but not wrap/filled
+        if origmsg:
+            text = NL.join(
+                [text,
+                 '---------- ' + _('Original Message') + ' ----------',
+                 str(origmsg)
+                 ])
+        subject = _('Request to mailing list "$realname" rejected')
+    finally:
+        i18n.set_translation(otrans)
+    msg = Message.UserNotification(recip, mlist.bounces_address,
+                                   subject, text, lang)
+    msg.send(mlist)
+
+
+
+def readMessage(path):
+    # For backwards compatibility, we must be able to read either a flat text
+    # file or a pickle.
+    ext = os.path.splitext(path)[1]
+    with open(path) as fp:
+        if ext == '.txt':
+            msg = email.message_from_file(fp, Message.Message)
+        else:
+            assert ext == '.pck'
+            msg = cPickle.load(fp)
+    return msg
+
+
+
+def handle_request(mlist, id, value,
+                   comment=None, preserve=None, forward=None, addr=None):
+    requestsdb = config.db.get_list_requests(mlist)
+    key, data = requestsdb.get_record(id)
+
+    self._opendb()
+    rtype, data = self._db[id]
+    if rtype == HELDMSG:
+        status = self._handlepost(data, value, comment, preserve,
+                                  forward, addr)
+    elif rtype == UNSUBSCRIPTION:
+        status = self._handleunsubscription(data, value, comment)
+    else:
+        assert rtype == SUBSCRIPTION
+        status = self._handlesubscription(data, value, comment)
+    if status <> DEFER:
+        # BAW: Held message ids are linked to Pending cookies, allowing
+        # the user to cancel their post before the moderator has approved
+        # it.  We should probably remove the cookie associated with this
+        # id, but we have no way currently of correlating them. :(
+        del self._db[id]
diff --git a/Mailman/bin/checkdbs.py b/Mailman/bin/checkdbs.py
index 691417780..aab9a7c34 100755
--- a/Mailman/bin/checkdbs.py
+++ b/Mailman/bin/checkdbs.py
@@ -26,6 +26,7 @@ from Mailman import Message
 from Mailman import Utils
 from Mailman import Version
 from Mailman import i18n
+from Mailman.app.requests import handle_request
 from Mailman.configuration import config
 
 _ = i18n._
@@ -62,23 +63,35 @@ def pending_requests(mlist):
     lcset = Utils.GetCharSet(mlist.preferred_language)
     pending = []
     first = True
-    for id in mlist.GetSubscriptionIds():
+    requestsdb = config.db.get_list_requests(mlist)
+    for request in requestsdb.of_type(RequestType.subscription):
         if first:
             pending.append(_('Pending subscriptions:'))
             first = False
-        when, addr, fullname, passwd, digest, lang = mlist.GetRecord(id)
+        key, data = requestsdb.get_request(request.id)
+        when = data['when']
+        addr = data['addr']
+        fullname = data['fullname']
+        passwd = data['passwd']
+        digest = data['digest']
+        lang = data['lang']
         if fullname:
             if isinstance(fullname, unicode):
                 fullname = fullname.encode(lcset, 'replace')
             fullname = ' (%s)' % fullname
         pending.append('    %s%s %s' % (addr, fullname, time.ctime(when)))
     first = True
-    for id in mlist.GetHeldMessageIds():
+    for request in requestsdb.of_type(RequestType.held_message):
         if first:
             pending.append(_('\nPending posts:'))
             first = False
-        info = mlist.GetRecord(id)
-        when, sender, subject, reason, text, msgdata = mlist.GetRecord(id)
+        key, data = requestsdb.get_request(request.id)
+        when = data['when']
+        sender = data['sender']
+        subject = data['subject']
+        reason = data['reason']
+        text = data['text']
+        msgdata = data['msgdata']
         subject = Utils.oneline(subject, lcset)
         date = time.ctime(when)
         reason = _(reason)
@@ -115,11 +128,13 @@ def auto_discard(mlist):
     # Discard old held messages
     discard_count = 0
     expire = config.days(mlist.max_days_to_hold)
-    heldmsgs = mlist.GetHeldMessageIds()
+    requestsdb = config.db.get_list_requests(mlist)
+    heldmsgs = list(requestsdb.of_type(RequestType.held_message))
     if expire and heldmsgs:
-        for id in heldmsgs:
-            if now - mlist.GetRecord(id)[0] > expire:
-                mlist.HandleRequest(id, config.DISCARD)
+        for request in heldmsgs:
+            key, data = requestsdb.get_request(request.id)
+            if now - data['date'] > expire:
+                handle_request(mlist, request.id, config.DISCARD)
                 discard_count += 1
         mlist.Save()
     return discard_count
@@ -136,7 +151,7 @@ def main():
         # The list must be locked in order to open the requests database
         mlist = MailList.MailList(name)
         try:
-            count = mlist.NumRequestsPending()
+            count = config.db.requests.get_list_requests(mlist).count
             # While we're at it, let's evict yesterday's autoresponse data
             midnight_today = Utils.midnight()
             evictions = []
diff --git a/Mailman/database/messagestore.py b/Mailman/database/messagestore.py
index eb29fcfb4..bbaa6976b 100644
--- a/Mailman/database/messagestore.py
+++ b/Mailman/database/messagestore.py
@@ -48,15 +48,11 @@ class MessageStore:
     def add(self, message):
         # Ensure that the message has the requisite headers.
         message_ids = message.get_all('message-id', [])
-        dates = message.get_all('date', [])
-        if not (len(message_ids) == 1 and len(dates) == 1):
-            raise ValueError(
-                'Exactly one Message-ID and one Date header required')
+        if len(message_ids) <> 1:
+            raise ValueError('Exactly one Message-ID header required')
         # Calculate and insert the X-List-ID-Hash.
         message_id = message_ids[0]
-        date = dates[0]
         shaobj = hashlib.sha1(message_id)
-        shaobj.update(date)
         hash32 = base64.b32encode(shaobj.digest())
         del message['X-List-ID-Hash']
         message['X-List-ID-Hash'] = hash32
diff --git a/Mailman/database/model/requests.py b/Mailman/database/model/requests.py
index 59013452b..b0aa0d8d0 100644
--- a/Mailman/database/model/requests.py
+++ b/Mailman/database/model/requests.py
@@ -52,11 +52,22 @@ class ListRequests:
         results = _Request.select_by(mailing_list=self.mailing_list._data)
         return len(results)
 
+    def count_of(self, request_type):
+        results = _Request.select_by(mailing_list=self.mailing_list._data,
+                                     type=request_type)
+        return len(results)
+
     @property
     def held_requests(self):
         results = _Request.select_by(mailing_list=self.mailing_list._data)
         for request in results:
-            yield request.id, request.type
+            yield request
+
+    def of_type(self, request_type):
+        results = _Request.select_by(mailing_list=self.mailing_list._data,
+                                     type=request_type)
+        for request in results:
+            yield request
 
     def hold_request(self, request_type, key, data=None):
         if request_type not in RequestType:
@@ -72,6 +83,15 @@ class ListRequests:
             pendable.update(data)
             token = config.db.pendings.add(pendable, timedelta(days=5000))
             data_hash = token
+        # XXX This would be a good other way to do it, but it causes the
+        # select_by()'s in .count and .held_requests() to fail, even with
+        # flush()'s.
+##         result = _Request.table.insert().execute(
+##             key=key, type=request_type,
+##             mailing_list=self.mailing_list._data,
+##             data_hash=data_hash)
+##         row_id = result.last_inserted_ids()[0]
+##         return row_id
         result = _Request(key=key, type=request_type,
                           mailing_list=self.mailing_list._data,
                           data_hash=data_hash)
diff --git a/Mailman/docs/requests.txt b/Mailman/docs/requests.txt
index 09bce6e3c..f96a12608 100644
--- a/Mailman/docs/requests.txt
+++ b/Mailman/docs/requests.txt
@@ -1,5 +1,5 @@
-Held requests
-=============
+Moderator requests
+==================
 
 Various actions will be held for moderator approval, such as subscriptions to
 closed lists, or postings by non-members.  The requests database is the low
@@ -12,13 +12,30 @@ Here is a helper function for printing out held requests.
 
     >>> def show_holds(requests):
     ...     for request in requests.held_requests:
-    ...         print request[0], str(request[1])
+    ...         key, data = requests.get_request(request.id)
+    ...         if data is not None:
+    ...             data = sorted(data.items())
+    ...         print request.id, str(request.type), key, data
+
+And another helper for displaying messages in the virgin queue.
+
+    >>> from Mailman.Queue.sbcache import get_switchboard
+    >>> virginq = get_switchboard(config.VIRGINQUEUE_DIR)
+    >>> def dequeue(whichq=None):
+    ...     if whichq is None:
+    ...         whichq = virginq
+    ...     assert len(whichq.files) == 1, (
+    ...         'Unexpected file count: %d' % len(whichq.files))
+    ...     filebase = whichq.files[0]
+    ...     qmsg, qdata = whichq.dequeue(filebase)
+    ...     whichq.finish(filebase)
+    ...     return qmsg, qdata
 
 
 Mailing list centric
 --------------------
 
-A set of requests are always centric to a particular mailing list, so given a
+A set of requests are always related to a particular mailing list, so given a
 mailing list you need to get its requests object.
 
     >>> from Mailman.interfaces import IListRequests, IRequests
@@ -62,11 +79,17 @@ And of course, now we can see that there are four requests being held.
 
     >>> requests.count
     4
+    >>> requests.count_of(RequestType.held_message)
+    2
+    >>> requests.count_of(RequestType.subscription)
+    1
+    >>> requests.count_of(RequestType.unsubscription)
+    1
     >>> show_holds(requests)
-    1 RequestType.held_message
-    2 RequestType.subscription
-    3 RequestType.unsubscription
-    4 RequestType.held_message
+    1 RequestType.held_message hold_1 None
+    2 RequestType.subscription hold_2 None
+    3 RequestType.unsubscription hold_3 None
+    4 RequestType.held_message hold_4 None
 
 If we try to hold a request with a bogus type, we get an exception.
 
@@ -85,11 +108,11 @@ We can hold requests with additional data.
     >>> requests.count
     5
     >>> show_holds(requests)
-    1 RequestType.held_message
-    2 RequestType.subscription
-    3 RequestType.unsubscription
-    4 RequestType.held_message
-    5 RequestType.held_message
+    1 RequestType.held_message hold_1 None
+    2 RequestType.subscription hold_2 None
+    3 RequestType.unsubscription hold_3 None
+    4 RequestType.held_message hold_4 None
+    5 RequestType.held_message hold_5 [('bar', 'no'), ('foo', 'yes')]
 
 
 Getting requests
@@ -123,6 +146,25 @@ If we ask for a request that is not in the database, we get None back.
     None
 
 
+Iterating over requests
+-----------------------
+
+To make it easier to find specific requests, the list requests can be iterated
+over by type.
+
+    >>> requests.count_of(RequestType.held_message)
+    3
+    >>> for request in requests.of_type(RequestType.held_message):
+    ...     assert request.type is RequestType.held_message
+    ...     key, data = requests.get_request(request.id)
+    ...     if data is not None:
+    ...         data = sorted(data.items())
+    ...     print request.id, key, data
+    1 hold_1 None
+    4 hold_4 None
+    5 hold_5 [('bar', 'no'), ('foo', 'yes')]
+
+
 Deleting requests
 -----------------
 
@@ -134,10 +176,10 @@ database.
     >>> requests.count
     4
     >>> show_holds(requests)
-    1 RequestType.held_message
-    3 RequestType.unsubscription
-    4 RequestType.held_message
-    5 RequestType.held_message
+    1 RequestType.held_message hold_1 None
+    3 RequestType.unsubscription hold_3 None
+    4 RequestType.held_message hold_4 None
+    5 RequestType.held_message hold_5 [('bar', 'no'), ('foo', 'yes')]
     >>> print requests.get_request(2)
     None
 
@@ -147,3 +189,294 @@ We get an exception if we ask to delete a request that isn't in the database.
     Traceback (most recent call last):
     ...
     KeyError: 801
+
+For the next section, we first clean up all the current requests.
+
+    >>> for request in requests.held_requests:
+    ...     requests.delete_request(request.id)
+    >>> flush()
+    >>> requests.count
+    0
+
+
+Application support
+-------------------
+
+There are several higher level interfaces available in the Mailman.app package
+which can be used to hold messages, subscription, and unsubscriptions.  There
+are also interfaces for disposing of these requests in an application specific
+and consistent way.
+
+    >>> from Mailman.app import moderator
+
+
+Holding messages
+----------------
+
+For this section, we need a mailing list and at least one message.
+
+    >>> mlist = config.db.list_manager.create('alist@example.com')
+    >>> mlist.preferred_language = 'en'
+    >>> mlist.real_name = 'A Test List'
+    >>> flush()
+    >>> from email import message_from_string
+    >>> from Mailman.Message import Message
+    >>> msg = message_from_string("""\
+    ... From: aperson@example.org
+    ... To: alist@example.com
+    ... Subject: Something important
+    ...
+    ... Here's something important about our mailing list.
+    ... """, Message)
+
+Holding a message means keeping a copy of it that a moderator must approve
+before the message is posted to the mailing list.  To hold the message, you
+must supply the message, message metadata, and a text reason for the hold.  In
+this case, we won't include any additional metadata.
+
+    >>> id_1 = moderator.hold_message(mlist, msg, {}, 'Needs approval')
+    >>> flush()
+    >>> requests.get_request(id_1) is not None
+    True
+
+We can also hold a message with some additional metadata.
+
+    >>> msgdata = dict(sender='aperson@example.com',
+    ...                approved=True,
+    ...                received_time=123.45)
+    >>> id_2 = moderator.hold_message(mlist, msg, msgdata, 'Feeling ornery')
+    >>> flush()
+    >>> requests.get_request(id_2) is not None
+    True
+
+Once held, the moderator can select one of several dispositions.  The most
+trivial is to simply defer a decision for now.
+
+    >>> from Mailman.constants import Action
+    >>> moderator.handle_message(mlist, id_1, Action.defer)
+    >>> flush()
+    >>> requests.get_request(id_1) is not None
+    True
+
+The moderator can also discard the message.  This is often done with spam.
+Bye bye message!
+
+    >>> moderator.handle_message(mlist, id_1, Action.discard)
+    >>> flush()
+    >>> print requests.get_request(id_1)
+    None
+    >>> virginq.files
+    []
+
+The message can be rejected, meaning it is bounced back to the sender.
+
+    >>> moderator.handle_message(mlist, id_2, Action.reject, 'Off topic')
+    >>> flush()
+    >>> print requests.get_request(id_2)
+    None
+    >>> qmsg, qdata = dequeue()
+    >>> print qmsg.as_string()
+    MIME-Version: 1.0
+    Content-Type: text/plain; charset="us-ascii"
+    Content-Transfer-Encoding: 7bit
+    Subject: Request to mailing list "A Test List" rejected
+    From: alist-bounces@example.com
+    To: aperson@example.org
+    Message-ID: ...
+    Date: ...
+    Precedence: bulk
+    <BLANKLINE>
+    Your request to the alist@example.com mailing list
+    <BLANKLINE>
+        Posting of your message titled "Something important"
+    <BLANKLINE>
+    has been rejected by the list moderator.  The moderator gave the
+    following reason for rejecting your request:
+    <BLANKLINE>
+    "Off topic"
+    <BLANKLINE>
+    Any questions or comments should be directed to the list administrator
+    at:
+    <BLANKLINE>
+        alist-owner@example.com
+    <BLANKLINE>
+    >>> sorted(qdata.items())
+    [('_parsemsg', False),
+     ('listname', 'alist@example.com'),
+     ('nodecorate', True),
+     ('received_time', ...),
+     ('recips', ['aperson@example.org']),
+     ('reduced_list_headers', True),
+     ('version', 3)]
+
+Or the message can be approved.  This actually places the message back into
+the incoming queue for further processing, however the message metadata
+indicates that the message has been approved.
+
+    >>> id_3 = moderator.hold_message(mlist, msg, msgdata, 'Needs approval')
+    >>> flush()
+    >>> moderator.handle_message(mlist, id_3, Action.accept)
+    >>> flush()
+    >>> inq = get_switchboard(config.INQUEUE_DIR)
+    >>> qmsg, qdata = dequeue(inq)
+    >>> print qmsg.as_string()
+    From: aperson@example.org
+    To: alist@example.com
+    Subject: Something important
+    Message-ID: ...
+    X-List-ID-Hash: ...
+    X-List-Sequence-Number: ...
+    X-Mailman-Approved-At: ...
+    <BLANKLINE>
+    Here's something important about our mailing list.
+    <BLANKLINE>
+    >>> sorted(qdata.items())
+    [('_parsemsg', False),
+     ('adminapproved', True), ('approved', True),
+     ('received_time', ...), ('sender', 'aperson@example.com'),
+     ('version', 3)]
+
+In addition to any of the above dispositions, the message can also be
+preserved for further study.  Ordinarily the message is removed from the
+global message store after its disposition (though approved messages may be
+re-added to the message store).  When handling a message, we can tell the
+moderator interface to also preserve a copy, essentially telling it not to
+delete the message from the storage.  First, without the switch, the message
+is deleted.
+
+    >>> msg = message_from_string("""\
+    ... From: aperson@example.org
+    ... To: alist@example.com
+    ... Subject: Something important
+    ... Message-ID: <12345>
+    ...
+    ... Here's something important about our mailing list.
+    ... """, Message)
+    >>> id_4 = moderator.hold_message(mlist, msg, {}, 'Needs approval')
+    >>> flush()
+    >>> moderator.handle_message(mlist, id_4, Action.discard)
+    >>> flush()
+    >>> msgs = config.db.message_store.get_messages_by_message_id('<12345>')
+    >>> list(msgs)
+    []
+
+But if we ask to preserve the message when we discard it, it will be held in
+the message store after disposition.
+
+    >>> id_4 = moderator.hold_message(mlist, msg, {}, 'Needs approval')
+    >>> flush()
+    >>> moderator.handle_message(mlist, id_4, Action.discard, preserve=True)
+    >>> flush()
+    >>> msgs = config.db.message_store.get_messages_by_message_id('<12345>')
+    >>> msgs = list(msgs)
+    >>> len(msgs)
+    1
+    >>> print msgs[0].as_string()
+    From: aperson@example.org
+    To: alist@example.com
+    Subject: Something important
+    Message-ID: <12345>
+    X-List-ID-Hash: 4CF7EAU3SIXBPXBB5S6PEUMO62MWGQN6
+    X-List-Sequence-Number: 1
+    <BLANKLINE>
+    Here's something important about our mailing list.
+    <BLANKLINE>
+
+Orthogonal to preservation, the message can also be forwarded to another
+address.  This is helpful for getting the message into the inbox of one of the
+moderators.
+
+    >>> id_4 = moderator.hold_message(mlist, msg, {}, 'Needs approval')
+    >>> flush()
+    >>> moderator.handle_message(mlist, id_4, Action.discard,
+    ...                          forward=['zperson@example.com'])
+    >>> flush()
+    >>> qmsg, qdata = dequeue()
+    >>> print qmsg.as_string()
+    XXX
+    >>> sorted(qdata.items())
+    XXX
+
+
+Holding subscription requests
+-----------------------------
+
+For closed lists, subscription requests will also be held for moderator
+approval.  In this case, several pieces of information related to the
+subscription must be provided, including the subscriber's address and real
+name, their password (possibly hashed), what kind of delivery option they are
+chosing and their preferred language.
+
+    >>> from Mailman.constants import DeliveryMode
+    >>> mlist.admin_immed_notify = False
+    >>> flush()
+    >>> id_3 = moderator.hold_subscription(mlist,
+    ...     'bperson@example.org', 'Ben Person',
+    ...     '{NONE}abcxyz', DeliveryMode.regular, 'en')
+    >>> flush()
+    >>> requests.get_request(id_3) is not None
+    True
+
+In the above case the mailing list was not configured to send the list
+moderators a notice about the hold, so no email message is in the virgin
+queue.
+
+    >>> virginq.files
+    []
+
+But if we set the list up to notify the list moderators immediately when a
+message is held for approval, there will be a message placed in the virgin
+queue when the message is held.
+
+    >>> mlist.admin_immed_notify = True
+    >>> flush()
+    >>> id_4 = moderator.hold_subscription(mlist,
+    ...     'cperson@example.org', 'Claire Person',
+    ...     '{NONE}zyxcba, DeliveryMode.regular, 'en')
+    >>> flush()
+    >>> requests.get_request(id_4) is not None
+    True
+    >>> qmsg, qdata = dequeue()
+    >>> print qmsg.as_string()
+    XXX
+    >>> sorted(qdata.items())
+    XXX
+
+
+Holding unsubscription requests
+-------------------------------
+
+Some lists, though it is rare, require moderator approval for unsubscriptions.
+In this case, only the unsubscribing address is required.  Like subscriptions,
+unsubscription holds can send the list's moderators an immediate notification.
+
+    >>> mlist.admin_immed_notify = False
+    >>> flush()
+    >>> from Mailman.constants import MemberRole
+    >>> user_1 = config.db.user_manager.create_user('dperson@example.com')
+    >>> flush()
+    >>> address_1 = list(user_1.addresses)[0]
+    >>> address_1.subscribe(mlist, MemberRole.member)
+    <Member: <dperson@example.com> on
+             _xtest@example.com as MemberRole.member>
+    >>> user_2 = config.db.user_manager.create_user('eperson@example.com')
+    >>> flush()
+    >>> address_2 = list(user_2.addresses)[0]
+    >>> address_2.subscribe(mlist, MemberRole.member)
+    <Member: <eperson@example.com> on
+             _xtest@example.com as MemberRole.member>
+    >>> flush()
+    >>> id_5 = moderator.hold_unsubscription(mlist, 'dperson@example.com')
+    >>> flush()
+    >>> requests.get_request(id_5) is not None)
+    True
+    >>> virginq.files
+    []
+    >>> mlist.admin_immed_notify = True
+    >>> id_6 = moderator.hold_unsubscription(mlist, 'eperson@example.com')
+    >>> flush()
+    >>> qmsg, qdata = dequeue()
+    >>> print qmsg.as_string()
+    XXX
+    >>> sorted(qdata.items())
+    XXX
diff --git a/Mailman/interfaces/requests.py b/Mailman/interfaces/requests.py
index 0a817d43c..b4b5902af 100644
--- a/Mailman/interfaces/requests.py
+++ b/Mailman/interfaces/requests.py
@@ -42,10 +42,17 @@ class IListRequests(Interface):
     count = Attribute(
         """The total number of requests held for the mailing list.""")
 
+    def count_of(request_type):
+        """The total number of requests held of the given request type.
+
+        :param request_type: A `RequestType` enum value.
+        :return: An integer.
+        """
+
     def hold_request(request_type, key, data=None):
         """Hold some data for moderator approval.
 
-        :param request_type: A `Request` enum value.
+        :param request_type: A `RequestType` enum value.
         :param key: The key piece of request data being held.
         :param data: Additional optional data in the form of a dictionary that
             is associated with the held request.
@@ -53,12 +60,23 @@ class IListRequests(Interface):
         """
 
     held_requests = Attribute(
-        """An iterator over the held requests, yielding a 2-tuple.
+        """An iterator over the held requests.
 
-        The tuple has the form: (id, type) where `id` is the held request's
-        unique id and the `type` is a `Request` enum value.
+        Returned items have two attributes:
+         * `id` is the held request's unique id;
+         * `type` is a `RequestType` enum value.
         """)
 
+    def of_type(request_type):
+        """An iterator over the held requests of the given type.
+
+        Returned items have two  attributes:
+         * `id` is the held request's unique id;
+         * `type` is a `RequestType` enum value.
+
+         Only items with a matching `type' are returned.
+         """
+
     def get_request(request_id):
         """Get the data associated with the request id, or None.