Massive rewrite, for which I'm too tired to detail (yeah, I'll regret

this five years from now ;).  In brief,

We now organize held postings by sender address and this "summary" is
what's presented when .../mailman/admindb/listname is visited.  The
pending subs and unsubs don't change, but the preamble is shorted
(with links to more detailed instructions).

For each group of held messages, you can click on one link to see the
details of the all the messages sent by a single address, or you can
click on a different link to view the details of just a single
message.  A third link lets you view the details of all the held
messages, just like the old admindb page used to give you.

The details page is just like the old admindb in form and function,
except that ADMINDB_PAGE_TEXT_LIMIT now works ;) and there are links
back to the summary page.

One additional neat hack is that on the summary page, there's an
option to add an email address to one of the auto sender filter lists
(viewable in the admin/privacy/sender screen).  This is only visible
if the address isn't already on one of the four filter lists.

Where this should be a boon is if you've got a bunch of messages being
held that are coming from the same address, and they all appear to be
spam.  You can discard them all in one fell swoop (no click-'n'-scroll
necessary!), and add them to the auto-discard list, so you never have
to worry about them again.

Note that if what you wanted to do was add a regexp to say the
auto-discard list, you'd have to first add this address, then go to
the admin/privacy/sender page and edit the address into a regexp
filter.  Easily done, while reducing the complexity of the admindb
summary page, which already pushes the edge of "too busy".

1 files changed, 370 insertions, 90 deletions

diff --git a/Mailman/Cgi/admindb.py b/Mailman/Cgi/admindb.py
index 19474d1e3..c574bedad 100644
--- a/Mailman/Cgi/admindb.py
+++ b/Mailman/Cgi/admindb.py
@@ -18,11 +18,12 @@
 
 import sys
 import os
-import types
 import cgi
 import errno
 import signal
 import email
+from types import ListType
+from urllib import quote_plus, unquote_plus
 
 from Mailman import mm_cfg
 from Mailman import Utils
@@ -45,6 +46,28 @@ i18n.set_language(mm_cfg.DEFAULT_SERVER_LANGUAGE)
 
 
 
+def helds_by_sender(mlist):
+    heldmsgs = mlist.GetHeldMessageIds()
+    bysender = {}
+    for id in heldmsgs:
+        sender = mlist.GetRecord(id)[1]
+        bysender.setdefault(sender, []).append(id)
+    return bysender
+
+
+def hacky_radio_buttons(btnname, labels, values, defaults, spacing=3):
+    # We can't use a RadioButtonArray here because horizontal placement can be
+    # confusing to the user and vertical placement takes up too much
+    # real-estate.  This is a hack!
+    space = ' ' * spacing
+    btns = Table(cellspacing='5', cellpadding='0')
+    btns.AddRow([space + text + space for text in labels])
+    btns.AddRow([Center(RadioButton(btnname, value, default))
+                 for value, default in zip(values, defaults)])
+    return btns
+
+
+
 def main():
     # Figure out which list is being requested
     parts = Utils.GetPathPieces()
@@ -84,6 +107,25 @@ def main():
     doc = Document()
     doc.set_language(mlist.preferred_language)
     
+    # See if we're requesting all the messages for a particular sender, or if
+    # we want a specific held message.
+    sender = None
+    msgid = None
+    details = None
+    envar = os.environ.get('QUERY_STRING')
+    if envar:
+        # POST methods, even if their actions have a query string, don't get
+        # put into FieldStorage's keys :-(
+        qs = cgi.parse_qs(envar).get('sender')
+        if qs and type(qs) == ListType:
+            sender = qs[0]
+        qs = cgi.parse_qs(envar).get('msgid')
+        if qs and type(qs) == ListType:
+            msgid = qs[0]
+        qs = cgi.parse_qs(envar).get('details')
+        if qs and type(qs) == ListType:
+            details = qs[0]
+
     # We need a signal handler to catch the SIGTERM that can come from Apache
     # when the user hits the browser's STOP button.  See the comment in
     # admin.py for details.
@@ -110,18 +152,80 @@ def main():
             # If this is not a form submission (i.e. there are no keys in the
             # form), then all we don't need to do much special.
             doc.SetTitle(_('%(realname)s Administrative Database'))
-        else:
+        elif not details:
             # This is a form submission
             doc.SetTitle(_('%(realname)s Administrative Database Results'))
             process_form(mlist, doc, cgidata)
         # Now print the results and we're done
-        show_requests(mlist, doc)
+        # Short circuit for when there are no pending requests
+        if not mlist.NumRequestsPending():
+            title = _('%(realname)s Administrative Database')
+            doc.SetTitle(title)
+            doc.AddItem(Header(2, title))
+            doc.AddItem(_('There are no pending requests.'))
+            doc.AddItem(mlist.GetMailmanFooter())
+            print doc.Format()
+            return
+
+        admindburl = mlist.GetScriptURL('admindb', absolute=1)
+        form = Form(admindburl)
+        # Add the instructions template
+        if details:
+            doc.AddItem(Header(
+                2, _('Detailed instructions for the administrative database')))
+        else:
+            doc.AddItem(Header(
+                2,
+                _('Administrative requests for mailing list:')
+                + ' <em>%s</em>' % mlist.real_name))
+        if not details:
+            form.AddItem(Center(SubmitButton('submit', _('Submit All Data'))))
+        # Add a link back to the overview, if we're not viewing the overview!
+        adminurl = mlist.GetScriptURL('admin', absolute=1)
+        d = {'listname'  : mlist.real_name,
+             'detailsurl': admindburl + '?details=instructions',
+             'summaryurl': admindburl,
+             'viewallurl': admindburl + '?details=all',
+             'adminurl'  : adminurl,
+             'filterurl' : adminurl + '/privacy/sender',
+             }
+        if sender:
+            d['description'] = _("all the %(esender)s's held messages.")
+            doc.AddItem(Utils.maketext('admindbpreamble.html', d,
+                                       raw=1, mlist=mlist))
+            show_sender_requests(mlist, form, sender)
+        elif msgid:
+            d['description'] = _('a single held message.')
+            doc.AddItem(Utils.maketext('admindbpreamble.html', d,
+                                       raw=1, mlist=mlist))
+            show_message_requests(mlist, form, msgid)
+        elif details == 'all':
+            d['description'] = _('all held messages.')
+            doc.AddItem(Utils.maketext('admindbpreamble.html', d,
+                                       raw=1, mlist=mlist))
+            show_detailed_requests(mlist, form)
+        elif details == 'instructions':
+            doc.AddItem(Utils.maketext('admindbdetails.html', d,
+                                       raw=1, mlist=mlist))
+        else:
+            # Show a summary of all requests
+            doc.AddItem(Utils.maketext('admindbsummary.html', d,
+                                       raw=1, mlist=mlist))
+            show_pending_subs(mlist, form)
+            show_pending_unsubs(mlist, form)
+            show_helds_overview(mlist, form)
+        # Finish up the document, adding buttons to the form
+        if not details:
+            doc.AddItem(form)
+            form.AddItem('<hr>')
+            form.AddItem(Center(SubmitButton('submit', _('Submit All Data'))))
+        doc.AddItem(mlist.GetMailmanFooter())
+        print doc.Format()
+        # Commit all changes
         mlist.Save()
     finally:
         mlist.Unlock()
 
-    print doc.Format()
-
 
 
 def handle_no_list(msg=''):
@@ -142,94 +246,211 @@ def handle_no_list(msg=''):
 
 
 
-def show_requests(mlist, doc):
-    # Print all the requests outstanding in the database.  The only ones we
-    # know about are subscription and post requests.  Anything else that might
-    # have gotten in here somehow we'll just ignore (This should never happen
-    # unless someone is hacking at the code).
-    doc.AddItem(Header(2, _('Administrative requests for mailing list:')
-                       + ' <em>%s</em>' % mlist.real_name))
-    # Short circuit for when there are no pending requests
-    if not mlist.NumRequestsPending():
-        doc.AddItem(_('There are no pending requests.'))
-        doc.AddItem(mlist.GetMailmanFooter())
-        return
-
-    # Add the preamble template
-    doc.AddItem(Utils.maketext(
-        'admindbpreamble.html',
-        {'listname': mlist.real_name},
-        raw=1, mlist=mlist))
-
-    # Form submits back to this script
-    form = Form(mlist.GetScriptURL('admindb'))
-    doc.AddItem(form)
-    form.AddItem(SubmitButton('submit', _('Submit All Data')))
+def show_pending_subs(mlist, form):
     # Add the subscription request section
     pendingsubs = mlist.GetSubscriptionIds()
-    if pendingsubs:
-        form.AddItem('<hr>')
-        form.AddItem(Center(Header(2, _('Subscription Requests'))))
-        table = Table(border=2)
-        table.AddRow([Center(Bold(_('User address/name'))),
-                      Center(Bold(_('Your decision'))),
-                      Center(Bold(_('Reason for refusal')))
+    if not pendingsubs:
+        return
+    form.AddItem('<hr>')
+    form.AddItem(Center(Header(2, _('Subscription Requests'))))
+    table = Table(border=2)
+    table.AddRow([Center(Bold(_('User address/name'))),
+                  Center(Bold(_('Your decision'))),
+                  Center(Bold(_('Reason for refusal')))
+                  ])
+    for id in pendingsubs:
+        time, addr, fullname, passwd, digest, lang = mlist.GetRecord(id)
+        table.AddRow(['%s<br><em>%s</em>' % (addr, fullname),
+                      RadioButtonArray(id, (_('Defer'),
+                                            _('Approve'),
+                                            _('Reject'),
+                                            _('Discard')),
+                                       values=(mm_cfg.DEFER,
+                                               mm_cfg.SUBSCRIBE,
+                                               mm_cfg.REJECT,
+                                               mm_cfg.DISCARD),
+                                       checked=0),
+                      TextBox('comment-%d' % id, size=45)
                       ])
-        for id in pendingsubs:
-            time, addr, fullname, passwd, digest, lang = mlist.GetRecord(id)
-            table.AddRow(['%s<br><em>%s</em>' % (addr, fullname),
-                          RadioButtonArray(id, (_('Defer'),
-                                                _('Approve'),
-                                                _('Reject'),
-                                                _('Discard')),
-                                           values=(mm_cfg.DEFER,
-                                                   mm_cfg.SUBSCRIBE,
-                                                   mm_cfg.REJECT,
-                                                   mm_cfg.DISCARD),
-                                           checked=0),
-                          TextBox('comment-%d' % id, size=45)
-                          ])
-        form.AddItem(table)
+    form.AddItem(table)
+
+    
+
+def show_pending_unsubs(mlist, form):
     # Add the pending unsubscription request section
     pendingunsubs = mlist.GetUnsubscriptionIds()
-    if pendingunsubs:
-        form.AddItem('<hr>')
-        form.AddItem(Center(Header(2, _('Unsubscription Requests'))))
-        table = Table(border=2)
-        table.AddRow([Center(Bold(_('User address/name'))),
-                      Center(Bold(_('Your decision'))),
-                      Center(Bold(_('Reason for refusal')))
+    if not pendingunsubs:
+        return
+    form.AddItem('<hr>')
+    form.AddItem(Center(Header(2, _('Unsubscription Requests'))))
+    table = Table(border=2)
+    table.AddRow([Center(Bold(_('User address/name'))),
+                  Center(Bold(_('Your decision'))),
+                  Center(Bold(_('Reason for refusal')))
+                  ])
+    for id in pendingunsubs:
+        addr = mlist.GetRecord(id)
+        fullname = mlist.getMemberName(addr)
+        if fullname is None:
+            fullname = ''
+        table.AddRow(['%s<br><em>%s</em>' % (addr, fullname),
+                      RadioButtonArray(id, (_('Defer'),
+                                            _('Approve'),
+                                            _('Reject'),
+                                            _('Discard')),
+                                       values=(mm_cfg.DEFER,
+                                               mm_cfg.UNSUBSCRIBE,
+                                               mm_cfg.REJECT,
+                                               mm_cfg.DISCARD),
+                                       checked=0),
+                      TextBox('comment-%d' % id, size=45)
                       ])
-        for id in pendingunsubs:
-            addr = mlist.GetRecord(id)
-            fullname = mlist.getMemberName(addr)
-            if fullname is None:
-                fullname = ''
-            table.AddRow(['%s<br><em>%s</em>' % (addr, fullname),
-                          RadioButtonArray(id, (_('Defer'),
-                                                _('Approve'),
-                                                _('Reject'),
-                                                _('Discard')),
-                                           values=(mm_cfg.DEFER,
-                                                   mm_cfg.UNSUBSCRIBE,
-                                                   mm_cfg.REJECT,
-                                                   mm_cfg.DISCARD),
-                                           checked=0),
-                          TextBox('comment-%d' % id, size=45)
-                          ])
-        form.AddItem(table)
-    # Post holds are handled differently
-    heldmsgs = mlist.GetHeldMessageIds()
-    total = len(heldmsgs)
-    if total:
-        count = 1
-        for id in heldmsgs:
+    form.AddItem(table)
+
+
+
+def show_helds_overview(mlist, form):
+    # Sort the held messages by sender
+    bysender = helds_by_sender(mlist)
+    if not bysender:
+        return
+    # Add the by-sender overview tables
+    admindburl = mlist.GetScriptURL('admindb', absolute=1)
+    table = Table(border=0)
+    form.AddItem(table)
+    senders = bysender.keys()
+    senders.sort()
+    for sender in senders:
+        qsender = quote_plus(sender)
+        esender = cgi.escape(sender)
+        senderurl = admindburl + '?sender=' + qsender
+        # The encompassing sender table
+        stable = Table(border=1)
+        stable.AddRow([Center(Bold(_('From:')).Format() + esender)])
+        stable.AddCellInfo(stable.GetCurrentRowIndex(), 0, colspan=2)
+        left = Table(border=0)
+        left.AddRow([_('Action to take on all these held messages:')])
+        left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
+        btns = hacky_radio_buttons(
+            'senderaction-' + qsender,
+            (_('Defer'), _('Accept'), _('Reject'), _('Discard')),
+            (mm_cfg.DEFER, mm_cfg.APPROVE, mm_cfg.REJECT, mm_cfg.DISCARD),
+            (1, 0, 0, 0))
+        left.AddRow([btns])
+        left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
+        left.AddRow([
+            CheckBox('senderpreserve-' + qsender, 1).Format() +
+            '&nbsp;' +
+            _('Preserve messages for the site administrator')
+            ])
+        left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
+        left.AddRow([
+            CheckBox('senderforward-' + qsender, 1).Format() +
+            '&nbsp;' +
+            _('Forward messages (individually) to:')
+            ])
+        left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
+        left.AddRow([
+            TextBox('senderforwardto-' + qsender,
+                    value=mlist.GetOwnerEmail())
+            ])
+        left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
+        # Ask if this address should be added to one of the sender filters,
+        # but only if it isn't already in one of the filters.
+        if sender not in (mlist.accept_these_nonmembers +
+                          mlist.hold_these_nonmembers +
+                          mlist.reject_these_nonmembers +
+                          mlist.discard_these_nonmembers):
+            left.AddRow([
+                CheckBox('senderfilterp-' + qsender, 1).Format() +
+                '&nbsp;' +
+                _('Add <b>%(esender)s</b> to a sender filter')
+                ])
+            left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
+            btns = hacky_radio_buttons(
+                'senderfilter-' + qsender,
+                (_('Accepts'), _('Holds'), _('Rejects'), _('Discards')),
+                (mm_cfg.ACCEPT, mm_cfg.HOLD, mm_cfg.REJECT, mm_cfg.DISCARD),
+                (0, 0, 0, 1))
+            left.AddRow([btns])
+            left.AddCellInfo(left.GetCurrentRowIndex(), 0, colspan=2)
+        right = Table(border=0)
+        right.AddRow([
+            _("""Click on the message number to view the individual
+            message, or you can """) +
+            Link(senderurl, _('view all messages from %(esender)s')).Format()
+            ])
+        right.AddCellInfo(right.GetCurrentRowIndex(), 0, colspan=2)
+        right.AddRow(['&nbsp;', '&nbsp;'])
+        counter = 1
+        for id in bysender[sender]:
             info = mlist.GetRecord(id)
-            show_post_requests(mlist, id, info, total, count, form)
-            count += 1
-    form.AddItem('<hr>')
-    form.AddItem(SubmitButton('submit', _('Submit All Data')))
-    doc.AddItem(mlist.GetMailmanFooter())
+            if len(info) == 5:
+                ptime, sender, subject, reason, filename = info
+                msgdata = {}
+            else:
+                ptime, sender, subject, reason, filename, msgdata = info
+            # BAW: This is really the size of the message pickle, which should
+            # be close, but won't be exact.  Sigh, good enough.
+            try:
+                size = os.path.getsize(os.path.join(mm_cfg.DATA_DIR, filename))
+            except OSError, e:
+                if e.errno <> errno.ENOENT: raise
+                # This message must have gotten lost, i.e. it's already been
+                # handled by the time we got here.
+                mlist.HandleRequest(id, mm_cfg.DISCARD)
+                continue
+            t = Table(border=0)
+            t.AddRow([Link(admindburl + '?msgid=%d' % id, '[%d]' % counter),
+                      Bold(_('Subject:')),
+                      cgi.escape(subject)
+                      ])
+            t.AddRow(['&nbsp;', Bold(_('Size:')), str(size) + _(' bytes')])
+            t.AddRow(['&nbsp;', Bold(_('Reason:')),
+                      reason or _('not available')])
+            counter += 1
+            right.AddRow([t])
+        stable.AddRow([left, right])
+        table.AddRow([stable])
+
+
+
+def show_sender_requests(mlist, form, sender):
+    bysender = helds_by_sender(mlist)
+    if not bysender:
+        return
+    sender_ids = bysender.get(sender)
+    if sender_ids is None:
+        # BAW: should we print an error message?
+        return
+    total = len(sender_ids)
+    count = 1
+    for id in sender_ids:
+        info = mlist.GetRecord(id)
+        show_post_requests(mlist, id, info, total, count, form)
+        count += 1
+
+
+
+def show_message_requests(mlist, form, id):
+    try:
+        id = int(id)
+        info = mlist.GetRecord(id)
+    except (ValueError, KeyError):
+        # BAW: print an error message?
+        return
+    show_post_requests(mlist, id, info, 1, 1, form)
+
+
+
+def show_detailed_requests(mlist, form):
+    all = mlist.GetHeldMessageIds()
+    total = len(all)
+    count = 1
+    for id in mlist.GetHeldMessageIds():
+        info = mlist.GetRecord(id)
+        show_post_requests(mlist, id, info, total, count, form)
+        count += 1
 
 
 
@@ -277,12 +498,18 @@ def show_post_requests(mlist, id, info, total, count, form):
     # Get the header text and the message body excerpt
     lines = []
     chars = 0
+    # A negative value means, include the entire message regardless of size
+    limit = mm_cfg.ADMINDB_PAGE_TEXT_LIMIT
     for line in email.Iterators.body_line_iterator(msg):
         lines.append(line)
         chars += len(line)
-        if chars > mm_cfg.ADMINDB_PAGE_TEXT_LIMIT:
+        if chars > limit > 0:
             break
-    body = EMPTYSTRING.join(lines)[:mm_cfg.ADMINDB_PAGE_TEXT_LIMIT]
+    # Negative values mean display the entire message, regardless of size
+    if limit > 0:
+        body = EMPTYSTRING.join(lines)[:mm_cfg.ADMINDB_PAGE_TEXT_LIMIT]
+    else:
+        body = EMPTYSTRING.join(lines)
     hdrtxt = NL.join(['%s: %s' % (k, v) for k, v in msg.items()])
 
     # Okay, we've reconstituted the message just fine.  Now for the fun part!
@@ -341,17 +568,70 @@ def show_post_requests(mlist, id, info, total, count, form):
 
 
 def process_form(mlist, doc, cgidata):
-    # Process the form and make updates to the admin database.
+    senderactions = {}
+    # Sender-centric actions
+    for k in cgidata.keys():
+        for prefix in ('senderaction-', 'senderpreserve-', 'senderforward-',
+                       'senderforwardto-', 'senderfilterp-', 'senderfilter-'):
+            if k.startswith(prefix):
+                action = k[:len(prefix)-1]
+                sender = unquote_plus(k[len(prefix):])
+                value = cgidata.getvalue(k)
+                senderactions.setdefault(sender, {})[action] = value
+    for sender in senderactions.keys():
+        actions = senderactions[sender]
+        # Handle what to do about all this sender's held messages
+        try:
+            action = int(actions.get('senderaction', mm_cfg.DEFER))
+        except ValueError:
+            action = mm_cfg.DEFER
+        if action in (mm_cfg.DEFER, mm_cfg.APPROVE,
+                      mm_cfg.REJECT, mm_cfg.DISCARD):
+            preserve = actions.get('senderpreserve', 0)
+            forward = actions.get('senderforward', 0)
+            forwardaddr = actions.get('senderforwardto', '')
+            comment = _('No reason given')
+            bysender = helds_by_sender(mlist)
+            for id in bysender.get(sender, []):
+                try:
+                    mlist.HandleRequest(id, action, comment, preserve,
+                                        forward, forwardaddr)
+                except (KeyError, Errors.LostHeldMessage):
+                    # That's okay, it just means someone else has already
+                    # updated the database while we were staring at the page,
+                    # so just ignore it
+                    continue
+        # Now see if this sender should be added to one of the nonmember
+        # sender filters.
+        if actions.get('senderfilterp', 0):
+            try:
+                which = int(actions.get('senderfilter'))
+            except ValueError:
+                # Bogus form
+                which = 'ignore'
+            if which == mm_cfg.ACCEPT:
+                mlist.accept_these_nonmembers.append(sender)
+            elif which == mm_cfg.HOLD:
+                mlist.hold_these_nonmembers.append(sender)
+            elif which == mm_cfg.REJECT:
+                mlist.reject_these_nonmembers.append(sender)
+            elif which == mm_cfg.DISCARD:
+                mlist.discard_these_nonmembers.append(sender)
+            # Otherwise, it's a bogus form, so ignore it
+    # Now, do message specific actions
     erroraddrs = []
     for k in cgidata.keys():
         formv = cgidata[k]
-        if type(formv) == types.ListType:
+        if type(formv) == ListType:
             continue
         try:
             v = int(formv.value)
             request_id = int(k)
         except ValueError:
             continue
+        if v not in (mm_cfg.DEFER, mm_cfg.APPROVE,
+                     mm_cfg.REJECT, mm_cfg.DISCARD):
+            continue
         # Get the action comment and reasons if present.
         commentkey = 'comment-%d' % request_id
         preservekey = 'preserve-%d' % request_id