Move all cron scripts to the new Mailman.bin package layout and complete the

conversion to optparse style option parsing.  Remove mailpasswds as password
reminders will go away for MM2.2.

6 files changed, 894 insertions, 0 deletions

diff --git a/Mailman/bin/bumpdigests.py b/Mailman/bin/bumpdigests.py
new file mode 100644
index 000000000..fe63410d6
--- /dev/null
+++ b/Mailman/bin/bumpdigests.py
@@ -0,0 +1,73 @@
+# Copyright (C) 1998-2006 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+
+import sys
+import optparse
+
+from Mailman import Errors
+from Mailman import MailList
+from Mailman import Utils
+from Mailman import mm_cfg
+from Mailman.i18n import _
+
+# Work around known problems with some RedHat cron daemons
+import signal
+signal.signal(signal.SIGCHLD, signal.SIG_DFL)
+
+__i18n_templates__ = True
+
+
+
+def parseargs():
+    parser = optparse.OptionParser(version=mm_cfg.MAILMAN_VERSION,
+                                   usage=_("""\
+%prog [options] [listname ...]
+
+Increment the digest volume number and reset the digest number to one.  All
+the lists named on the command line are bumped.  If no list names are given,
+all lists are bumped."""))
+    opts, args = parser.parse_args()
+    return opts, args, parser
+
+
+
+def main():
+    opts, args, parser = parseargs()
+
+    listnames = set(args or Utils.list_names())
+    if not listnames:
+        print _('Nothing to do.')
+        sys.exit(0)
+
+    for listname in listnames:
+        try:
+            # Be sure the list is locked
+            mlist = MailList.MailList(listname)
+        except Errors.MMListError, e:
+            parser.print_help()
+            print >> sys.stderr, _('No such list: $listname')
+            sys.exit(1)
+        try:
+            mlist.bump_digest_volume()
+        finally:
+            mlist.Save()
+            mlist.Unlock()
+
+
+
+if __name__ == '__main__':
+    main()
diff --git a/Mailman/bin/checkdbs.py b/Mailman/bin/checkdbs.py
new file mode 100755
index 000000000..ebcbdb177
--- /dev/null
+++ b/Mailman/bin/checkdbs.py
@@ -0,0 +1,181 @@
+# Copyright (C) 1998-2006 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+
+import sys
+import time
+import optparse
+
+from email.Charset import Charset
+
+from Mailman import MailList
+from Mailman import Message
+from Mailman import Utils
+from Mailman import i18n
+from Mailman import mm_cfg
+
+_ = i18n._
+i18n.set_language(mm_cfg.DEFAULT_SERVER_LANGUAGE)
+
+__i18n_templates__ = True
+
+# Work around known problems with some RedHat cron daemons
+import signal
+signal.signal(signal.SIGCHLD, signal.SIG_DFL)
+
+NL = u'\n'
+now = time.time()
+
+
+
+def parseargs():
+    parser = optparse.OptionParser(version=mm_cfg.MAILMAN_VERSION,
+                                   usage=_("""\
+%prog [options]
+
+Check for pending admin requests and mail the list owners if necessary."""))
+    opts, args = parser.parse_args()
+    if args:
+        parser.print_help()
+        print >> sys.stderr, _('Unexpected arguments')
+        sys.exit(1)
+    return opts, args, parser
+
+
+
+def pending_requests(mlist):
+    # Must return a byte string
+    lcset = Utils.GetCharSet(mlist.preferred_language)
+    pending = []
+    first = True
+    for id in mlist.GetSubscriptionIds():
+        if first:
+            pending.append(_('Pending subscriptions:'))
+            first = False
+        when, addr, fullname, passwd, digest, lang = mlist.GetRecord(id)
+        if fullname:
+            if isinstance(fullname, unicode):
+                fullname = fullname.encode(lcset, 'replace')
+            fullname = ' (%s)' % fullname
+        pending.append('    %s%s %s' % (addr, fullname, time.ctime(when)))
+    first = True
+    for id in mlist.GetHeldMessageIds():
+        if first:
+            pending.append(_('\nPending posts:'))
+            first = False
+        info = mlist.GetRecord(id)
+        when, sender, subject, reason, text, msgdata = mlist.GetRecord(id)
+        subject = Utils.oneline(subject, lcset)
+        date = time.ctime(when)
+        reason = _(reason)
+        pending.append(_("""\
+From: $sender on $date
+Subject: $subject
+Cause: $reason"""))
+        pending.append('')
+    # Coerce all items in pending to a Unicode so we can join them
+    upending = []
+    charset = Utils.GetCharSet(mlist.preferred_language)
+    for s in pending:
+        if isinstance(s, unicode):
+            upending.append(s)
+        else:
+            upending.append(unicode(s, charset, 'replace'))
+    # Make sure that the text we return from here can be encoded to a byte
+    # string in the charset of the list's language.  This could fail if for
+    # example, the request was pended while the list's language was French,
+    # but then it was changed to English before checkdbs ran.
+    text = NL.join(upending)
+    charset = Charset(Utils.GetCharSet(mlist.preferred_language))
+    incodec = charset.input_codec or 'ascii'
+    outcodec = charset.output_codec or 'ascii'
+    if isinstance(text, unicode):
+        return text.encode(outcodec, 'replace')
+    # Be sure this is a byte string encodeable in the list's charset
+    utext = unicode(text, incodec, 'replace')
+    return utext.encode(outcodec, 'replace')
+
+
+
+def auto_discard(mlist):
+    # Discard old held messages
+    discard_count = 0
+    expire = mm_cfg.days(mlist.max_days_to_hold)
+    heldmsgs = mlist.GetHeldMessageIds()
+    if expire and heldmsgs:
+        for id in heldmsgs:
+            if now - mlist.GetRecord(id)[0] > expire:
+                mlist.HandleRequest(id, mm_cfg.DISCARD)
+                discard_count += 1
+        mlist.Save()
+    return discard_count
+
+
+
+def main():
+    opts, args, parser = parseargs()
+
+    for name in Utils.list_names():
+        # The list must be locked in order to open the requests database
+        mlist = MailList.MailList(name)
+        try:
+            count = mlist.NumRequestsPending()
+            # While we're at it, let's evict yesterday's autoresponse data
+            midnight_today = Utils.midnight()
+            evictions = []
+            for sender in mlist.hold_and_cmd_autoresponses.keys():
+                date, respcount = mlist.hold_and_cmd_autoresponses[sender]
+                if Utils.midnight(date) < midnight_today:
+                    evictions.append(sender)
+            if evictions:
+                for sender in evictions:
+                    del mlist.hold_and_cmd_autoresponses[sender]
+                # This is the only place we've changed the list's database
+                mlist.Save()
+            if count:
+                i18n.set_language(mlist.preferred_language)
+                realname = mlist.real_name
+                discarded = auto_discard(mlist)
+                if discarded:
+                    count = count - discarded
+                    text = _(
+                 'Notice: $discarded old request(s) automatically expired.\n\n')
+                else:
+                    text = ''
+                if count:
+                    text += Utils.maketext(
+                        'checkdbs.txt',
+                        {'count'    : count,
+                         'host_name': mlist.host_name,
+                         'adminDB'  : mlist.GetScriptURL('admindb', absolute=1),
+                         'real_name': realname,
+                         }, mlist=mlist)
+                    text += '\n' + pending_requests(mlist)
+                    subject = _('$count $realname moderator request(s) waiting')
+                else:
+                    subject = _('$realname moderator request check result')
+                msg = Message.UserNotification(mlist.GetOwnerEmail(),
+                                               mlist.GetBouncesEmail(),
+                                               subject, text,
+                                               mlist.preferred_language)
+                msg.send(mlist, **{'tomoderators': True})
+        finally:
+            mlist.Unlock()
+
+
+
+if __name__ == '__main__':
+    main()
diff --git a/Mailman/bin/disabled.py b/Mailman/bin/disabled.py
new file mode 100644
index 000000000..161a06712
--- /dev/null
+++ b/Mailman/bin/disabled.py
@@ -0,0 +1,200 @@
+# Copyright (C) 2001-2006 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+
+import sys
+import time
+import logging
+import optparse
+
+from Mailman import Errors
+from Mailman import MailList
+from Mailman import MemberAdaptor
+from Mailman import Pending
+from Mailman import Utils
+from Mailman import loginit
+from Mailman import mm_cfg
+from Mailman.Bouncer import _BounceInfo
+from Mailman.i18n import _
+
+__i18n_templates__ = True
+
+# Work around known problems with some RedHat cron daemons
+import signal
+signal.signal(signal.SIGCHLD, signal.SIG_DFL)
+
+loginit.initialize(propagate=True)
+elog = logging.getLogger('mailman.error')
+blog = logging.getLogger('mailman.bounce')
+
+ALL = (MemberAdaptor.BYBOUNCE,
+       MemberAdaptor.BYADMIN,
+       MemberAdaptor.BYUSER,
+       MemberAdaptor.UNKNOWN,
+       )
+
+
+
+def who_callback(option, opt, value, parser):
+    dest = getattr(parser.values, option.dest)
+    if opt in ('-o', '--byadmin'):
+        dest.add(MemberAdaptor.BYADMIN)
+    elif opt in ('-m', '--byuser'):
+        dest.add(MemberAdaptor.BYUSER)
+    elif opt in ('-u', '--unknown'):
+        dest.add(MemberAdaptor.UNKNOWN)
+    elif opt in ('-b', '--notbybounce'):
+        dest.discard(MemberAdaptor.BYBOUNCE)
+    elif opt in ('-a', '--all'):
+        dest.update(ALL)
+
+
+def parseargs():
+    parser = optparse.OptionParser(version=mm_cfg.MAILMAN_VERSION,
+                                   usage=_("""\
+%prog [options]
+
+Process disabled members, recommended once per day.
+
+This script iterates through every mailing list looking for members whose
+delivery is disabled.  If they have been disabled due to bounces, they will
+receive another notification, or they may be removed if they've received the
+maximum number of notifications.
+
+Use the --byadmin, --byuser, and --unknown flags to also send notifications to
+members whose accounts have been disabled for those reasons.  Use --all to
+send the notification to all disabled members."""))
+    # This is the set of working flags for who to send notifications to.  By
+    # default, we notify anybody who has been disable due to bounces.
+    parser.set_defaults(who=set([MemberAdaptor.BYBOUNCE]))
+    parser.add_option('-o', '--byadmin',
+                      callback=who_callback, action='callback', dest='who',
+                      help=_("""\
+Also send notifications to any member disabled by the list
+owner/administrator."""))
+    parser.add_option('-m', '--byuser',
+                      callback=who_callback, action='callback', dest='who',
+                      help=_("""\
+Also send notifications to any member who has disabled themself."""))
+    parser.add_option('-u', '--unknown',
+                      callback=who_callback, action='callback', dest='who',
+                      help=_("""\
+Also send notifications to any member disabled for unknown reasons
+(usually a legacy disabled address)."""))
+    parser.add_option('-b', '--notbybounce',
+                      callback=who_callback, action='callback', dest='who',
+                      help=_("""\
+Don't send notifications to members disabled because of bounces (the
+default is to notify bounce disabled members)."""))
+    parser.add_option('-a', '--all',
+                      callback=who_callback, action='callback', dest='who',
+                      help=_('Send notifications to all disabled members'))
+    parser.add_option('-f', '--force',
+                      default=False, action='store_true',
+                      help=_("""\
+Send notifications to disabled members even if they're not due a new
+notification yet."""))
+    parser.add_option('-l', '--listname',
+                      dest='listnames', action='append', default=[],
+                      type='string', help=_("""\
+Process only the given list, otherwise do all lists."""))
+    opts, args = parser.parse_args()
+    return opts, args, parser
+
+
+
+def main():
+    opts, args, parser = parseargs()
+
+    listnames = set(opts.listnames or Utils.list_names())
+    who = tuple(opts.who)
+
+    msg = _('[disabled by periodic sweep and cull, no message available]')
+    today = time.mktime(time.localtime()[:3] + (0,) * 6)
+    for listname in listnames:
+        # List of members to notify
+        notify = []
+        mlist = MailList.MailList(listname)
+        try:
+            interval = mlist.bounce_you_are_disabled_warnings_interval
+            # Find all the members who are currently bouncing and see if
+            # they've reached the disable threshold but haven't yet been
+            # disabled.  This is a sweep through the membership catching
+            # situations where they've bounced a bunch, then the list admin
+            # lowered the threshold, but we haven't (yet) seen more bounces
+            # from the member.  Note: we won't worry about stale information
+            # or anything else since the normal bounce processing code will
+            # handle that.
+            disables = []
+            for member in mlist.getBouncingMembers():
+                if mlist.getDeliveryStatus(member) <> MemberAdaptor.ENABLED:
+                    continue
+                info = mlist.getBounceInfo(member)
+                if info.score >= mlist.bounce_score_threshold:
+                    disables.append((member, info))
+            if disables:
+                for member, info in disables:
+                    mlist.disableBouncingMember(member, info, msg)
+            # Go through all the members who have delivery disabled, and find
+            # those that are due to have another notification.  If they are
+            # disabled for another reason than bouncing, and we're processing
+            # them (because of the command line switch) then they won't have a
+            # bounce info record.  We can piggyback on that for all disable
+            # purposes.
+            members = mlist.getDeliveryStatusMembers(who)
+            for member in members:
+                info = mlist.getBounceInfo(member)
+                if not info:
+                    # See if they are bounce disabled, or disabled for some
+                    # other reason.
+                    status = mlist.getDeliveryStatus(member)
+                    if status == MemberAdaptor.BYBOUNCE:
+                        elog.error(
+                            '%s disabled BYBOUNCE lacks bounce info, list: %s',
+                            member, mlist.internal_name())
+                        continue
+                    info = _BounceInfo(
+                        member, 0, today,
+                        mlist.bounce_you_are_disabled_warnings,
+                        mlist.pend_new(Pending.RE_ENABLE,
+                                       mlist.internal_name(),
+                                       member))
+                    mlist.setBounceInfo(member, info)
+                lastnotice = time.mktime(info.lastnotice + (0,) * 6)
+                if opts.force or today >= lastnotice + interval:
+                    notify.append(member)
+            # Now, send notifications to anyone who is due
+            for member in notify:
+                blog.info('Notifying disabled member %s for list: %s',
+                          member, mlist.internal_name())
+                try:
+                    mlist.sendNextNotification(member)
+                except Errors.NotAMemberError:
+                    # There must have been some problem with the data we have
+                    # on this member.  Most likely it's that they don't have a
+                    # password assigned.  Log this and delete the member.
+                    blog.info(
+                        'NotAMemberError when sending disabled notice: %s',
+                        member)
+                    mlist.ApprovedDeleteMember(member, 'cron/disabled')
+            mlist.Save()
+        finally:
+            mlist.Unlock()
+
+
+
+if __name__ == '__main__':
+    main()
diff --git a/Mailman/bin/gate_news.py b/Mailman/bin/gate_news.py
new file mode 100755
index 000000000..4c55273cd
--- /dev/null
+++ b/Mailman/bin/gate_news.py
@@ -0,0 +1,243 @@
+# Copyright (C) 1998-2006 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+
+import os
+import sys
+import time
+import socket
+import logging
+import nntplib
+import optparse
+
+import email.Errors
+from email.Parser import Parser
+
+from Mailman import LockFile
+from Mailman import MailList
+from Mailman import Message
+from Mailman import Utils
+from Mailman import loginit
+from Mailman import mm_cfg
+from Mailman.Queue.sbcache import get_switchboard
+from Mailman.i18n import _
+
+# Work around known problems with some RedHat cron daemons
+import signal
+signal.signal(signal.SIGCHLD, signal.SIG_DFL)
+
+GATENEWS_LOCK_FILE = os.path.join(mm_cfg.LOCK_DIR, 'gate_news.lock')
+
+LOCK_LIFETIME = mm_cfg.hours(2)
+NL = '\n'
+
+loginit.initialize(propagate=True)
+log = logging.getLogger('mailman.fromusenet')
+
+class _ContinueLoop(Exception):
+    pass
+
+__i18n_templates__ = True
+
+
+
+def parseargs():
+    parser = optparse.OptionParser(version=mm_cfg.MAILMAN_VERSION,
+                                   usage=_("""\
+%prog [options]
+
+Poll the NNTP servers for messages to be gatewayed to mailing lists."""))
+    opts, args = parser.parse_args()
+    if args:
+        parser.print_help()
+        print >> sys.stderr, _('Unexpected arguments')
+        sys.exit(1)
+    return opts, args, parser
+
+
+
+_hostcache = {}
+
+def open_newsgroup(mlist):
+    # Split host:port if given
+    nntp_host, nntp_port = Utils.nntpsplit(mlist.nntp_host)
+    # Open up a "mode reader" connection to nntp server.  This will be shared
+    # for all the gated lists having the same nntp_host.
+    conn = _hostcache.get(mlist.nntp_host)
+    if conn is None:
+        try:
+            conn = nntplib.NNTP(nntp_host, nntp_port,
+                                readermode=True,
+                                user=mm_cfg.NNTP_USERNAME,
+                                password=mm_cfg.NNTP_PASSWORD)
+        except (socket.error, nntplib.NNTPError, IOError), e:
+            log.error('error opening connection to nntp_host: %s\n%s',
+                      mlist.nntp_host, e)
+            raise
+        _hostcache[mlist.nntp_host] = conn
+    # Get the GROUP information for the list, but we're only really interested
+    # in the first article number and the last article number
+    r, c, f, l, n = conn.group(mlist.linked_newsgroup)
+    return conn, int(f), int(l)
+
+
+def clearcache():
+    for conn in set(_hostcache.values()):
+        conn.quit()
+    _hostcache.clear()
+
+
+
+# This function requires the list to be locked.
+def poll_newsgroup(mlist, conn, first, last, glock):
+    listname = mlist.internal_name()
+    # NEWNEWS is not portable and has synchronization issues.
+    for num in range(first, last):
+        glock.refresh()
+        try:
+            headers = conn.head(repr(num))[3]
+            found_to = False
+            beenthere = False
+            for header in headers:
+                i = header.find(':')
+                value = header[:i].lower()
+                if i > 0 and value == 'to':
+                    found_to = True
+                if value <> 'x-beenthere':
+                    continue
+                if header[i:] == ': %s' % mlist.GetListEmail():
+                    beenthere = True
+                    break
+            if not beenthere:
+                body = conn.body(repr(num))[3]
+                # Usenet originated messages will not have a Unix envelope
+                # (i.e. "From " header).  This breaks Pipermail archiving, so
+                # we will synthesize one.  Be sure to use the format searched
+                # for by mailbox.UnixMailbox._isrealfromline().  BAW: We use
+                # the -bounces address here in case any downstream clients use
+                # the envelope sender for bounces; I'm not sure about this,
+                # but it's the closest to the old semantics.
+                lines = ['From %s  %s' % (mlist.GetBouncesEmail(),
+                                          time.ctime(time.time()))]
+                lines.extend(headers)
+                lines.append('')
+                lines.extend(body)
+                lines.append('')
+                p = Parser(Message.Message)
+                try:
+                    msg = p.parsestr(NL.join(lines))
+                except email.Errors.MessageError, e:
+                    log.error('email package exception for %s:%d\n%s',
+                              mlist.linked_newsgroup, num, e)
+                    raise _ContinueLoop
+                if found_to:
+                    del msg['X-Originally-To']
+                    msg['X-Originally-To'] = msg['To']
+                    del msg['To']
+                msg['To'] = mlist.GetListEmail()
+                # Post the message to the locked list
+                inq = get_switchboard(mm_cfg.INQUEUE_DIR)
+                inq.enqueue(msg,
+                            listname=mlist.internal_name(),
+                            fromusenet=True)
+                log.info('posted to list %s: %7d', listname, num)
+        except nntplib.NNTPError, e:
+            log.exception('NNTP error for list %s: %7d', listname, num)
+        except _ContinueLoop:
+            continue
+        # Even if we don't post the message because it was seen on the
+        # list already, update the watermark
+        mlist.usenet_watermark = num
+
+
+
+def process_lists(glock):
+    for listname in Utils.list_names():
+        glock.refresh()
+        # Open the list unlocked just to check to see if it is gating news to
+        # mail.  If not, we're done with the list.  Otherwise, lock the list
+        # and gate the group.
+        mlist = MailList.MailList(listname, lock=False)
+        if not mlist.gateway_to_mail:
+            continue
+        # Get the list's watermark, i.e. the last article number that we gated
+        # from news to mail.  None means that this list has never polled its
+        # newsgroup and that we should do a catch up.
+        watermark = getattr(mlist, 'usenet_watermark', None)
+        # Open the newsgroup, but let most exceptions percolate up.
+        try:
+            conn, first, last = open_newsgroup(mlist)
+        except (socket.error, nntplib.NNTPError):
+            break
+        log.info('%s: [%d..%d]', listname, first, last)
+        try:
+            try:
+                if watermark is None:
+                    mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT)
+                    # This is the first time we've tried to gate this
+                    # newsgroup.  We essentially do a mass catch-up, otherwise
+                    # we'd flood the mailing list.
+                    mlist.usenet_watermark = last
+                    log.info('%s caught up to article %d', listname, last)
+                else:
+                    # The list has been polled previously, so now we simply
+                    # grab all the messages on the newsgroup that have not
+                    # been seen by the mailing list.  The first such article
+                    # is the maximum of the lowest article available in the
+                    # newsgroup and the watermark.  It's possible that some
+                    # articles have been expired since the last time gate_news
+                    # has run.  Not much we can do about that.
+                    start = max(watermark + 1, first)
+                    if start > last:
+                        log.info('nothing new for list %s', listname)
+                    else:
+                        mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT)
+                        log.info('gating %s articles [%d..%d]',
+                                 listname, start, last)
+                        # Use last+1 because poll_newsgroup() employes a for
+                        # loop over range, and this will not include the last
+                        # element in the list.
+                        poll_newsgroup(mlist, conn, start, last + 1, glock)
+            except LockFile.TimeOutError:
+                log.error('Could not acquire list lock: %s', listname)
+        finally:
+            if mlist.Locked():
+                mlist.Save()
+                mlist.Unlock()
+        log.info('%s watermark: %d', listname, mlist.usenet_watermark)
+
+
+
+def main():
+    opts, args, parser = parseargs()
+    lock = LockFile.LockFile(GATENEWS_LOCK_FILE,
+                             # It's okay to hijack this
+                             lifetime=LOCK_LIFETIME)
+    try:
+        lock.lock(timeout=0.5)
+    except LockFile.TimeOutError:
+        log.error('Could not acquire gate_news lock')
+        return
+    try:
+        process_lists(lock)
+    finally:
+        clearcache()
+        lock.unlock(unconditionally=True)
+
+
+
+if __name__ == '__main__':
+    main()
diff --git a/Mailman/bin/nightly_gzip.py b/Mailman/bin/nightly_gzip.py
new file mode 100644
index 000000000..037612adb
--- /dev/null
+++ b/Mailman/bin/nightly_gzip.py
@@ -0,0 +1,126 @@
+#! @PYTHON@
+#
+# Copyright (C) 1998-2006 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+
+import os
+import sys
+import optparse
+
+try:
+    import gzip
+except ImportError:
+    sys.exit(0)
+
+from Mailman import mm_cfg
+from Mailman import Utils
+from Mailman import MailList
+from Mailman.i18n import _
+
+__i18n_templates__ = True
+
+
+
+def parseargs():
+    parser = optparse.OptionParser(version=mm_cfg.MAILMAN_VERSION,
+                                   usage=_("""\
+%prog [options] [listname ...]
+
+Re-generate the Pipermail gzip'd archive flat files."""))
+    parser.add_option('-v', '--verbose',
+                      default=False, action='store_true',
+                      help=_("Print each file as it's being gzip'd"))
+    parser.add_option('-z', '--level',
+                      default=6, type='int',
+                      help=_('Specifies the compression level'))
+    opts, args = parser.parse_args()
+    if opts.level < 1 or opts.level > 9:
+        parser.print_help()
+        print >> sys.stderr, _('Illegal compression level: $opts.level')
+        sys.exit(1)
+    return opts, args, parser
+
+
+
+def compress(txtfile, opts):
+    if opts.verbose:
+        print _("gzip'ing: $txtfile")
+    infp = outfp = None
+    try:
+        infp = open(txtfile)
+        outfp = gzip.open(txtfile + '.gz', 'wb', opts.level)
+        outfp.write(infp.read())
+    finally:
+        if outfp:
+            outfp.close()
+        if infp:
+            infp.close()
+
+
+
+def main():
+    if mm_cfg.ARCHIVE_TO_MBOX not in (1, 2) or mm_cfg.GZIP_ARCHIVE_TXT_FILES:
+        # We're only going to run the nightly archiver if messages are
+        # archived to the mbox, and the gzip file is not created on demand
+        # (i.e. for every individual post).  This is the normal mode of
+        # operation.
+        return
+
+    opts, args, parser = parseargs()
+
+    # Process all the specified lists
+    for listname in set(args or Utils.list_names()):
+        mlist = MailList.MailList(listname, lock=False)
+        if not mlist.archive:
+            continue
+        dir = mlist.archive_dir()
+        try:
+            allfiles = os.listdir(dir)
+        except OSError:
+            # Has the list received any messages?  If not, last_post_time will
+            # be zero, so it's not really a bogus archive dir.
+            if mlist.last_post_time > 0:
+                print _('List $listname has a bogus archive_directory: $dir')
+            continue
+        if opts.verbose:
+            print _('Processing list: $listname')
+        files = []
+        for f in allfiles:
+            if os.path.splitext(f)[1] <> '.txt':
+                continue
+            # stat both the .txt and .txt.gz files and append them only if
+            # the former is newer than the latter.
+            txtfile = os.path.join(dir, f)
+            gzpfile = txtfile + '.gz'
+            txt_mtime = os.path.getmtime(txtfile)
+            try:
+                gzp_mtime = os.path.getmtime(gzpfile)
+            except OSError:
+                gzp_mtime = -1
+            if txt_mtime > gzp_mtime:
+                files.append(txtfile)
+        for f in files:
+            compress(f, opts)
+
+
+
+if __name__ == '__main__':
+    omask = os.umask(002)
+    try:
+        main()
+    finally:
+        os.umask(omask)
diff --git a/Mailman/bin/senddigests.py b/Mailman/bin/senddigests.py
new file mode 100755
index 000000000..fa01d3666
--- /dev/null
+++ b/Mailman/bin/senddigests.py
@@ -0,0 +1,71 @@
+# Copyright (C) 1998-2006 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+
+import sys
+import optparse
+
+from Mailman import MailList
+from Mailman import Utils
+from Mailman import mm_cfg
+from Mailman.i18n import _
+
+# Work around known problems with some RedHat cron daemons
+import signal
+signal.signal(signal.SIGCHLD, signal.SIG_DFL)
+
+__i18n_templates__ = True
+
+
+
+def parseargs():
+    parser = optparse.OptionParser(version=mm_cfg.MAILMAN_VERSION,
+                                   usage=_("""\
+%prog [options]
+
+Dispatch digests for lists w/pending messages and digest_send_periodic
+set."""))
+    parser.add_option('-l', '--listname',
+                      type='string', default=[], action='append',
+                      dest='listnames', help=_("""\
+Send the digest for the given list only, otherwise the digests for all
+lists are sent out.  Multiple -l options may be given."""))
+    opts, args = parser.parse_args()
+    if args:
+        parser.print_help()
+        print >> sys.stderr, _('Unexpected arguments')
+        sys.exit(1)
+    return opts, args, parser
+
+
+
+def main():
+    opts, args, parser = parseargs()
+
+    for listname in set(opts.listnames or Utils.list_names()):
+        mlist = MailList.MailList(listname, lock=False)
+        if mlist.digest_send_periodic:
+            mlist.Lock()
+            try:
+                mlist.send_digest_now()
+                mlist.Save()
+            finally:
+                mlist.Unlock()
+
+
+
+if __name__ == '__main__':
+    main()