Move all cron scripts to the new Mailman.bin package layout and complete the

conversion to optparse style option parsing.  Remove mailpasswds as password
reminders will go away for MM2.2.

9 files changed, 9 insertions, 1295 deletions

diff --git a/cron/Makefile.in b/cron/Makefile.in
index afb033705..ba2ed882e 100644
--- a/cron/Makefile.in
+++ b/cron/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2006 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -12,7 +12,8 @@
 # 
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software 
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
 
 # NOTE: Makefile.in is converted into Makefile by the configure script
 # in the parent directory.  Once configure has run, you can recreate
@@ -30,6 +31,7 @@ DESTDIR=
 CC=		@CC@
 CHMOD=  	@CHMOD@
 INSTALL=	@INSTALL@
+LN_S=		@LN_S@
 
 DEFS=   	@DEFS@
 
@@ -38,13 +40,12 @@ DEFS=   	@DEFS@
 OPT=		@OPT@
 CFLAGS=		$(OPT) $(DEFS)
 CRONDIR= 	$(prefix)/cron
+SCRIPTSDIR= 	../bin
 
 SHELL=		/bin/sh
 
-PROGRAMS=	checkdbs mailpasswds senddigests gate_news \
-		nightly_gzip bumpdigests disabled
+LN_PROGRAMS=	bumpdigests checkdbs disabled gate_news nightly_gzip senddigests
 FILES=  	crontab.in
-
 BUILDDIR=	../build/cron
 
 # Modes for directories and executables created by the install
@@ -63,9 +64,10 @@ install:
 	do \
 	    $(INSTALL) -m $(FILEMODE) $$f $(DESTDIR)$(CRONDIR); \
 	done
-	for f in $(PROGRAMS); \
+	for f in $(LN_PROGRAMS); \
 	do \
-	    $(INSTALL) -m $(EXEMODE) $(BUILDDIR)/$$f $(DESTDIR)$(CRONDIR); \
+	    rm -f $(DESTDIR)$(CRONDIR)/$$f; \
+	    (cd $(DESTDIR)$(CRONDIR); $(LN_S) $(SCRIPTSDIR)/mmshell $$f); \
 	done
 
 finish:
diff --git a/cron/bumpdigests b/cron/bumpdigests
deleted file mode 100644
index 57cc45e1e..000000000
--- a/cron/bumpdigests
+++ /dev/null
@@ -1,96 +0,0 @@
-#! @PYTHON@
-#
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software 
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-
-"""Increment the digest volume number and reset the digest number to one.
-
-Usage: %(PROGRAM)s [options] [listname ...]
-
-Options:
-
-    --help/-h
-        Print this message and exit.
-
-The lists named on the command line are bumped.  If no list names are given,
-all lists are bumped.
-"""
-
-import sys
-import getopt
-
-import paths
-from Mailman import mm_cfg
-from Mailman import Utils
-from Mailman import MailList
-from Mailman import Errors
-from Mailman.i18n import _
-
-# Work around known problems with some RedHat cron daemons
-import signal
-signal.signal(signal.SIGCHLD, signal.SIG_DFL)
-
-PROGRAM = sys.argv[0]
-
-
-
-def usage(code, msg=''):
-    if code:
-        fd = sys.stderr
-    else:
-        fd = sys.stdout
-    print >> fd, _(__doc__)
-    if msg:
-        print >> fd, msg
-    sys.exit(code)
-
-
-
-def main():
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], 'h', ['help'])
-    except getopt.error, msg:
-        usage(1, msg)
-
-    for opt, arg in opts:
-        if opt in ('-h', '--help'):
-            usage(0)
-
-    if args:
-        listnames = args
-    else:
-        listnames = Utils.list_names()
-
-    if not listnames:
-        print _('Nothing to do.')
-        sys.exit(0)
-
-    for listname in listnames:
-        try:
-            # be sure the list is locked
-            mlist = MailList.MailList(listname)
-        except Errors.MMListError, e:
-            usage(1, _('No such list: %(listname)s'))
-        try:
-            mlist.bump_digest_volume()
-        finally:
-            mlist.Save()
-            mlist.Unlock()
-
-
-
-if __name__ == '__main__':
-    main()
diff --git a/cron/checkdbs b/cron/checkdbs
deleted file mode 100755
index 126981093..000000000
--- a/cron/checkdbs
+++ /dev/null
@@ -1,203 +0,0 @@
-#! @PYTHON@
-#
-# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-
-"""Check for pending admin requests and mail the list owners if necessary.
-
-Usage: %(PROGRAM)s [options]
-
-Options:
-
-    -h/--help
-        Print this message and exit.
-"""
-
-import sys
-import time
-import getopt
-from types import UnicodeType
-
-import paths
-
-# Import this after paths so we get Mailman's copy of the email package
-from email.Charset import Charset
-
-from Mailman import mm_cfg
-from Mailman import Utils
-from Mailman import MailList
-from Mailman import Message
-from Mailman import i18n
-
-# Work around known problems with some RedHat cron daemons
-import signal
-signal.signal(signal.SIGCHLD, signal.SIG_DFL)
-
-NL = '\n'
-PROGRAM = sys.argv[0]
-
-_ = i18n._
-i18n.set_language(mm_cfg.DEFAULT_SERVER_LANGUAGE)
-
-now = time.time()
-
-
-
-def usage(code, msg=''):
-    if code:
-        fd = sys.stderr
-    else:
-        fd = sys.stdout
-    print >> fd, _(__doc__)
-    if msg:
-        print >> fd, msg
-    sys.exit(code)
-
-
-
-def main():
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], 'h', ['help'])
-    except getopt.error, msg:
-        usage(1, msg)
-
-    for opt, arg in opts:
-        if opt in ('-h', '--help'):
-            usage(0)
-
-    if args:
-        usage(1)
-
-    for name in Utils.list_names():
-        # the list must be locked in order to open the requests database
-        mlist = MailList.MailList(name)
-        try:
-            count = mlist.NumRequestsPending()
-            # While we're at it, let's evict yesterday's autoresponse data
-            midnightToday = Utils.midnight()
-            evictions = []
-            for sender in mlist.hold_and_cmd_autoresponses.keys():
-                date, respcount = mlist.hold_and_cmd_autoresponses[sender]
-                if Utils.midnight(date) < midnightToday:
-                    evictions.append(sender)
-            if evictions:
-                for sender in evictions:
-                    del mlist.hold_and_cmd_autoresponses[sender]
-                # Only here have we changed the list's database
-                mlist.Save()
-            if count:
-                i18n.set_language(mlist.preferred_language)
-                realname = mlist.real_name
-                discarded = auto_discard(mlist)
-                if discarded:
-                    count = count - discarded
-                    text = _(
-             'Notice: %(discarded)d old request(s) automatically expired.\n\n')
-                else:
-                    text = ''
-                if count:
-                    text += Utils.maketext(
-                        'checkdbs.txt',
-                        {'count'    : count,
-                         'host_name': mlist.host_name,
-                         'adminDB'  : mlist.GetScriptURL('admindb', absolute=1),
-                         'real_name': realname,
-                         }, mlist=mlist)
-                    text += '\n' + pending_requests(mlist)
-                    subject = _(
-                        '%(count)d %(realname)s moderator request(s) waiting')
-                else:
-                    subject = _(
-                        '%(realname)s moderator request check result')
-                msg = Message.UserNotification(mlist.GetOwnerEmail(),
-                                               mlist.GetBouncesEmail(),
-                                               subject, text,
-                                               mlist.preferred_language)
-                msg.send(mlist, **{'tomoderators': 1})
-        finally:
-            mlist.Unlock()
-
-
-
-
-def pending_requests(mlist):
-    # Must return a byte string
-    lcset = Utils.GetCharSet(mlist.preferred_language)
-    pending = []
-    first = 1
-    for id in mlist.GetSubscriptionIds():
-        if first:
-            pending.append(_('Pending subscriptions:'))
-            first = 0
-        when, addr, fullname, passwd, digest, lang = mlist.GetRecord(id)
-        if fullname:
-            if isinstance(fullname, UnicodeType):
-                fullname = fullname.encode(lcset, 'replace')
-            fullname = ' (%s)' % fullname
-        pending.append('    %s%s %s' % (addr, fullname, time.ctime(when)))
-    first = 1
-    for id in mlist.GetHeldMessageIds():
-        if first:
-            pending.append(_('\nPending posts:'))
-            first = 0
-        info = mlist.GetRecord(id)
-        when, sender, subject, reason, text, msgdata = mlist.GetRecord(id)
-        subject = Utils.oneline(subject, lcset)
-        date = time.ctime(when)
-        reason = _(reason)
-        pending.append(_("""\
-From: %(sender)s on %(date)s
-Subject: %(subject)s
-Cause: %(reason)s"""))
-        pending.append('')
-    # Coerce all items in pending to a Unicode so we can join them
-    upending = []
-    charset = Utils.GetCharSet(mlist.preferred_language)
-    for s in pending:
-        if isinstance(s, UnicodeType):
-            upending.append(s)
-        else:
-            upending.append(unicode(s, charset, 'replace'))
-    # Make sure that the text we return from here can be encoded to a byte
-    # string in the charset of the list's language.  This could fail if for
-    # example, the request was pended while the list's language was French,
-    # but then it was changed to English before checkdbs ran.
-    text = u'\n'.join(upending)
-    charset = Charset(Utils.GetCharSet(mlist.preferred_language))
-    incodec = charset.input_codec or 'ascii'
-    outcodec = charset.output_codec or 'ascii'
-    if isinstance(text, UnicodeType):
-        return text.encode(outcodec, 'replace')
-    # Be sure this is a byte string encodeable in the list's charset
-    utext = unicode(text, incodec, 'replace')
-    return utext.encode(outcodec, 'replace')
-
-def auto_discard(mlist):
-    # Discard old held messages
-    discard_count = 0
-    expire = mlist.max_days_to_hold * 86400 # days
-    heldmsgs = mlist.GetHeldMessageIds()
-    if expire and len(heldmsgs):
-        for id in heldmsgs:
-            if now - mlist.GetRecord(id)[0] > expire:
-                mlist.HandleRequest(id, mm_cfg.DISCARD)
-                discard_count += 1
-        mlist.Save()
-    return discard_count
-
-
-if __name__ == '__main__':
-    main()
diff --git a/cron/crontab.in.in b/cron/crontab.in.in
index 49f27c729..5101ef2b3 100755
--- a/cron/crontab.in.in
+++ b/cron/crontab.in.in
@@ -10,9 +10,6 @@
 # Noon, mail digests for lists that do periodic as well as threshhold delivery.
 0 12 * * * @PYTHON@ -S @prefix@/cron/senddigests
 #
-# 5 AM on the first of each month, mail out password reminders.
-0 5 1 * * @PYTHON@ -S @prefix@/cron/mailpasswds
-#
 # Every 5 mins, try to gate news to mail.  You can comment this one out
 # if you don't want to allow gating, or don't have any going on right now,
 # or want to exclusively use a callback strategy instead of polling.
diff --git a/cron/disabled b/cron/disabled
deleted file mode 100644
index 314339f1a..000000000
--- a/cron/disabled
+++ /dev/null
@@ -1,226 +0,0 @@
-#! @PYTHON@
-#
-# Copyright (C) 2001-2006 by the Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
-# USA.
-
-"""Process disabled members, recommended once per day.
-
-This script cruises through every mailing list looking for members whose
-delivery is disabled.  If they have been disabled due to bounces, they will
-receive another notification, or they may be removed if they've received the
-maximum number of notifications.
-
-Use the --byadmin, --byuser, and --unknown flags to also send notifications to
-members whose accounts have been disabled for those reasons.  Use --all to
-send the notification to all disabled members.
-
-Usage: %(PROGRAM)s [options]
-
-Options:
-    -h / --help
-        Print this message and exit.
-
-    -o / --byadmin
-        Also send notifications to any member disabled by the list
-        owner/administrator.
-
-    -m / --byuser
-        Also send notifications to any member disabled by themselves.
-
-    -u / --unknown
-        Also send notifications to any member disabled for unknown reasons
-        (usually a legacy disabled address).
-
-    -b / --notbybounce
-        Don't send notifications to members disabled because of bounces (the
-        default is to notify bounce disabled members).
-
-    -a / --all
-        Send notifications to all disabled members.
-
-    -f / --force
-        Send notifications to disabled members even if they're not due a new
-        notification yet.
-
-    -l listname
-    --listname=listname
-        Process only the given list, otherwise do all lists.
-"""
-
-import sys
-import time
-import getopt
-import logging
-
-import paths
-# mm_cfg must be imported before the other modules, due to the side-effect of
-# it hacking sys.paths to include site-packages.  Without this, running this
-# script from cron with python -S will fail.
-from Mailman import mm_cfg
-
-from Mailman import Errors
-from Mailman import MailList
-from Mailman import MemberAdaptor
-from Mailman import Pending
-from Mailman import Utils
-from Mailman import loginit
-from Mailman.Bouncer import _BounceInfo
-from Mailman.i18n import _
-
-# Work around known problems with some RedHat cron daemons
-import signal
-signal.signal(signal.SIGCHLD, signal.SIG_DFL)
-
-PROGRAM = sys.argv[0]
-
-loginit.initialize(propagate=True)
-elog = logging.getLogger('mailman.error')
-blog = logging.getLogger('mailman.bounce')
-
-
-
-def usage(code, msg=''):
-    if code:
-        fd = sys.stderr
-    else:
-        fd = sys.stdout
-    print >> fd, _(__doc__)
-    if msg:
-        print >> fd, msg
-    sys.exit(code)
-
-
-
-def main():
-    try:
-        opts, args = getopt.getopt(
-            sys.argv[1:], 'hl:omubaf',
-            ['byadmin', 'byuser', 'unknown', 'notbybounce', 'all',
-             'listname=', 'help', 'force'])
-    except getopt.error, msg:
-        usage(1, msg)
-
-    if args:
-        usage(1)
-
-    force = 0
-    listnames = []
-    who = [MemberAdaptor.BYBOUNCE]
-    for opt, arg in opts:
-        if opt in ('-h', '--help'):
-            usage(0)
-        elif opt in ('-l', '--list'):
-            listnames.append(arg)
-        elif opt in ('-o', '--byadmin'):
-            who.append(MemberAdaptor.BYADMIN)
-        elif opt in ('-m', '--byuser'):
-            who.append(MemberAdaptor.BYUSER)
-        elif opt in ('-u', '--unknown'):
-            who.append(MemberAdaptor.UNKNOWN)
-        elif opt in ('-b', '--notbybounce'):
-            try:
-                who.remove(MemberAdaptor.BYBOUNCE)
-            except ValueError:
-                # Already removed
-                pass
-        elif opt in ('-a', '--all'):
-            who = [MemberAdaptor.BYBOUNCE, MemberAdaptor.BYADMIN,
-                   MemberAdaptor.BYUSER, MemberAdaptor.UNKNOWN]
-        elif opt in ('-f', '--force'):
-            force = 1
-
-    who = tuple(who)
-
-    if not listnames:
-        listnames = Utils.list_names()
-
-    msg = _('[disabled by periodic sweep and cull, no message available]')
-    today = time.mktime(time.localtime()[:3] + (0,) * 6)
-    for listname in listnames:
-        # List of members to notify
-        notify = []
-        mlist = MailList.MailList(listname)
-        try:
-            interval = mlist.bounce_you_are_disabled_warnings_interval
-            # Find all the members who are currently bouncing and see if
-            # they've reached the disable threshold but haven't yet been
-            # disabled.  This is a sweep through the membership catching
-            # situations where they've bounced a bunch, then the list admin
-            # lowered the threshold, but we haven't (yet) seen more bounces
-            # from the member.  Note: we won't worry about stale information
-            # or anything else since the normal bounce processing code will
-            # handle that.
-            disables = []
-            for member in mlist.getBouncingMembers():
-                if mlist.getDeliveryStatus(member) <> MemberAdaptor.ENABLED:
-                    continue
-                info = mlist.getBounceInfo(member)
-                if info.score >= mlist.bounce_score_threshold:
-                    disables.append((member, info))
-            if disables:
-                for member, info in disables:
-                    mlist.disableBouncingMember(member, info, msg)
-            # Go through all the members who have delivery disabled, and find
-            # those that are due to have another notification.  If they are
-            # disabled for another reason than bouncing, and we're processing
-            # them (because of the command line switch) then they won't have a
-            # bounce info record.  We can piggyback on that for all disable
-            # purposes.
-            members = mlist.getDeliveryStatusMembers(who)
-            for member in members:
-                info = mlist.getBounceInfo(member)
-                if not info:
-                    # See if they are bounce disabled, or disabled for some
-                    # other reason.
-                    status = mlist.getDeliveryStatus(member)
-                    if status == MemberAdaptor.BYBOUNCE:
-                        elog.error(
-                            '%s disabled BYBOUNCE lacks bounce info, list: %s',
-                            member, mlist.internal_name())
-                        continue
-                    info = _BounceInfo(
-                        member, 0, today,
-                        mlist.bounce_you_are_disabled_warnings,
-                        mlist.pend_new(Pending.RE_ENABLE,
-                                       mlist.internal_name(),
-                                       member))
-                    mlist.setBounceInfo(member, info)
-                lastnotice = time.mktime(info.lastnotice + (0,) * 6)
-                if force or today >= lastnotice + interval:
-                    notify.append(member)
-            # Now, send notifications to anyone who is due
-            for member in notify:
-                blog.info('Notifying disabled member %s for list: %s',
-                          member, mlist.internal_name())
-                try:
-                    mlist.sendNextNotification(member)
-                except Errors.NotAMemberError:
-                    # There must have been some problem with the data we have
-                    # on this member.  Most likely it's that they don't have a
-                    # password assigned.  Log this and delete the member.
-                    blog.info(
-                        'NotAMemberError when sending disabled notice: %s',
-                        member)
-                    mlist.ApprovedDeleteMember(member, 'cron/disabled')
-            mlist.Save()
-        finally:
-            mlist.Unlock()
-
-
-
-if __name__ == '__main__':
-    main()
diff --git a/cron/gate_news b/cron/gate_news
deleted file mode 100755
index f0701ca8b..000000000
--- a/cron/gate_news
+++ /dev/null
@@ -1,269 +0,0 @@
-#! @PYTHON@
-#
-# Copyright (C) 1998-2006 by the Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
-# USA.
-
-"""Poll the NNTP servers for messages to be gatewayed to mailing lists.
-
-Usage: gate_news [options]
-
-Where options are
-
-    --help
-    -h
-        Print this text and exit.
-
-"""
-
-import os
-import sys
-import time
-import getopt
-import socket
-import logging
-import nntplib
-
-import paths
-# mm_cfg must be imported before the other modules, due to the side-effect of
-# it hacking sys.paths to include site-packages.  Without this, running this
-# script from cron with python -S will fail.
-from Mailman import mm_cfg
-
-# Import this /after/ paths so that the sys.path is properly hacked
-import email.Errors
-from email.Parser import Parser
-
-from Mailman import LockFile
-from Mailman import MailList
-from Mailman import Message
-from Mailman import Utils
-from Mailman import loginit
-from Mailman.Queue.sbcache import get_switchboard
-from Mailman.i18n import _
-
-# Work around known problems with some RedHat cron daemons
-import signal
-signal.signal(signal.SIGCHLD, signal.SIG_DFL)
-
-GATENEWS_LOCK_FILE = os.path.join(mm_cfg.LOCK_DIR, 'gate_news.lock')
-
-LOCK_LIFETIME = mm_cfg.hours(2)
-NL = '\n'
-
-loginit.initialize(propagate=True)
-log = logging.getLogger('mailman.fromusenet')
-
-
-
-def usage(status, msg=''):
-    if code:
-        fd = sys.stderr
-    else:
-        fd = sys.stdout
-    print >> fd, _(__doc__)
-    if msg:
-        print >> fd, msg
-    sys.exit(code)
-
-
-
-_hostcache = {}
-
-def open_newsgroup(mlist):
-    # Split host:port if given
-    nntp_host, nntp_port = Utils.nntpsplit(mlist.nntp_host)
-    # Open up a "mode reader" connection to nntp server.  This will be shared
-    # for all the gated lists having the same nntp_host.
-    conn = _hostcache.get(mlist.nntp_host)
-    if conn is None:
-        try:
-            conn = nntplib.NNTP(nntp_host, nntp_port,
-                                readermode=True,
-                                user=mm_cfg.NNTP_USERNAME,
-                                password=mm_cfg.NNTP_PASSWORD)
-        except (socket.error, nntplib.NNTPError, IOError), e:
-            log.error('error opening connection to nntp_host: %s\n%s',
-                      mlist.nntp_host, e)
-            raise
-        _hostcache[mlist.nntp_host] = conn
-    # Get the GROUP information for the list, but we're only really interested
-    # in the first article number and the last article number
-    r,c,f,l,n = conn.group(mlist.linked_newsgroup)
-    return conn, int(f), int(l)
-
-
-def clearcache():
-    reverse = {}
-    for conn in _hostcache.values():
-        reverse[conn] = 1
-    for conn in reverse.keys():
-        conn.quit()
-    _hostcache.clear()
-
-
-
-# This function requires the list to be locked.
-def poll_newsgroup(mlist, conn, first, last, glock):
-    listname = mlist.internal_name()
-    # NEWNEWS is not portable and has synchronization issues.
-    for num in range(first, last):
-        glock.refresh()
-        try:
-            headers = conn.head(`num`)[3]
-            found_to = 0
-            beenthere = 0
-            for header in headers:
-                i = header.find(':')
-                value = header[:i].lower()
-                if i > 0 and value == 'to':
-                    found_to = 1
-                if value <> 'x-beenthere':
-                    continue
-                if header[i:] == ': %s' % mlist.GetListEmail():
-                    beenthere = 1
-                    break
-            if not beenthere:
-                body = conn.body(`num`)[3]
-                # Usenet originated messages will not have a Unix envelope
-                # (i.e. "From " header).  This breaks Pipermail archiving, so
-                # we will synthesize one.  Be sure to use the format searched
-                # for by mailbox.UnixMailbox._isrealfromline().  BAW: We use
-                # the -bounces address here in case any downstream clients use
-                # the envelope sender for bounces; I'm not sure about this,
-                # but it's the closest to the old semantics.
-                lines = ['From %s  %s' % (mlist.GetBouncesEmail(),
-                                          time.ctime(time.time()))]
-                lines.extend(headers)
-                lines.append('')
-                lines.extend(body)
-                lines.append('')
-                p = Parser(Message.Message)
-                try:
-                    msg = p.parsestr(NL.join(lines))
-                except email.Errors.MessageError, e:
-                    log.error('email package exception for %s:%d\n%s',
-                              mlist.linked_newsgroup, num, e)
-                    raise _ContinueLoop
-                if found_to:
-                    del msg['X-Originally-To']
-                    msg['X-Originally-To'] = msg['To']
-                    del msg['To']
-                msg['To'] = mlist.GetListEmail()
-                # Post the message to the locked list
-                inq = get_switchboard(mm_cfg.INQUEUE_DIR)
-                inq.enqueue(msg,
-                            listname = mlist.internal_name(),
-                            fromusenet = 1)
-                log.info('posted to list %s: %7d', listname, num)
-        except nntplib.NNTPError, e:
-            log.exception('NNTP error for list %s: %7d', listname, num)
-        except _ContinueLoop:
-            continue
-        # Even if we don't post the message because it was seen on the
-        # list already, update the watermark
-        mlist.usenet_watermark = num
-
-
-
-def process_lists(glock):
-    for listname in Utils.list_names():
-        glock.refresh()
-        # Open the list unlocked just to check to see if it is gating news to
-        # mail.  If not, we're done with the list.  Otherwise, lock the list
-        # and gate the group.
-        mlist = MailList.MailList(listname, lock=0)
-        if not mlist.gateway_to_mail:
-            continue
-        # Get the list's watermark, i.e. the last article number that we gated
-        # from news to mail.  `None' means that this list has never polled its
-        # newsgroup and that we should do a catch up.
-        watermark = getattr(mlist, 'usenet_watermark', None)
-        # Open the newsgroup, but let most exceptions percolate up.
-        try:
-            conn, first, last = open_newsgroup(mlist)
-        except (socket.error, nntplib.NNTPError):
-            break
-        log.info('%s: [%d..%d]', listname, first, last)
-        try:
-            try:
-                if watermark is None:
-                    mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT)
-                    # This is the first time we've tried to gate this
-                    # newsgroup.  We essentially do a mass catch-up, otherwise
-                    # we'd flood the mailing list.
-                    mlist.usenet_watermark = last
-                    log.info('%s caught up to article %d', listname, last)
-                else:
-                    # The list has been polled previously, so now we simply
-                    # grab all the messages on the newsgroup that have not
-                    # been seen by the mailing list.  The first such article
-                    # is the maximum of the lowest article available in the
-                    # newsgroup and the watermark.  It's possible that some
-                    # articles have been expired since the last time gate_news
-                    # has run.  Not much we can do about that.
-                    start = max(watermark+1, first)
-                    if start > last:
-                        log.info('nothing new for list %s', listname)
-                    else:
-                        mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT)
-                        log.info('gating %s articles [%d..%d]',
-                                 listname, start, last)
-                        # Use last+1 because poll_newsgroup() employes a for
-                        # loop over range, and this will not include the last
-                        # element in the list.
-                        poll_newsgroup(mlist, conn, start, last+1, glock)
-            except LockFile.TimeOutError:
-                log.error('Could not acquire list lock: %s', listname)
-        finally:
-            if mlist.Locked():
-                mlist.Save()
-                mlist.Unlock()
-        log.info('%s watermark: %d', listname, mlist.usenet_watermark)
-
-
-
-def main():
-    lock = LockFile.LockFile(GATENEWS_LOCK_FILE,
-                             # it's okay to hijack this
-                             lifetime=LOCK_LIFETIME)
-    try:
-        lock.lock(timeout=0.5)
-    except LockFile.TimeOutError:
-        log.error('Could not acquire gate_news lock')
-        return
-    try:
-        process_lists(lock)
-    finally:
-        clearcache()
-        lock.unlock(unconditionally=True)
-
-
-
-if __name__ == '__main__':
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], 'h', ['help'])
-    except getopt.error, msg:
-        usage(1, msg)
-
-    if args:
-        usage(1, 'No args are expected')
-
-    for opt, arg in opts:
-        if opt in ('-h', '--help'):
-            usage(0)
-
-    main()
diff --git a/cron/mailpasswds b/cron/mailpasswds
deleted file mode 100755
index 5745265a1..000000000
--- a/cron/mailpasswds
+++ /dev/null
@@ -1,241 +0,0 @@
-#! @PYTHON@
-#
-# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-
-"""Send password reminders for all lists to all users.
-
-This program scans all mailing lists and collects users and their passwords,
-grouped by the list's host_name if mm_cfg.VIRTUAL_HOST_OVERVIEW is true.  Then
-one email message is sent to each unique user (per-virtual host) containing
-the list passwords and options url for the user.  The password reminder comes
-from the mm_cfg.MAILMAN_SITE_LIST, which must exist.
-
-Usage: %(PROGRAM)s [options]
-
-Options:
-    -l listname
-    --listname=listname
-        Send password reminders for the named list only.  If omitted,
-        reminders are sent for all lists.  Multiple -l/--listname options are
-        allowed.
-
-    -h/--help
-        Print this message and exit.
-"""
-
-# This puppy should probably do lots of logging.
-import sys
-import os
-import errno
-import getopt
-from types import UnicodeType
-
-import paths
-# mm_cfg must be imported before the other modules, due to the side-effect of
-# it hacking sys.paths to include site-packages.  Without this, running this
-# script from cron with python -S will fail.
-from Mailman import mm_cfg
-from Mailman import MailList
-from Mailman import Errors
-from Mailman import Utils
-from Mailman import Message
-from Mailman import i18n
-from Mailman.Logging.Syslog import syslog
-
-# Work around known problems with some RedHat cron daemons
-import signal
-signal.signal(signal.SIGCHLD, signal.SIG_DFL)
-
-NL = '\n'
-PROGRAM = sys.argv[0]
-
-_ = i18n._
-
-
-
-def usage(code, msg=''):
-    if code:
-        fd = sys.stderr
-    else:
-        fd = sys.stdout
-    print >> fd, _(__doc__)
-    if msg:
-        print >> fd, msg
-    sys.exit(code)
-
-
-
-def tounicode(s, enc):
-    if isinstance(s, UnicodeType):
-        return s
-    return unicode(s, enc, 'replace')
-
-
-
-def main():
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], 'l:h',
-                                   ['listname=', 'help'])
-    except getopt.error, msg:
-        usage(1, msg)
-
-    if args:
-        usage(1)
-
-    listnames = None
-    for opt, arg in opts:
-        if opt in ('-h', '--help'):
-            usage(0)
-        if opt in ('-l', '--listname'):
-            if listnames is None:
-                listnames = [arg]
-            else:
-                listnames.append(arg)
-
-    if listnames is None:
-        listnames = Utils.list_names()
-
-    # This is the list that all the reminders will look like they come from,
-    # but with the host name coerced to the virtual host we're processing.
-    try:
-        sitelist = MailList.MailList(mm_cfg.MAILMAN_SITE_LIST, lock=0)
-    except Errors.MMUnknownListError:
-        # Do it this way for I18n's _()
-        sitelistname = mm_cfg.MAILMAN_SITE_LIST
-        print >> sys.stderr, _('Site list is missing: %(sitelistname)s')
-        syslog('error', 'Site list is missing: %s', mm_cfg.MAILMAN_SITE_LIST)
-        sys.exit(1)
-
-    # Group lists by host_name if VIRTUAL_HOST_OVERVIEW is true, otherwise
-    # there's only one key in this dictionary: mm_cfg.DEFAULT_EMAIL_HOST.  The
-    # values are lists of the unlocked MailList instances.
-    byhost = {}
-    for listname in listnames:
-        mlist = MailList.MailList(listname, lock=0)
-        if not mlist.send_reminders:
-            continue
-        if mm_cfg.VIRTUAL_HOST_OVERVIEW:
-            host = mlist.host_name
-        else:
-            # See the note in Defaults.py concerning DEFAULT_HOST_NAME
-            # vs. DEFAULT_EMAIL_HOST.
-            host = mm_cfg.DEFAULT_HOST_NAME or mm_cfg.DEFAULT_EMAIL_HOST
-        byhost.setdefault(host, []).append(mlist)
-
-    # Now for each virtual host, collate the user information.  Each user
-    # entry has the form (listaddr, password, optionsurl)
-    for host in byhost.keys():
-        # Site owner is `mailman@dom.ain'
-        userinfo = {}
-        for mlist in byhost[host]:
-            listaddr = mlist.GetListEmail()
-            for member in mlist.getMembers():
-                # The user may have disabled reminders for this list
-                if mlist.getMemberOption(member,
-                                         mm_cfg.SuppressPasswordReminder):
-                    continue
-                # Group by the lower-cased address, since Mailman always
-                # treates person@dom.ain the same as PERSON@dom.ain.
-                try:
-                    password = mlist.getMemberPassword(member)
-                except Errors.NotAMemberError:
-                    # Here's a member with no passwords, which I think was
-                    # possible in older versions of Mailman.  Log this and
-                    # move on.
-                    syslog('error', 'password-less member %s for list %s',
-                           member, mlist.internal_name())
-                    continue
-                optionsurl = mlist.GetOptionsURL(member)
-                lang = mlist.getMemberLanguage(member)
-                info = (listaddr, password, optionsurl, lang)
-                userinfo.setdefault(member, []).append(info)
-        # Now that we've collected user information for this host, send each
-        # user the password reminder.
-        for addr in userinfo.keys():
-            # If the person is on more than one list, it is possible that they
-            # have different preferred languages, and there's no good way to
-            # know which one they want their password reminder in.  Pick the
-            # most popular, and break the tie randomly.
-            #
-            # Also, we need an example -request address for cronpass.txt and
-            # again, there's no clear winner.  Just take the first one in this
-            # case.
-            table = []
-            langs = {}
-            for listaddr, password, optionsurl, lang in userinfo[addr]:
-                langs[lang] = langs.get(lang, 0) + 1
-                # If the list address is really long, break it across two
-                # lines.
-                if len(listaddr) > 39:
-                    fmt = '%s\n           %-10s\n%s\n'
-                else:
-                    fmt = '%-40s %-10s\n%s\n'
-                table.append(fmt % (listaddr, password, optionsurl))
-            # Figure out which language to use
-            langcnt = 0
-            poplang = None
-            for lang, cnt in langs.items():
-                if cnt > langcnt:
-                    poplang = lang
-                    langcnt = cnt
-            enc = Utils.GetCharSet(poplang)
-            # Now we're finally ready to send the email!
-            siteowner = Utils.get_site_email(host, 'owner')
-            sitereq = Utils.get_site_email(host, 'request')
-            sitebounce = Utils.get_site_email(host, 'bounces')
-            text = Utils.maketext(
-                'cronpass.txt',
-                {'hostname': host,
-                 'useraddr': addr,
-                 'exreq'   : sitereq,
-                 'owner'   : siteowner,
-                 }, lang=poplang)
-            # Coerce everything to Unicode
-            text = tounicode(text, enc)
-            table = [tounicode(_t, enc) for _t in table]
-            # Translate the message and headers to user's suggested lang
-            otrans = i18n.get_translation()
-            try:
-                i18n.set_language(poplang)
-                # Craft table header after language was set
-                header = '%-40s %-10s\n%-40s %-10s' % (
-                         _('List'), _('Password // URL'), '----', '--------')
-                header = tounicode(header, enc)
-                # Add the table to the end so it doesn't get wrapped/filled
-                text += (header + '\n' + NL.join(table))
-                msg = Message.UserNotification(
-                    addr, siteowner,
-                    _('%(host)s mailing list memberships reminder'),
-                    text.encode(enc, 'replace'), poplang)
-                # Note that text must be encoded into 'enc' because unicode
-                # cause error within email module in some language (Japanese).
-            finally:
-                i18n.set_translation(otrans)
-            msg['X-No-Archive'] = 'yes'
-            # We want to make this look like it's coming from the siteowner's
-            # list, but we also want to be sure that the apparent host name is
-            # the current virtual host.  Look in CookHeaders.py for why this
-            # trick works.  Blarg.
-            msg.send(sitelist, **{'errorsto': sitebounce,
-                                  '_nolist' : 1,
-                                  'verp'    : mm_cfg.VERP_PASSWORD_REMINDERS,
-                                  })
-
-
-
-if __name__ == '__main__':
-    main()
diff --git a/cron/nightly_gzip b/cron/nightly_gzip
deleted file mode 100644
index 0a0f4e332..000000000
--- a/cron/nightly_gzip
+++ /dev/null
@@ -1,156 +0,0 @@
-#! @PYTHON@
-# 
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software 
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-#
-"""Re-generate the Pipermail gzip'd archive flat files.
-
-This script should be run nightly from cron.  When run from the command line,
-the following usage is understood:
-
-Usage: %(program)s [-v] [-h] [listnames]
-
-Where:
-    --verbose
-    -v
-        print each file as it's being gzip'd
-
-    --help
-    -h
-        print this message and exit
-
-    listnames
-        Optionally, only compress the .txt files for the named lists.  Without 
-        this, all archivable lists are processed.
-
-"""
-
-import sys
-import os
-import time
-from stat import *
-import getopt
-
-try:
-    import gzip
-except ImportError:
-    gzip = None
-
-import paths
-# mm_cfg must be imported before the other modules, due to the side-effect of
-# it hacking sys.paths to include site-packages.  Without this, running this
-# script from cron with python -S will fail.
-from Mailman import mm_cfg
-from Mailman import Utils
-from Mailman import MailList
-
-
-
-program = sys.argv[0]
-VERBOSE = 0
-
-def usage(code, msg=''):
-    if code:
-        fd = sys.stderr
-    else:
-        fd = sys.stdout
-    print >> fd, _(__doc__) % globals()
-    if msg:
-        print >> fd, msg
-    sys.exit(code)
-
-
-
-def compress(txtfile):
-    if VERBOSE:
-        print "gzip'ing:", txtfile
-    infp = open(txtfile)
-    outfp = gzip.open(txtfile+'.gz', 'wb', 6)
-    outfp.write(infp.read())
-    outfp.close()
-    infp.close()
-
-
-
-def main():
-    global VERBOSE
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], 'vh', ['verbose', 'help'])
-    except getopt.error, msg:
-        usage(1, msg)
-
-    # defaults
-    for opt, arg in opts:
-        if opt in ('-h', '--help'):
-            usage(0)
-        elif opt in ('-v', '--verbose'):
-            VERBOSE = 1
-
-    # limit to the specified lists?
-    if args:
-        listnames = args
-    else:
-        listnames = Utils.list_names()
-
-    # process all the specified lists
-    for name in listnames:
-        mlist = MailList.MailList(name, lock=0)
-        if not mlist.archive:
-            continue
-        dir = mlist.archive_dir()
-        try:
-            allfiles = os.listdir(dir)
-        except os.error:
-            # has the list received any messages?  if not, last_post_time will
-            # be zero, so it's not really a bogus archive dir.
-            if mlist.last_post_time > 0:
-                print 'List', name, 'has a bogus archive_directory:', dir
-            continue
-        if VERBOSE:
-            print 'Processing list:', name
-        files = []
-        for f in allfiles:
-            if f[-4:] <> '.txt':
-                continue
-            # stat both the .txt and .txt.gz files and append them only if 
-            # the former is newer than the latter.
-            txtfile = os.path.join(dir, f)
-            gzpfile = txtfile + '.gz'
-            txt_mtime = os.stat(txtfile)[ST_MTIME]
-            try:
-                gzp_mtime = os.stat(gzpfile)[ST_MTIME]
-            except os.error:
-                gzp_mtime = -1
-            if txt_mtime > gzp_mtime:
-                files.append(txtfile)
-        for f in files:
-            compress(f)
-
-        
-
-if __name__ == '__main__' and \
-   gzip is not None and \
-   mm_cfg.ARCHIVE_TO_MBOX in (1, 2) and \
-   not mm_cfg.GZIP_ARCHIVE_TXT_FILES:
-    # we're only going to run the nightly archiver if messages are archived to
-    # the mbox, and the gzip file is not created on demand (i.e. for every
-    # individual post).  This is the normal mode of operation.  Also, be sure
-    # we can actually import the gzip module!
-    omask = os.umask(002)
-    try:
-        main()
-    finally:
-        os.umask(omask)
diff --git a/cron/senddigests b/cron/senddigests
deleted file mode 100755
index d3f2781bc..000000000
--- a/cron/senddigests
+++ /dev/null
@@ -1,94 +0,0 @@
-#! @PYTHON@
-#
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software 
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-
-"""Dispatch digests for lists w/pending messages and digest_send_periodic set.
-
-Usage: %(PROGRAM)s [options]
-
-Options:
-    -h / --help
-        Print this message and exit.
-
-    -l listname
-    --listname=listname
-        Send the digest for the given list only, otherwise the digests for all
-        lists are sent out.
-"""
-
-import sys
-import getopt
-
-import paths
-from Mailman import mm_cfg
-from Mailman import Utils
-from Mailman import MailList
-from Mailman.i18n import _
-
-# Work around known problems with some RedHat cron daemons
-import signal
-signal.signal(signal.SIGCHLD, signal.SIG_DFL)
-
-PROGRAM = sys.argv[0]
-
-
-
-def usage(code, msg=''):
-    if code:
-        fd = sys.stderr
-    else:
-        fd = sys.stdout
-    print >> fd, _(__doc__)
-    if msg:
-        print >> fd, msg
-    sys.exit(code)
-
-
-
-def main():
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], 'hl:', ['help', 'listname='])
-    except getopt.error, msg:
-        usage(1, msg)
-
-    if args:
-        usage(1)
-
-    listnames = []
-    for opt, arg in opts:
-        if opt in ('-h', '--help'):
-            usage(0)
-        elif opt in ('-l', '--listname'):
-            listnames.append(arg)
-
-    if not listnames:
-        listnames = Utils.list_names()
-
-    for listname in listnames:
-        mlist = MailList.MailList(listname, lock=0)
-        if mlist.digest_send_periodic:
-            mlist.Lock()
-            try:
-                mlist.send_digest_now()
-                mlist.Save()
-            finally:
-                mlist.Unlock()
-
-
-
-if __name__ == '__main__':
-    main()