back port from 2.1.6 / adding new files.

12 files changed, 667 insertions, 120 deletions

diff --git a/bin/Makefile.in b/bin/Makefile.in
index 15c3a810d..dc4eee833 100644
--- a/bin/Makefile.in
+++ b/bin/Makefile.in
@@ -1,17 +1,17 @@
-# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2004 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software 
+# along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 
 # NOTE: Makefile.in is converted into Makefile by the configure script
@@ -48,7 +48,8 @@ SCRIPTS=	mmsitepass newlist rmlist add_members \
 		version config_list list_lists dumpdb cleanarch \
 		list_admins genaliases change_pw mailmanctl qrunner inject \
 		unshunt fix_url.py convert.py transcheck b4b5-archfix \
-		list_owners msgfmt.py show_qfiles discard
+		list_owners msgfmt.py show_qfiles discard rb-archfix \
+		reset_pw.py
 
 BUILDDIR=	../build/bin
 
diff --git a/bin/arch b/bin/arch
index 9abef3e41..40a75a0e3 100644
--- a/bin/arch
+++ b/bin/arch
@@ -163,7 +163,20 @@ def main():
         lock.lock()
         # Maybe wipe the old archives
         if wipe:
+            if mlist.scrub_nondigest:
+                # TK: save the attachments dir because they are not in mbox
+                saved = 0
+                try:
+                    atchdir = os.path.join(mlist.archive_dir(), 'attachments')
+                    savedir = os.path.join(mlist.archive_dir() + '.mbox',
+                                           'attachments')
+                    os.rename(atchdir, savedir)
+                    saved = 1
+                except:
+                    pass
             shutil.rmtree(mlist.archive_dir())
+            if mlist.scrub_nondigest and saved:
+                os.renames(savedir, atchdir)
         try:
             fp = open(mbox)
         except IOError, msg:
diff --git a/bin/change_pw b/bin/change_pw
index 1426420b0..1305e1a4c 100644
--- a/bin/change_pw
+++ b/bin/change_pw
@@ -1,19 +1,19 @@
 #! @PYTHON@
 #
-# Copyright (C) 2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 2001-2004 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software 
+# along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 
 """Change a list's password.
@@ -123,7 +123,7 @@ def main():
     domains = {}
     password = None
     quiet = 0
-    
+
     for opt, arg in opts:
         if opt in ('-h', '--help'):
             usage(0)
@@ -142,7 +142,7 @@ def main():
     if args:
         strargs = SPACE.join(args)
         usage(1, _('Bad arguments: %(strargs)s'))
-    
+
     if password is not None:
         if not password:
             usage(1, _('Empty list passwords are not allowed'))
@@ -164,7 +164,8 @@ def main():
         mlist.Lock()
         try:
             if password is None:
-                randompw = Utils.MakeRandomPassword(8)
+                randompw = Utils.MakeRandomPassword(
+                    mm_cfg.ADMIN_PASSWORD_LENGTH)
                 shapassword = sha.new(randompw).hexdigest()
                 notifypassword = randompw
             else:
diff --git a/bin/check_perms b/bin/check_perms
index f281ba44c..7c807745d 100755
--- a/bin/check_perms
+++ b/bin/check_perms
@@ -1,6 +1,6 @@
 #! @PYTHON@
 #
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2005 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -23,7 +23,6 @@ Usage: %(PROGRAM)s [-f] [-v] [-h]
 With no arguments, just check and report all the files that have bogus
 permissions or group ownership.  With -f (and run as root), fix all the
 permission problems found.  With -v be verbose.
-
 """
 
 import os
@@ -86,6 +85,18 @@ def statgidmode(path):
 
 seen = {}
 
+# libc's getgrgid re-opens /etc/group each time :(
+_gidcache = {}
+
+def getgrgid(gid):
+    data = _gidcache.get(gid)
+    if data is None:
+        data = grp.getgrgid(gid)
+        _gidcache[gid] = data
+    return data
+
+
+
 def checkwalk(arg, dirname, names):
     # Short-circuit duplicates
     if seen.has_key(dirname):
@@ -102,7 +113,7 @@ def checkwalk(arg, dirname, names):
             continue
         if gid <> MAILMAN_GID:
             try:
-                groupname = grp.getgrgid(gid)[0]
+                groupname = getgrgid(gid)[0]
             except KeyError:
                 groupname = '<anon gid %d>' % gid
             arg.ERRORS += 1
@@ -198,7 +209,14 @@ def checkarchives():
             os.chmod(private, mode & ~S_IROTH)
         else:
             print
-
+    # In addition, on a multiuser system you may want to hide the private
+    # archives so other users can't read them.
+    if mode & S_IXOTH:
+        print _("""\
+Warning: Private archive directory is other-executable (o+x).
+         This could allow other users on your system to read private archives.
+         If you're on a shared multiuser system, you should consult the
+         installation manual on how to fix this.""")
 
 MBOXPERMS = S_IRGRP | S_IWGRP | S_IRUSR | S_IWUSR
 
diff --git a/bin/config_list b/bin/config_list
index 15a349fdf..6f410a610 100644
--- a/bin/config_list
+++ b/bin/config_list
@@ -1,6 +1,6 @@
 #! @PYTHON@
 #
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -283,6 +283,15 @@ def do_input(listname, infile, checkonly, verbose):
                     if k == 'subscribe_policy' and \
                            not mm_cfg.ALLOW_OPEN_SUBSCRIBE:
                         validval -= 1
+                    # BAW: Another horrible hack.  This one is just too hard
+                    # to fix in a principled way in Mailman 2.1
+                    elif k == 'new_member_options':
+                        # Because this is a Checkbox, _getValidValue()
+                        # transforms the value into a list of one item.
+                        validval = validval[0]
+                        validval = [bitfield for bitfield, bitval
+                                    in mm_cfg.OPTINFO.items()
+                                    if validval & bitval]
                     gui._setValue(mlist, k, validval, fakedoc)
             # BAW: when to do gui._postValidate()???
     finally:
diff --git a/bin/dumpdb b/bin/dumpdb
index 2b06ec6ae..0bde09b80 100644
--- a/bin/dumpdb
+++ b/bin/dumpdb
@@ -1,6 +1,6 @@
 #! @PYTHON@
 #
-# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2005 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -45,22 +45,27 @@ Python pickle.  In either case, if you want to override the default assumption
 -- or if the file ends in neither suffix -- use the -p or -m flags.
 """
 
-import sys
 import os
+import sys
 import getopt
 import pprint
-import cPickle as pickle
+from cPickle import load
+from types import StringType
 
 import paths
 # Import this /after/ paths so that the sys.path is properly hacked
 from email.Generator import Generator
-
-from Mailman.Queue.Switchboard import DumperSwitchboard
 from Mailman.i18n import _
 
 PROGRAM = sys.argv[0]
 COMMASPACE = ', '
 
+try:
+    True, False
+except NameError:
+    True = 1
+    False = 0
+
 
 
 def usage(code, msg=''):
@@ -85,7 +90,7 @@ def main():
     # Options.
     # None == guess, 0 == pickle, 1 == marshal
     filetype = None
-    doprint = 1
+    doprint = True
 
     for opt, arg in opts:
         if opt in ('-h', '--help'):
@@ -95,7 +100,7 @@ def main():
         elif opt in ('-m', '--marshal'):
             filetype = 1
         elif opt in ('-n', '--noprint'):
-            doprint = 0
+            doprint = False
 
     if len(args) < 1:
         usage(1, _('No filename given.'))
@@ -123,9 +128,29 @@ def main():
             pp.pprint(d)
         return d
     else:
-        m = pickle.load(open(filename))
-        if doprint:
-            pp.pprint(m)
+        fp = open(filename)
+        m = []
+        try:
+            cnt = 1
+            if doprint:
+                print _('[----- start pickle file -----]')
+            while True:
+                try:
+                    obj = load(fp)
+                except EOFError:
+                    if doprint:
+                        print _('[----- end pickle file -----]')
+                    break
+                if doprint:
+                    print _('<----- start object %(cnt)s ----->')
+                    if isinstance(obj, StringType):
+                        print obj
+                    else:
+                        pp.pprint(obj)
+                cnt += 1
+                m.append(obj)
+        finally:
+            fp.close()
         return m
 
 
diff --git a/bin/mailmanctl b/bin/mailmanctl
index 630c7c347..cabf29a5c 100644
--- a/bin/mailmanctl
+++ b/bin/mailmanctl
@@ -1,6 +1,6 @@
 #! @PYTHON@
 
-# Copyright (C) 2001-2003 by the Free Software Foundation, Inc.
+# Copyright (C) 2001-2004 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -288,6 +288,14 @@ def check_privs():
     uid = pwd.getpwnam(mm_cfg.MAILMAN_USER)[2]
     myuid = os.getuid()
     if myuid == 0:
+        # Set the process's supplimental groups.
+        groups = [x[2] for x in grp.getgrall() if mm_cfg.MAILMAN_USER in x[3]]
+        groups.append(gid)
+        try:
+            os.setgroups(groups)
+        except AttributeError:
+            # Python 2.1 doesn't have setgroups
+            syslog('error', 'Warning: unable to setgroups(%s)' % groups)
         os.setgid(gid)
         os.setuid(uid)
     elif myuid <> uid:
diff --git a/bin/newlist b/bin/newlist
index a86aa709b..70e9cb8c1 100755
--- a/bin/newlist
+++ b/bin/newlist
@@ -1,19 +1,19 @@
 #! @PYTHON@
 #
-# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2005 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation; either version 2
 # of the License, or (at your option) any later version.
-# 
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software 
+# along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 
 """Create a new, unpopulated mailing list.
@@ -23,10 +23,18 @@ Usage: %(PROGRAM)s [options] [listname [listadmin-addr [admin-password]]]
 Options:
 
     -l language
-    --language language
+    --language=language
         Make the list's preferred language `language', which must be a two
         letter language code.
 
+    -u urlhost
+    --urlhost=urlhost
+        Gives the list's web interface host name.
+
+    -e emailhost
+    --emailhost=emailhost
+        Gives the list's email domain name.
+
     -q/--quiet
         Normally the administrator is notified by email (after a prompt) that
         their list has been created.  This option suppresses the prompt and
@@ -44,18 +52,35 @@ configured Mailman, certain defaults were calculated, but if you are running
 multiple virtual Mailman sites, then the defaults may not be appropriate for
 the list you are creating.
 
-You can specify the domain to create your new list in by spelling the listname
+You also specify the domain to create your new list in by typing the command
 like so:
 
-    mylist@www.mydom.ain
+    newlist --urlhost=www.mydom.ain mylist
 
 where `www.mydom.ain' should be the base hostname for the URL to this virtual
-hosts's lists.  E.g. with is setting people will view the general list
+hosts's lists.  E.g. with this setting people will view the general list
 overviews at http://www.mydom.ain/mailman/listinfo.  Also, www.mydom.ain
-should be a key in the VIRTUAL_HOSTS mapping in mm_cfg.py/Defaults.py.  It
-will be looked up to give the email hostname.  If this can't be found, then
-www.mydom.ain will be used for both the web interface and the email
-interface.
+should be a key in the VIRTUAL_HOSTS mapping in mm_cfg.py/Defaults.py if
+the email hostname to be automatically determined.
+
+If you want the email hostname to be different from the one looked up by the
+VIRTUAL_HOSTS or if urlhost is not registered in VIRTUAL_HOSTS, you can specify
+`emailhost' like so:
+
+    newlist --urlhost=www.mydom.ain --emailhost=mydom.ain mylist
+
+where `mydom.ain' is the mail domain name. If you don't specify emailhost but
+urlhost is not in the virtual host list, then mm_cfg.DEFAULT_EMAIL_HOST will
+be used for the email interface.
+
+For backward compatibility, you can also specify the domain to create your
+new list in by spelling the listname like so:
+
+    mylist@www.mydom.ain
+
+where www.mydom.ain is used for `urlhost' but it will also be used for
+`emailhost' if it is not found in the virtual host table. Note that
+'--urlhost' and '--emailhost' have precedence to this notation.
 
 If you spell the list name as just `mylist', then the email hostname will be
 taken from DEFAULT_EMAIL_HOST and the url will be taken from DEFAULT_URL (as
@@ -98,13 +123,16 @@ def usage(code, msg=''):
 
 def main():
     try:
-        opts, args = getopt.getopt(sys.argv[1:], 'hql:',
-                                   ['help', 'quiet', 'language='])
+        opts, args = getopt.getopt(sys.argv[1:], 'hql:u:e:',
+                                   ['help', 'quiet', 'language=',
+                                    'urlhost=', 'emailhost='])
     except getopt.error, msg:
         usage(1, msg)
 
     lang = mm_cfg.DEFAULT_SERVER_LANGUAGE
     quiet = 0
+    urlhost = None
+    emailhost = None
     for opt, arg in opts:
         if opt in ('-h', '--help'):
             usage(0)
@@ -112,35 +140,43 @@ def main():
             quiet = 1
         if opt in ('-l', '--language'):
             lang = arg
+        if opt in ('-u', '--urlhost'):
+            urlhost = arg
+        if opt in ('-e', '--emailhost'):
+            emailhost = arg
 
     # Is the language known?
     if lang not in mm_cfg.LC_DESCRIPTIONS.keys():
         usage(1, _('Unknown language: %(lang)s'))
 
     if len(args) > 0:
-	listname = args[0]
+        listname = args[0]
     else:
-	listname = raw_input(_('Enter the name of the list: '))
+        listname = raw_input(_('Enter the name of the list: '))
     listname = listname.lower()
 
-    host_name = None
-    web_page_url = None
     if '@' in listname:
+        # note that --urlhost and --emailhost have precedence
         listname, domain = listname.split('@', 1)
-        host_name = mm_cfg.VIRTUAL_HOSTS.get(domain, domain)
-        web_page_url = mm_cfg.DEFAULT_URL_PATTERN % domain
+        urlhost = urlhost or domain
+        emailhost = emailhost or mm_cfg.VIRTUAL_HOSTS.get(domain, domain)
+
+    urlhost = urlhost or mm_cfg.DEFAULT_URL_HOST
+    host_name = emailhost or \
+                mm_cfg.VIRTUAL_HOSTS.get(urlhost, mm_cfg.DEFAULT_EMAIL_HOST)
+    web_page_url = mm_cfg.DEFAULT_URL_PATTERN % urlhost
 
     if Utils.list_exists(listname):
         usage(1, _('List already exists: %(listname)s'))
 
     if len(args) > 1:
-	owner_mail = args[1]
+        owner_mail = args[1]
     else:
-	owner_mail = raw_input(
-	    _('Enter the email of the person running the list: '))
+        owner_mail = raw_input(
+            _('Enter the email of the person running the list: '))
 
     if len(args) > 2:
-	listpasswd = args[2]
+        listpasswd = args[2]
     else:
         listpasswd = getpass.getpass(_('Initial %(listname)s password: '))
     # List passwords cannot be empty
@@ -162,15 +198,14 @@ def main():
                 os.umask(oldmask)
         except Errors.BadListNameError, s:
             usage(1, _('Illegal list name: %(s)s'))
-        except Errors.MMBadEmailError, s:
+        except Errors.EmailAddressError, s:
             usage(1, _('Bad owner email address: %(s)s'))
         except Errors.MMListAlreadyExistsError:
             usage(1, _('List already exists: %(listname)s'))
 
         # Assign domain-specific attributes
-        if host_name:
-            mlist.host_name = host_name
-            mlist.web_page_url = web_page_url
+        mlist.host_name = host_name
+        mlist.web_page_url = web_page_url
 
         # And assign the preferred language
         mlist.preferred_language = lang
@@ -189,15 +224,15 @@ def main():
     if not quiet:
         print _('Hit enter to notify %(listname)s owner...'),
         sys.stdin.readline()
-        siteadmin = Utils.get_site_email(mlist.host_name, 'admin')
+        siteowner = Utils.get_site_email(mlist.host_name, 'owner')
         text = Utils.maketext(
             'newlist.txt',
             {'listname'    : listname,
-             'password'    : listpasswd, 
-             'admin_url'   : mlist.GetScriptURL('admin', absolute=1), 
+             'password'    : listpasswd,
+             'admin_url'   : mlist.GetScriptURL('admin', absolute=1),
              'listinfo_url': mlist.GetScriptURL('listinfo', absolute=1),
              'requestaddr' : mlist.GetRequestEmail(),
-             'siteowner'   : siteadmin,
+             'siteowner'   : siteowner,
              }, mlist=mlist)
         # Set the I18N language to the list's preferred language so the header
         # will match the template language.  Stashing and restoring the old
@@ -206,7 +241,7 @@ def main():
         i18n.set_language(mlist.preferred_language)
         try:
             msg = Message.UserNotification(
-                owner_mail, siteadmin,
+                owner_mail, siteowner,
                 _('Your new mailing list: %(listname)s'),
                 text, mlist.preferred_language)
             msg.send(mlist)
diff --git a/bin/rb-archfix b/bin/rb-archfix
new file mode 100644
index 000000000..fceadc273
--- /dev/null
+++ b/bin/rb-archfix
@@ -0,0 +1,108 @@
+#! @PYTHON@
+#
+# Copyright (C) 2003 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+# Author: Richard Barrett
+
+"""Reduce disk space usage for Pipermail archives.
+
+Usage: %(PROGRAM)s [options] file ...
+
+Where options are:
+    -h / --help
+        Print this help message and exit.
+
+Only use this to 'fix' archive -article database files that have been written
+with Mailman 2.1.3 or earlier and have html_body attributes in them.  These
+attributes can cause huge amounts of memory bloat and impact performance for
+high activity lists, particularly those where large text postings are made to
+them.
+
+Example:
+
+%% ls -1 archives/private/*/database/*-article | xargs %(PROGRAM)s
+
+You should run `bin/check_perms -f' after running this script.
+
+You will probably want to delete the -article.bak files created by this script
+when you are satisfied with the results.
+
+This script is provided for convenience purposes only.  It isn't supported.
+"""
+
+import os
+import sys
+import getopt
+import marshal
+import cPickle as pickle
+
+# Required to get the right classes for unpickling
+import paths
+from Mailman.i18n import _
+
+PROGRAM = sys.argv[0]
+
+
+
+def usage(code, msg=''):
+    if code:
+        fd = sys.stderr
+    else:
+        fd = sys.stdout
+    print >> fd, _(__doc__)
+    if msg:
+        print >> fd, msg
+    sys.exit(code)
+
+
+
+def main():
+    # get command line arguments
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], 'h', ['help'])
+    except getopt.error, msg:
+        usage(1, msg)
+
+    for opt, arg in opts:
+        if opt in ('-h', '--help'):
+            usage(0)
+
+    for filename in args:
+        print 'processing:', filename
+        fp = open(filename, 'rb')
+        d = marshal.load(fp)
+        fp.close()
+        newd = {}
+        for key, pckstr in d.items():
+            article = pickle.loads(pckstr)
+            try:
+                del article.html_body
+            except AttributeError:
+                pass
+            newd[key] = pickle.dumps(article)
+        fp = open(filename + '.tmp', 'wb')
+        marshal.dump(newd, fp)
+        fp.close()
+        os.rename(filename, filename + '.bak')
+        os.rename(filename + '.tmp', filename)
+
+    print 'You should now run "bin/check_perms -f"'
+
+
+
+if __name__ == '__main__':
+    main()
diff --git a/bin/reset_pw.py b/bin/reset_pw.py
new file mode 100644
index 000000000..e829aefe6
--- /dev/null
+++ b/bin/reset_pw.py
@@ -0,0 +1,90 @@
+#! @PYTHON@
+#
+# Copyright (C) 2004 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+
+# Inspired by Florian Weimer.
+
+"""Reset the passwords for members of a mailing list.
+
+This script resets all the passwords of a mailing list's members.  It can also
+be used to reset the lists of all members of all mailing lists, but it is your
+responsibility to let the users know that their passwords have been changed.
+
+This script is intended to be run as a bin/withlist script, i.e.
+
+% bin/withlist -l -r reset_pw listname [options]
+
+Options:
+    -v / --verbose
+        Print what the script is doing.
+"""
+
+import sys
+import getopt
+
+import paths
+from Mailman import Utils
+from Mailman.i18n import _
+
+
+try:
+    True, False
+except NameError:
+    True = 1
+    False = 0
+
+
+
+def usage(code, msg=''):
+    if code:
+        fd = sys.stderr
+    else:
+        fd = sys.stdout
+    print >> fd, _(__doc__.replace('%', '%%'))
+    if msg:
+        print >> fd, msg
+    sys.exit(code)
+
+
+
+def reset_pw(mlist, *args):
+    try:
+        opts, args = getopt.getopt(args, 'v', ['verbose'])
+    except getopt.error, msg:
+        usage(1, msg)
+
+    verbose = False
+    for opt, args in opts:
+        if opt in ('-v', '--verbose'):
+            verbose = True
+
+    listname = mlist.internal_name()
+    if verbose:
+        print _('Changing passwords for list: %(listname)s')
+
+    for member in mlist.getMembers():
+        randompw = Utils.MakeRandomPassword()
+        mlist.setMemberPassword(member, randompw)
+        if verbose:
+            print _('New password for member %(member)40s: %(randompw)s')
+
+    mlist.Save()
+
+
+
+if __name__ == '__main__':
+    usage(0)
diff --git a/bin/update b/bin/update
index 5a6b5325b..23429508e 100755
--- a/bin/update
+++ b/bin/update
@@ -1,6 +1,6 @@
 #! @PYTHON@
 #
-# Copyright (C) 1998-2004 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2005 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -40,12 +40,17 @@ import time
 import errno
 import getopt
 import shutil
+import cPickle
 import marshal
 
 import paths
+import email
+
 from Mailman import mm_cfg
 from Mailman import Utils
 from Mailman import MailList
+from Mailman import Message
+from Mailman import Pending
 from Mailman.LockFile import TimeOutError
 from Mailman.i18n import _
 from Mailman.Queue.Switchboard import Switchboard
@@ -347,12 +352,16 @@ script.
         for f in os.listdir(b4_tmpl_dir):
             o_tmpl = os.path.join(b4_tmpl_dir, f)
             n_tmpl = os.path.join(new_tmpl_dir, f)
-            if not os.path.exists(n_tmpl):
-                os.rename(o_tmpl, n_tmpl)
-                print _('- moved %(o_tmpl)s to %(n_tmpl)s')
+            if os.path.exists(o_tmpl):
+                if not os.path.exists(n_tmpl):
+                    os.rename(o_tmpl, n_tmpl)
+                    print _('- moved %(o_tmpl)s to %(n_tmpl)s')
+                else:
+                    print _("""\
+- both %(o_tmpl)s and %(n_tmpl)s exist, leaving untouched""")
             else:
                 print _("""\
-- both %(o_tmpl)s and %(n_tmpl)s exist, leaving untouched""")
+- %(o_tmpl)s doesn't exist, leaving untouched""")
     #
     # Move all the templates to the en language subdirectory as required for
     # Mailman 2.1
@@ -402,17 +411,240 @@ def update_qfiles():
     # Be sure the qfiles/in directory exists (we don't really need the
     # switchboard object, but it's convenient for creating the directory).
     sb = Switchboard(mm_cfg.INQUEUE_DIR)
-    for file in os.listdir(mm_cfg.QUEUE_DIR):
-        # Updating means just mokving the .db and .msg files to qfiles/in where
+    for filename in os.listdir(mm_cfg.QUEUE_DIR):
+        # Updating means just moving the .db and .msg files to qfiles/in where
         # it should be dequeued, converted, and processed normally.
-        if file.endswith('.msg'):
-            oldmsgfile = os.path.join(mm_cfg.QUEUE_DIR, file)
-            newmsgfile = os.path.join(mm_cfg.INQUEUE_DIR, prefix + file)
+        if filename.endswith('.msg'):
+            oldmsgfile = os.path.join(mm_cfg.QUEUE_DIR, filename)
+            newmsgfile = os.path.join(mm_cfg.INQUEUE_DIR, prefix + filename)
             os.rename(oldmsgfile, newmsgfile)
-        elif file.endswith('.db'):
-            olddbfile = os.path.join(mm_cfg.QUEUE_DIR, file)
-            newdbfile = os.path.join(mm_cfg.INQUEUE_DIR, prefix + file)
+        elif filename.endswith('.db'):
+            olddbfile = os.path.join(mm_cfg.QUEUE_DIR, filename)
+            newdbfile = os.path.join(mm_cfg.INQUEUE_DIR, prefix + filename)
             os.rename(olddbfile, newdbfile)
+    # Now update for the Mailman 2.1.5 qfile format.  For every filebase in
+    # the qfiles/* directories that has both a .pck and a .db file, pull the
+    # data out and re-queue them.
+    for dirname in os.listdir(mm_cfg.QUEUE_DIR):
+        dirpath = os.path.join(mm_cfg.QUEUE_DIR, dirname)
+        if dirpath == mm_cfg.BADQUEUE_DIR:
+            # The files in qfiles/bad can't possibly be pickles
+            continue
+        sb = Switchboard(dirpath)
+        try:
+            for filename in os.listdir(dirpath):
+                filepath = os.path.join(dirpath, filename)
+                filebase, ext = os.path.splitext(filepath)
+                # Handle the .db metadata files as part of the handling of the
+                # .pck or .msg message files.
+                if ext not in ('.pck', '.msg'):
+                    continue
+                msg, data = dequeue(filebase)
+                if msg is not None and data is not None:
+                    sb.enqueue(msg, data)
+        except EnvironmentError, e:
+            if e.errno <> errno.ENOTDIR:
+                raise
+            print _('Warning!  Not a directory: %(dirpath)s')
+
+
+
+# Implementations taken from the pre-2.1.5 Switchboard
+def ext_read(filename):
+    fp = open(filename)
+    d = marshal.load(fp)
+    # Update from version 2 files
+    if d.get('version', 0) == 2:
+        del d['filebase']
+    # Do the reverse conversion (repr -> float)
+    for attr in ['received_time']:
+        try:
+            sval = d[attr]
+        except KeyError:
+            pass
+        else:
+            # Do a safe eval by setting up a restricted execution
+            # environment.  This may not be strictly necessary since we
+            # know they are floats, but it can't hurt.
+            d[attr] = eval(sval, {'__builtins__': {}})
+    fp.close()
+    return d
+
+
+def dequeue(filebase):
+    # Calculate the .db and .msg filenames from the given filebase.
+    msgfile = os.path.join(filebase + '.msg')
+    pckfile = os.path.join(filebase + '.pck')
+    dbfile = os.path.join(filebase + '.db')
+    # Now we are going to read the message and metadata for the given
+    # filebase.  We want to read things in this order: first, the metadata
+    # file to find out whether the message is stored as a pickle or as
+    # plain text.  Second, the actual message file.  However, we want to
+    # first unlink the message file and then the .db file, because the
+    # qrunner only cues off of the .db file
+    msg = None
+    try:
+        data = ext_read(dbfile)
+        os.unlink(dbfile)
+    except EnvironmentError, e:
+        if e.errno <> errno.ENOENT: raise
+        data = {}
+    # Between 2.1b4 and 2.1b5, the `rejection-notice' key in the metadata
+    # was renamed to `rejection_notice', since dashes in the keys are not
+    # supported in METAFMT_ASCII.
+    if data.has_key('rejection-notice'):
+        data['rejection_notice'] = data['rejection-notice']
+        del data['rejection-notice']
+    msgfp = None
+    try:
+        try:
+            msgfp = open(pckfile)
+            msg = cPickle.load(msgfp)
+            os.unlink(pckfile)
+        except EnvironmentError, e:
+            if e.errno <> errno.ENOENT: raise
+            msgfp = None
+            try:
+                msgfp = open(msgfile)
+                msg = email.message_from_file(msgfp, Message.Message)
+                os.unlink(msgfile)
+            except EnvironmentError, e:
+                if e.errno <> errno.ENOENT: raise
+            except (email.Errors.MessageParseError, ValueError), e:
+                # This message was unparsable, most likely because its
+                # MIME encapsulation was broken.  For now, there's not
+                # much we can do about it.
+                print _('message is unparsable: %(filebase)s')
+                msgfp.close()
+                msgfp = None
+                if mm_cfg.QRUNNER_SAVE_BAD_MESSAGES:
+                    # Cheapo way to ensure the directory exists w/ the
+                    # proper permissions.
+                    sb = Switchboard(mm_cfg.BADQUEUE_DIR)
+                    os.rename(msgfile, os.path.join(
+                        mm_cfg.BADQUEUE_DIR, filebase + '.txt'))
+                else:
+                    os.unlink(msgfile)
+                msg = data = None
+        except EOFError:
+            # For some reason the pckfile was empty.  Just delete it.
+            print _('Warning!  Deleting empty .pck file: %(pckfile)s')
+            os.unlink(pckfile)
+    finally:
+        if msgfp:
+            msgfp.close()
+    return msg, data
+
+
+
+def update_pending():
+    file20 = os.path.join(mm_cfg.DATA_DIR, 'pending_subscriptions.db')
+    file214 = os.path.join(mm_cfg.DATA_DIR, 'pending.pck')
+    db = None
+    # Try to load the Mailman 2.0 file
+    try:
+        fp = open(file20)
+    except IOError, e:
+        if e.errno <> errno.ENOENT: raise
+    else:
+        print _('Updating Mailman 2.0 pending_subscriptions.db database')
+        db = marshal.load(fp)
+        # Convert to the pre-Mailman 2.1.5 format
+        db = Pending._update(db)
+    if db is None:
+        # Try to load the Mailman 2.1.x where x < 5, file
+        try:
+            fp = open(file214)
+        except IOError, e:
+            if e.errno <> errno.ENOENT: raise
+        else:
+            print _('Updating Mailman 2.1.4 pending.pck database')
+            db = cPickle.load(fp)
+    if db is None:
+        print _('Nothing to do.')
+        return
+    # Now upgrade the database to the 2.1.5 format.  Each list now has its own
+    # pending.pck file, but only the RE_ENABLE operation actually recorded the
+    # listname in the request.  For the SUBSCRIPTION, UNSUBSCRIPTION, and
+    # CHANGE_OF_ADDRESS operations, we know the address of the person making
+    # the request so we can repend this request just for the lists the person
+    # is a member of.  For the HELD_MESSAGE operation, we can check the list's
+    # requests.pck file for correlation.  Evictions will take care of any
+    # misdirected pendings.
+    reenables_by_list = {}
+    addrops_by_address = {}
+    holds_by_id = {}
+    subs_by_address = {}
+    for key, val in db.items():
+        if key in ('evictions', 'version'):
+            continue
+        try:
+            op = val[0]
+            data = val[1:]
+        except (IndexError, ValueError):
+            print _('Ignoring bad pended data: %(key)s: %(val)s')
+            continue
+        if op in (Pending.UNSUBSCRIPTION, Pending.CHANGE_OF_ADDRESS):
+            # data[0] is the address being unsubscribed
+            addrops_by_address.setdefault(data[0], []).append((key, val))
+        elif op == Pending.SUBSCRIPTION:
+            # data[0] is a UserDesc object
+            addr = data[0].address
+            subs_by_address.setdefault(addr, []).append((key, val))
+        elif op == Pending.RE_ENABLE:
+            # data[0] is the mailing list's internal name
+            reenables_by_list.setdefault(data[0], []).append((key, val))
+        elif op == Pending.HELD_MESSAGE:
+            # data[0] is the hold id.  There better only be one entry per id
+            id = data[0]
+            if holds_by_id.has_key(id):
+                print _('WARNING: Ignoring duplicate pending ID: %(id)s.')
+            else:
+                holds_by_id[id] = (key, val)
+    # Now we have to lock every list and re-pend all the appropriate
+    # requests.  Note that this will reset all the expiration dates, but that
+    # should be fine.
+    for listname in Utils.list_names():
+        mlist = MailList.MailList(listname)
+        # This is not the most efficient way to do this because it loads and
+        # saves the pending.pck file each time. :(
+        try:
+            for cookie, data in reenables_by_list.get(listname, []):
+                mlist.pend_repend(cookie, data)
+            for id, (cookie, data) in holds_by_id.items():
+                try:
+                    rec = mlist.GetRecord(id)
+                except KeyError:
+                    # Not for this list
+                    pass
+                else:
+                    mlist.pend_repend(cookie, data)
+                    del holds_by_id[id]
+            for addr, recs in subs_by_address.items():
+                # We shouldn't have a subscription confirmation if the address
+                # is already a member of the mailing list.
+                if mlist.isMember(addr):
+                    continue
+                for cookie, data in recs:
+                    mlist.pend_repend(cookie, data)
+            for addr, recs in addrops_by_address.items():
+                # We shouldn't have unsubscriptions or change of address
+                # requests for addresses which aren't members of the list.
+                if not mlist.isMember(addr):
+                    continue
+                for cookie, data in recs:
+                    mlist.pend_repend(cookie, data)
+            mlist.Save()
+        finally:
+            mlist.Unlock()
+    try:
+        os.unlink(file20)
+    except OSError, e:
+        if e.errno <> errno.ENOENT: raise
+    try:
+        os.unlink(file214)
+    except OSError, e:
+        if e.errno <> errno.ENOENT: raise
 
 
 
@@ -475,31 +707,18 @@ If your archives are big, this could take a minute or two...""")
                 mlist.Unlock()
         os.unlink(wmfile)
         print _('- usenet watermarks updated and gate_watermarks removed')
-    #
-    # In Mailman 2.1, the pending database format and file name has changed.
-    #
-    oldpendingfile = os.path.join(mm_cfg.DATA_DIR, 'pending_subscriptions.db')
-    try:
-        fp = open(oldpendingfile)
-    except IOError, e:
-        if e.errno <> errno.ENOENT: raise
-    else:
-        print _('Updating old pending_subscriptions.db database')
-        from Mailman import Pending
-        db = marshal.load(fp)
-        Pending._update(db)
-        fp.close()
-        os.unlink(oldpendingfile)
-    #
+    # In Mailman 2.1, the pending database format and file name changed, but
+    # in Mailman 2.1.5 it changed again.  This should update all existing
+    # files to the 2.1.5 format.
+    update_pending()
     # In Mailman 2.1, the qfiles directory has a different structure and a
-    # different content.
-    #
+    # different content.  Also, in Mailman 2.1.5 we collapsed the message
+    # files from separate .msg (pickled Message objects) and .db (marshalled
+    # dictionaries) to a shared .pck file containing two pickles.
     update_qfiles()
-    #
     # This warning was necessary for the upgrade from 1.0b9 to 1.0b10.
     # There's no good way of figuring this out for releases prior to 2.0beta2
     # :(
-    #
     if lastversion == NOTFRESH:
         print _("""
 
@@ -536,8 +755,6 @@ def usage(code, msg=''):
 
 
 if __name__ == '__main__':
-    print _('XXX UPDATE CURRENTLY DISABLED.  USE Release_2_1-maint branch')
-    sys.exit(1)
     try:
         opts, args = getopt.getopt(sys.argv[1:], 'hf',
                                    ['help', 'force'])
diff --git a/bin/withlist b/bin/withlist
index e717a8609..ac0967b99 100644
--- a/bin/withlist
+++ b/bin/withlist
@@ -1,6 +1,6 @@
 #! @PYTHON@
 #
-# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2004 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -49,12 +49,12 @@ Options:
     --run [module.]callable
     -r [module.]callable
         This can be used to run a script with the opened MailList object.
-        This works by attempting to import `module' (which must already be
-        accessible on your sys.path), and then calling `callable' from the
-        module.  callable can be a class or function; it is called with the
-        MailList object as the first argument.  If additional args are given
-        on the command line, they are passed as subsequent positional args to
-        the callable.
+        This works by attempting to import `module' (which must be in the
+        directory containing withlist, or already be accessible on your
+        sys.path), and then calling `callable' from the module.  callable can
+        be a class or function; it is called with the MailList object as the
+        first argument.  If additional args are given on the command line,
+        they are passed as subsequent positional args to the callable.
 
         Note that `module.' is optional; if it is omitted then a module with
         the name `callable' will be imported.
@@ -115,24 +115,36 @@ def changepw(mlist, addr, newpasswd):
         print 'No address matched:', addr
 
 and run this from the command line:
-%% bin/withlist -l -r changepw mylist somebody@somewhere.org foobar
+ %% bin/withlist -l -r changepw mylist somebody@somewhere.org foobar
 """
 
+import os
 import sys
-import getopt
 import code
+import getopt
 
 import paths
-from Mailman import Utils
-from Mailman import MailList
 from Mailman import Errors
+from Mailman import MailList
+from Mailman import Utils
 from Mailman.i18n import _
 
+try:
+    True, False
+except NameError:
+    True = 1
+    False = 0
+
+
 # `m' will be the MailList object and `r' will be the results of the callable
 m = None
 r = None
-VERBOSE = 1
-LOCK = 0
+VERBOSE = True
+LOCK = False
+
+
+# Put the bin directory on sys.path -- last
+sys.path.append(os.path.dirname(sys.argv[0]))
 
 
 
@@ -204,23 +216,30 @@ def main():
 
     run = None
     interact = None
-    all = 0
+    all = False
+    dolist = True
     for opt, arg in opts:
         if opt in ('-h', '--help'):
             usage(0)
         elif opt in ('-l', '--lock'):
-            LOCK = 1
+            LOCK = True
         elif opt in ('-r', '--run'):
             run = arg
         elif opt in ('-q', '--quiet'):
-            VERBOSE = 0
+            VERBOSE = False
         elif opt in ('-i', '--interactive'):
-            interact = 1
+            interact = True
         elif opt in ('-a', '--all'):
-            all = 1
+            all = True
 
     if len(args) < 1 and not all:
-        usage(1, _('No list name supplied.'))
+        warning = _('No list name supplied.')
+        if interact:
+            # Let them keep going
+            print warning
+            dolist = False
+        else:
+            usage(1, warning)
 
     if all and not run:
         usage(1, _('--all requires --run'))
@@ -228,9 +247,9 @@ def main():
     # The default for interact is 1 unless -r was given
     if interact is None:
         if run is None:
-            interact = 1
+            interact = True
         else:
-            interact = 0
+            interact = False
 
     # try to import the module for the callable
     func = None
@@ -251,7 +270,7 @@ def main():
 
     if all:
         r = [do_list(listname, args, func) for listname in Utils.list_names()]
-    else:
+    elif dolist:
         listname = args.pop(0).lower().strip()
         r = do_list(listname, args, func)
 
@@ -266,8 +285,11 @@ def main():
             pass
         namespace = globals().copy()
         namespace.update(locals())
-        code.InteractiveConsole(namespace).interact(
-            _("The variable `m' is the %(listname)s MailList instance"))
+        if dolist:
+            ban = _("The variable `m' is the %(listname)s MailList instance")
+        else:
+            ban = None
+        code.InteractiveConsole(namespace).interact(ban)