7 files changed, 361 insertions, 19 deletions

diff --git a/Mailman/MailList.py b/Mailman/MailList.py
index 43ae3046c..9f790c242 100644
--- a/Mailman/MailList.py
+++ b/Mailman/MailList.py
@@ -173,7 +173,7 @@ class MailList(object, HTMLFormatter, Deliverer, ListAdmin,
         else:
             status = '(unlocked)'
         return '<mailing list "%s" %s at %x>' % (
-            self.internal_name(), status, id(self))
+            self.fqdn_listname, status, id(self))
 
 
     #
diff --git a/Mailman/bin/export.py b/Mailman/bin/export.py
index 04f18ef16..2bcdbf667 100644
--- a/Mailman/bin/export.py
+++ b/Mailman/bin/export.py
@@ -17,7 +17,10 @@
 
 """Export an XML representation of a mailing list."""
 
+import os
 import sys
+import sha
+import base64
 import datetime
 import optparse
 
@@ -40,6 +43,7 @@ DOLLAR_STRINGS  = ('msg_header', 'msg_footer',
                    'autoresponse_postings_text',
                    'autoresponse_admin_text',
                    'autoresponse_request_text')
+SALT_LENGTH     = 4 # bytes
 
 TYPES = {
     Defaults.Toggle         : 'bool',
@@ -177,7 +181,7 @@ class XMLDumper(object):
             else:
                 self._element('option', value, name=varname, type=widget_type)
 
-    def _dump_list(self, mlist, with_passwords):
+    def _dump_list(self, mlist, password_scheme):
         # Write list configuration values
         self._push_element('list', name=mlist.fqdn_listname)
         self._push_element('configuration')
@@ -202,8 +206,8 @@ class XMLDumper(object):
                 attrs['original'] = cased
             self._push_element('member', **attrs)
             self._element('realname', mlist.getMemberName(member))
-            if with_passwords:
-                self._element('password', mlist.getMemberPassword(member))
+            self._element('password',
+                          password_scheme(mlist.getMemberPassword(member)))
             self._element('language', mlist.getMemberLanguage(member))
             # Delivery status, combined with the type of delivery
             attrs = {}
@@ -247,7 +251,7 @@ class XMLDumper(object):
         self._pop_element('roster')
         self._pop_element('list')
 
-    def dump(self, listnames, with_passwords=False):
+    def dump(self, listnames, password_scheme):
         print >> self._fp, '<?xml version="1.0" encoding="UTF-8"?>'
         self._push_element('mailman', **{
             'xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance',
@@ -259,7 +263,7 @@ class XMLDumper(object):
             except Errors.MMUnknownListError:
                 print >> sys.stderr, _('No such list: $listname')
                 continue
-            self._dump_list(mlist, with_passwords)
+            self._dump_list(mlist, password_scheme)
         self._pop_element('mailman')
 
     def close(self):
@@ -268,6 +272,41 @@ class XMLDumper(object):
 
 
 
+def no_password(password):
+    return '{NONE}'
+
+
+def plaintext_password(password):
+    return '{PLAIN}' + password
+
+
+def sha_password(password):
+    h = sha.new(password)
+    return '{SHA}' + base64.b64encode(h.digest())
+
+
+def ssha_password(password):
+    salt = os.urandom(SALT_LENGTH)
+    h = sha.new(password)
+    h.update(salt)
+    return '{SSHA}' + base64.b64encode(h.digest() + salt)
+
+
+SCHEMES = {
+    'none'  : no_password,
+    'plain' : plaintext_password,
+    'sha'   : sha_password,
+    }
+
+try:
+    os.urandom(1)
+except NotImplementedError:
+    pass
+else:
+    SCHEMES['ssha'] = ssha_password
+
+
+
 def parseargs():
     parser = optparse.OptionParser(version=Version.MAILMAN_VERSION,
                                    usage=_("""\
@@ -279,10 +318,15 @@ Export the configuration and members of a mailing list in XML format."""))
                       help=_("""\
 Output XML to FILENAME.  If not given, or if FILENAME is '-', standard out is
 used."""))
-    parser.add_option('-p', '--with-passwords',
+    parser.add_option('-p', '--password-scheme',
+                      default='none', type='string', help=_("""\
+Specify the RFC 2307 style hashing scheme for passwords included in the
+output.  Use -P to get a list of supported schemes, which are
+case-insensitive."""))
+    parser.add_option('-P', '--list-hash-schemes',
                       default=False, action='store_true', help=_("""\
-With this option, user passwords are included in cleartext.  For this reason,
-the default is to not include passwords."""))
+List the supported password hashing schemes and exit.  The scheme labels are
+case-insensitive."""))
     parser.add_option('-l', '--listname',
                       default=[], action='append', type='string',
                       metavar='LISTNAME', dest='listnames', help=_("""\
@@ -294,6 +338,12 @@ included in the XML output.  Multiple -l flags may be given."""))
     if args:
         parser.print_help()
         parser.error(_('Unexpected arguments'))
+    if opts.list_hash_schemes:
+        for label in SCHEMES:
+            print label.upper()
+        sys.exit(0)
+    if opts.password_scheme.lower() not in SCHEMES:
+        parser.error(_('Invalid password scheme'))
     return parser, opts, args
 
 
@@ -317,13 +367,8 @@ def main():
                 listnames.append(listname)
         else:
             listnames = Utils.list_names()
-        dumper.dump(listnames, opts.with_passwords)
+        dumper.dump(listnames, SCHEMES[opts.password_scheme])
         dumper.close()
     finally:
         if fp is not sys.stdout:
             fp.close()
-
-
-
-if __name__ == '__main__':
-    main()
diff --git a/Mailman/bin/import.py b/Mailman/bin/import.py
new file mode 100644
index 000000000..b4d62a8f2
--- /dev/null
+++ b/Mailman/bin/import.py
@@ -0,0 +1,283 @@
+# Copyright (C) 2006 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+
+"""Import the XML representation of a mailing list."""
+
+import sys
+import optparse
+import traceback
+
+from xml.dom import minidom
+from xml.parsers.expat import ExpatError
+
+from Mailman import Defaults
+from Mailman import MemberAdaptor
+from Mailman import Utils
+from Mailman import Version
+from Mailman.MailList import MailList
+from Mailman.i18n import _
+from Mailman.initialize import initialize
+
+
+__i18n_templates__ = True
+
+
+
+
+def nodetext(node):
+    # Expect only one TEXT_NODE in the list of children
+    for child in node.childNodes:
+        if child.nodeType == node.TEXT_NODE:
+            return child.data
+    return u''
+
+
+def nodegen(node, *entities):
+    for child in node.childNodes:
+        if child.nodeType <> minidom.Node.ELEMENT_NODE:
+            continue
+        if entities and child.tagName not in entities:
+            print _('Ignoring unexpected entity: $node.tagName')
+        else:
+            yield child
+
+
+
+def parse_config(node):
+    config = dict()
+    for child in nodegen(node, 'option'):
+        name  = child.getAttribute('name')
+        if not name:
+            print _('Skipping unnamed option')
+            continue
+        vtype = child.getAttribute('type') or 'string'
+        if vtype in ('email_list', 'email_list_ex', 'checkbox'):
+            value = []
+            for subnode in nodegen(child):
+                value.append(nodetext(subnode))
+        elif vtype == 'bool':
+            value = nodetext(child)
+            try:
+                value = bool(int(value))
+            except ValueError:
+                value = {'true' : True,
+                         'false': False,
+                         }.get(value.lower())
+                if value is None:
+                    print _('Skipping bad boolean value: $value')
+                    continue
+        elif vtype == 'radio':
+            value = nodetext(child).lower()
+            boolval = {'true' : True,
+                       'false': False,
+                       }.get(value)
+            if boolval is None:
+                value = int(value)
+            else:
+                value = boolval
+        elif vtype == 'number':
+            value = nodetext(child)
+            # First try int then float
+            try:
+                value = int(value)
+            except ValueError:
+                value = float(value)
+        elif vtype == 'header_filter':
+            # XXX
+            value = []
+        elif vtype == 'topics':
+            # XXX
+            value = []
+        else:
+            value = nodetext(child)
+        # And now some special casing :(
+        if name == 'new_member_options':
+            value = int(nodetext(child))
+        config[name] = value
+    return config
+
+
+
+
+def parse_roster(node):
+    members = []
+    for child in nodegen(node, 'member'):
+        member = dict()
+        member['id'] = mid = child.getAttribute('id')
+        if not mid:
+            print _('Skipping member with no id')
+            continue
+        if VERBOSE:
+            print _('* Processing member: $mid')
+        for subnode in nodegen(child):
+            attr = subnode.tagName
+            if attr == 'delivery':
+                value = (subnode.getAttribute('status'),
+                         subnode.getAttribute('delivery'))
+            elif attr in ('hide', 'ack', 'notmetoo', 'nodupes', 'nomail'):
+                value = {'true' : True,
+                         'false': False,
+                         }.get(nodetext(subnode).lower(), False)
+            elif attr == 'topics':
+                # XXX
+                value = []
+            else:
+                value = nodetext(subnode)
+            member[attr] = value
+        members.append(member)
+    return members
+
+
+
+def load(fp):
+    try:
+        doc = minidom.parse(fp)
+    except ExpatError:
+        print _('Expat error in file: %s', fp.name)
+        traceback.print_exc()
+        sys.exit(1)
+    doc.normalize()
+    # Make sure there's only one top-level <mailman> node
+    gen = nodegen(doc, 'mailman')
+    top = gen.next()
+    try:
+        gen.next()
+    except StopIteration:
+        pass
+    else:
+        print _('Malformed XML; duplicate <mailman> nodes')
+        sys.exit(1)
+    all_listdata = []
+    for listnode in nodegen(top, 'list'):
+        listdata = dict()
+        name = listnode.getAttribute('name')
+        if VERBOSE:
+            print _('Processing list: $name')
+        if not name:
+            print _('Ignoring malformed <list> node')
+            continue
+        for child in nodegen(listnode, 'configuration', 'roster'):
+            if child.tagName == 'configuration':
+                list_config = parse_config(child)
+            else:
+                assert(child.tagName == 'roster')
+                list_roster = parse_roster(child)
+        all_listdata.append((name, list_config, list_roster))
+    return all_listdata
+
+
+
+def create(all_listdata):
+    for name, list_config, list_roster in all_listdata:
+        fqdn_listname = '%s@%s' % (name, list_config['host_name'])
+        if Utils.list_exists(fqdn_listname):
+            print _('Skipping already existing list: $fqdn_listname')
+            continue
+        mlist = MailList()
+        mlist.Create(fqdn_listname, list_config['owner'][0],
+                     list_config['password'])
+        if VERBOSE:
+            print _('Creating mailing list: $fqdn_listname')
+        # Save the list creation, then unlock and relock the list.  This is so
+        # that we use normal SQLAlchemy transactions to manage all the
+        # attribute and membership updates.  Without this, no transaction will
+        # get committed in the second Save() below and we'll lose all our
+        # updates.
+        mlist.Save()
+        mlist.Unlock()
+        mlist.Lock()
+        try:
+            for option, value in list_config.items():
+                setattr(mlist, option, value)
+            for member in list_roster:
+                mid = member['id']
+                if VERBOSE:
+                    print _('* Adding member: $mid')
+                status, delivery = member['delivery']
+                kws = {'password'   : member['password'],
+                       'language'   : member['language'],
+                       'realname'   : member['realname'],
+                       'digest'     : delivery <> 'regular',
+                       }
+                mlist.addNewMember(mid, **kws)
+                status = {'enabled'     : MemberAdaptor.ENABLED,
+                          'byuser'      : MemberAdaptor.BYUSER,
+                          'byadmin'     : MemberAdaptor.BYADMIN,
+                          'bybounce'    : MemberAdaptor.BYBOUNCE,
+                          }.get(status, MemberAdaptor.UNKNOWN)
+                mlist.setDeliveryStatus(mid, status)
+                for opt in ('hide', 'ack', 'notmetoo', 'nodupes', 'nomail'):
+                    mlist.setMemberOption(mid,
+                                          Defaults.OPTINFO[opt],
+                                          member[opt])
+                # XXX topics
+            mlist.Save()
+        finally:
+            mlist.Unlock()
+
+
+
+def parseargs():
+    parser = optparse.OptionParser(version=Version.MAILMAN_VERSION,
+                                   usage=_("""\
+%prog [options]
+
+Import the configuration and/or members of a mailing list in XML format.  The
+imported mailing list must not already exist.  All mailing lists named in the
+XML file are imported, but those that already exist are skipped unless --error
+is given."""))
+    parser.add_option('-i', '--inputfile',
+                      metavar='FILENAME', default=None, type='string',
+                      help=_("""\
+Input XML from FILENAME.  If not given, or if FILENAME is '-', standard input
+is used."""))
+    parser.add_option('-p', '--reset-passwords',
+                      default=False, action='store_true', help=_("""\
+With this option, user passwords in the XML are ignored and are reset to a
+random password.  If the generated passwords were not included in the input
+XML, they will always be randomly generated."""))
+    parser.add_option('-v', '--verbose',
+                      default=False, action='store_true',
+                      help=_('Produce more verbose output'))
+    parser.add_option('-C', '--config',
+                      help=_('Alternative configuration file to use'))
+    opts, args = parser.parse_args()
+    if args:
+        parser.print_help()
+        parser.error(_('Unexpected arguments'))
+    return parser, opts, args
+
+
+
+def main():
+    global VERBOSE
+
+    parser, opts, args = parseargs()
+    initialize(opts.config)
+    VERBOSE = opts.verbose
+    
+    if opts.inputfile in (None, '-'):
+        fp = sys.stdin
+    else:
+        fp = open(opts.inputfile)
+        
+    try:
+        listbags = load(fp)
+        create(listbags)
+    finally:
+        if fp is not sys.stdin:
+            fp.close()
diff --git a/Mailman/database/dbcontext.py b/Mailman/database/dbcontext.py
index b812c9b77..a13f20498 100644
--- a/Mailman/database/dbcontext.py
+++ b/Mailman/database/dbcontext.py
@@ -98,19 +98,33 @@ class DBContext(object):
             return
         txn = self.session.create_transaction()
         mref = MlistRef(mlist, self._unlock_mref)
-        self._mlist_txns[mlist.fqdn_listname] = txn
+        # If mlist.host_name is changed, its fqdn_listname attribute will no
+        # longer match, so its transaction will not get committed when the
+        # list is saved.  To avoid this, store on the mlist object the key
+        # under which its transaction is stored.
+        txnkey = mlist._txnkey = mlist.fqdn_listname
+        self._mlist_txns[txnkey] = txn
 
     def api_unlock(self, mlist):
-        txn = self._mlist_txns.pop(mlist.fqdn_listname, None)
+        try:
+            txnkey = mlist._txnkey
+        except AttributeError:
+            return
+        txn = self._mlist_txns.pop(txnkey, None)
         if txn is not None:
             txn.rollback()
+        del mlist._txnkey
 
     def api_save(self, mlist):
         # When dealing with MailLists, .Save() will always be followed by
         # .Unlock().  However lists can also be unlocked without saving.  But
         # if it's been locked it will always be unlocked.  So the rollback in
         # unlock will essentially be no-op'd if we've already saved the list.
-        txn = self._mlist_txns.pop(mlist.fqdn_listname, None)
+        try:
+            txnkey = mlist._txnkey
+        except AttributeError:
+            return
+        txn = self._mlist_txns.pop(txnkey, None)
         if txn is not None:
             txn.commit()
 
diff --git a/misc/Makefile.in b/misc/Makefile.in
index 413251641..4770225bd 100644
--- a/misc/Makefile.in
+++ b/misc/Makefile.in
@@ -58,7 +58,7 @@ SETUPCMD=	setup.py --quiet install $(SETUPINSTOPTS)
 EMAIL=		email-4.0.1
 SETUPTOOLS=	setuptools-0.6c3
 PYSQLITE=	pysqlite-2.3.2
-SQLALCHEMY=	SQLAlchemy-0.3.1
+SQLALCHEMY=	SQLAlchemy-0.3.3
 SETUPPKGS=	$(EMAIL) $(SETUPTOOLS) $(PYSQLITE) $(SQLALCHEMY)
 
 EZINSTOPTS=	--install-dir $(DESTDIR)$(PYTHONLIBDIR)
diff --git a/misc/SQLAlchemy-0.3.1.tar.gz b/misc/SQLAlchemy-0.3.1.tar.gz
deleted file mode 100644
index a2afa5589..000000000
--- a/misc/SQLAlchemy-0.3.1.tar.gz
+++ /dev/null
diff --git a/misc/SQLAlchemy-0.3.3.tar.gz b/misc/SQLAlchemy-0.3.3.tar.gz
new file mode 100644
index 000000000..5732923fd
--- /dev/null
+++ b/misc/SQLAlchemy-0.3.3.tar.gz