From 08a3192f2a8eaa9a094b84f06bd31a1ed28a4bac Mon Sep 17 00:00:00 2001
From: Aurélien Bompard
Date: Fri, 20 Sep 2013 18:27:49 +0200
Subject: When importing from 2.1, handle the archiving policy

---
 src/mailman/utilities/importer.py          |  9 +++++++++
 src/mailman/utilities/tests/test_import.py | 32 ++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)

(limited to 'src')

diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py
index f5aa8d10a..3cc7259d7 100644
--- a/src/mailman/utilities/importer.py
+++ b/src/mailman/utilities/importer.py
@@ -33,6 +33,7 @@ from mailman.interfaces.autorespond import ResponseAction
 from mailman.interfaces.digests import DigestFrequency
 from mailman.interfaces.mailinglist import Personalization, ReplyToMunging
 from mailman.interfaces.nntp import NewsgroupModeration
+from mailman.interfaces.archiver import ArchivePolicy
 
 
 
@@ -90,3 +91,11 @@ def import_config_pck(mlist, config_dict):
             except TypeError:
                 print('Type conversion error:', key, file=sys.stderr)
                 raise
+    # Handle the archiving policy
+    if config_dict.get("archive"):
+        if config_dict.get("archive_private"):
+            mlist.archive_policy = ArchivePolicy.private
+        else:
+            mlist.archive_policy = ArchivePolicy.public
+    else:
+        mlist.archive_policy = ArchivePolicy.never
diff --git a/src/mailman/utilities/tests/test_import.py b/src/mailman/utilities/tests/test_import.py
index c8da32e42..45df80d80 100644
--- a/src/mailman/utilities/tests/test_import.py
+++ b/src/mailman/utilities/tests/test_import.py
@@ -31,6 +31,7 @@ import unittest
 from mailman.app.lifecycle import create_list, remove_list
 from mailman.testing.layers import ConfigLayer
 from mailman.utilities.importer import import_config_pck
+from mailman.interfaces.archiver import ArchivePolicy
 from pkg_resources import resource_filename
 
 
@@ -68,3 +69,34 @@ class TestBasicImport(unittest.TestCase):
         self._import()
         self.assertTrue(self._mlist.allow_list_posts)
         self.assertTrue(self._mlist.include_rfc2369_headers)
+
+
+
+class TestArchiveImport(unittest.TestCase):
+    # The mlist.archive_policy gets set from the old list's archive and
+    # archive_private values
+
+    layer = ConfigLayer
+
+    def setUp(self):
+        self._mlist = create_list('blank@example.com')
+        self._mlist.archive_policy = "INITIAL-TEST-VALUE"
+
+    def tearDown(self):
+        remove_list(self._mlist)
+
+    def _do_test(self, pckdict, expected):
+        import_config_pck(self._mlist, pckdict)
+        self.assertEqual(self._mlist.archive_policy, expected)
+
+    def test_public(self):
+        self._do_test({ "archive": True, "archive_private": False },
+                      ArchivePolicy.public)
+
+    def test_private(self):
+        self._do_test({ "archive": True, "archive_private": True },
+                      ArchivePolicy.private)
+
+    def test_no_archive(self):
+        self._do_test({ "archive": False, "archive_private": False },
+                      ArchivePolicy.never)
-- 
cgit v1.3.1


From 8a81dca42b73cc53c525a121433998d8024f2e64 Mon Sep 17 00:00:00 2001
From: Aurélien Bompard
Date: Mon, 30 Sep 2013 11:11:25 +0200
Subject: Import most list parameters from the 2.1 pickle

---
 src/mailman/handlers/decorate.py           |   9 +-
 src/mailman/utilities/importer.py          | 252 ++++++++++++-
 src/mailman/utilities/tests/test_import.py | 568 ++++++++++++++++++++++++++++-
 3 files changed, 824 insertions(+), 5 deletions(-)

(limited to 'src')

diff --git a/src/mailman/handlers/decorate.py b/src/mailman/handlers/decorate.py
index 945c50bd5..815a24569 100644
--- a/src/mailman/handlers/decorate.py
+++ b/src/mailman/handlers/decorate.py
@@ -201,8 +201,8 @@ def process(mlist, msg, msgdata):
 
 
 def decorate(mlist, uri, extradict=None):
-    """Expand the decoration template."""
-    if uri is None:
+    """Expand the decoration template from its URI."""
+    if uri is None or uri == '':
         return ''
     # Get the decorator template.
     loader = getUtility(ITemplateLoader)
@@ -211,6 +211,11 @@ def decorate(mlist, uri, extradict=None):
         language=mlist.preferred_language.code,
         ))
     template = loader.get(template_uri)
+    return decorate_template(mlist, template, extradict)
+
+
+def decorate_template(mlist, template, extradict=None):
+    """Expand the decoration template."""
     # Create a dictionary which includes the default set of interpolation
     # variables allowed in headers and footers.  These will be augmented by
     # any key/value pairs in the extradict.
diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py
index 3cc7259d7..50d2e7640 100644
--- a/src/mailman/utilities/importer.py
+++ b/src/mailman/utilities/importer.py
@@ -27,32 +27,103 @@ __all__ = [
 
 import sys
 import datetime
+import os
+from urllib2 import URLError
 
-from mailman.interfaces.action import FilterAction
+from mailman.config import config
+from mailman.interfaces.action import FilterAction, Action
 from mailman.interfaces.autorespond import ResponseAction
 from mailman.interfaces.digests import DigestFrequency
 from mailman.interfaces.mailinglist import Personalization, ReplyToMunging
 from mailman.interfaces.nntp import NewsgroupModeration
 from mailman.interfaces.archiver import ArchivePolicy
+from mailman.interfaces.bans import IBanManager
+from mailman.interfaces.mailinglist import IAcceptableAliasSet
+from mailman.interfaces.bounce import UnrecognizedBounceDisposition
+from mailman.interfaces.usermanager import IUserManager
+from mailman.interfaces.member import DeliveryMode, DeliveryStatus, MemberRole
+from mailman.handlers.decorate import decorate, decorate_template
+from mailman.utilities.i18n import search
+from zope.component import getUtility
 
 
 
 def seconds_to_delta(value):
     return datetime.timedelta(seconds=value)
 
+
+def days_to_delta(value):
+    return datetime.timedelta(days=value)
+
+
+def list_members_to_unicode(value):
+    return [ unicode(item) for item in value ]
+
+
+def filter_action_mapping(value):
+    # The filter_action enum values have changed. In Mailman 2.1 the order was
+    # 'Discard', 'Reject', 'Forward to List Owner', 'Preserve'.
+    # In 3.0 it's 'hold', 'reject', 'discard', 'accept', 'defer', 'forward',
+    # 'preserve'
+    if value == 0:
+        return FilterAction.discard
+    elif value == 1:
+        return FilterAction.reject
+    elif value == 2:
+        return FilterAction.forward
+    elif value == 3:
+        return FilterAction.preserve
+    else:
+        raise ValueError("Unknown filter_action value: %s" % value)
+
+
+def member_action_mapping(value):
+    # The mlist.default_member_action and mlist.default_nonmember_action enum
+    # values are different in Mailman 2.1, because they have been merged into a
+    # single enum in Mailman 3
+    # For default_member_action, which used to be called
+    # member_moderation_action, the values were:
+    # 0==Hold, 1=Reject, 2==Discard
+    if value == 0:
+        return Action.hold
+    elif value == 1:
+        return Action.reject
+    elif value == 2:
+        return Action.discard
+def nonmember_action_mapping(value):
+    # For default_nonmember_action, which used to be called
+    # generic_nonmember_action, the values were:
+    # 0==Accept, 1==Hold, 2==Reject, 3==Discard
+    if value == 0:
+        return Action.accept
+    elif value == 1:
+        return Action.hold
+    elif value == 2:
+        return Action.reject
+    elif value == 3:
+        return Action.discard
+
 
 # Attributes in Mailman 2 which have a different type in Mailman 3.
 TYPES = dict(
     autorespond_owner=ResponseAction,
     autorespond_postings=ResponseAction,
     autorespond_requests=ResponseAction,
+    autoresponse_grace_period=days_to_delta,
     bounce_info_stale_after=seconds_to_delta,
     bounce_you_are_disabled_warnings_interval=seconds_to_delta,
     digest_volume_frequency=DigestFrequency,
-    filter_action=FilterAction,
+    filter_action=filter_action_mapping,
     newsgroup_moderation=NewsgroupModeration,
     personalize=Personalization,
     reply_goes_to_list=ReplyToMunging,
+    filter_types=list_members_to_unicode,
+    pass_types=list_members_to_unicode,
+    filter_extensions=list_members_to_unicode,
+    pass_extensions=list_members_to_unicode,
+    forward_unrecognized_bounces_to=UnrecognizedBounceDisposition,
+    default_member_action=member_action_mapping,
+    default_nonmember_action=nonmember_action_mapping,
     )
 
 
@@ -61,6 +132,28 @@ NAME_MAPPINGS = dict(
     host_name='mail_host',
     include_list_post_header='allow_list_posts',
     real_name='display_name',
+    last_post_time='last_post_at',
+    autoresponse_graceperiod='autoresponse_grace_period',
+    autorespond_admin='autorespond_owner',
+    autoresponse_admin_text='autoresponse_owner_text',
+    filter_mime_types='filter_types',
+    pass_mime_types='pass_types',
+    filter_filename_extensions='filter_extensions',
+    pass_filename_extensions='pass_extensions',
+    bounce_processing='process_bounces',
+    bounce_unrecognized_goes_to_list_owner='forward_unrecognized_bounces_to',
+    mod_password='moderator_password',
+    news_moderation='newsgroup_moderation',
+    news_prefix_subject_too='nntp_prefix_subject_too',
+    send_welcome_msg='send_welcome_message',
+    send_goodbye_msg='send_goodbye_message',
+    member_moderation_action='default_member_action',
+    generic_nonmember_action='default_nonmember_action',
+    )
+
+EXCLUDES = (
+    "members",
+    "digest_members",
     )
 
 
@@ -74,6 +167,9 @@ def import_config_pck(mlist, config_dict):
     :type config_dict: dict
     """
     for key, value in config_dict.items():
+        # Some attributes must not be directly imported
+        if key in EXCLUDES:
+            continue
         # Some attributes from Mailman 2 were renamed in Mailman 3.
         key = NAME_MAPPINGS.get(key, key)
         # Handle the simple case where the key is an attribute of the
@@ -99,3 +195,155 @@ def import_config_pck(mlist, config_dict):
             mlist.archive_policy = ArchivePolicy.public
     else:
         mlist.archive_policy = ArchivePolicy.never
+    # Handle ban list
+    for addr in config_dict.get('ban_list', []):
+        IBanManager(mlist).ban(addr)
+    # Handle acceptable aliases
+    for addr in config_dict.get('acceptable_aliases', '').splitlines():
+        addr = addr.strip()
+        if not addr:
+            continue
+        IAcceptableAliasSet(mlist).add(addr)
+    # Handle conversion to URIs
+    convert_to_uri = {
+        "welcome_msg": "welcome_message_uri",
+        "goodbye_msg": "goodbye_message_uri",
+        "msg_header": "header_uri",
+        "msg_footer": "footer_uri",
+        "digest_header": "digest_header_uri",
+        "digest_footer": "digest_footer_uri",
+        }
+    convert_placeholders = { # only the most common ones
+        "%(real_name)s": "$display_name",
+        "%(real_name)s@%(host_name)s": "$fqdn_listname",
+        "%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s": "$listinfo_uri",
+    }
+    for oldvar, newvar in convert_to_uri.iteritems():
+        if oldvar not in config_dict:
+            continue
+        text = config_dict[oldvar]
+        for oldph, newph in convert_placeholders.iteritems():
+            text = text.replace(oldph, newph)
+        default_value = getattr(mlist, newvar)
+        if not text and not default_value:
+            continue
+        # Check if the value changed from the default
+        try:
+            default_text = decorate(mlist, default_value)
+            expanded_text = decorate_template(mlist, text)
+        except (URLError, KeyError):
+            # Use case: importing the old a@ex.com into b@ex.com
+            # We can't check if it changed from the default
+            # -> don't import, we may do more harm than good and it's easy to
+            # change if needed
+            continue
+        if not text and not default_text:
+            continue # both are empty, leave it
+        if expanded_text.strip() == default_text.strip():
+            continue # keep the default
+        # Write the custom value to the right file
+        base_uri = "mailman:///$listname/$language/"
+        if default_value:
+            filename = default_value.rpartition("/")[2]
+        else:
+            filename = "%s.txt" % newvar[:-4]
+        if not default_value or not default_value.startswith(base_uri):
+            setattr(mlist, newvar, base_uri + filename)
+        filepath = list(search(filename, mlist))[0]
+        try:
+            os.makedirs(os.path.dirname(filepath))
+        except OSError, e:
+            if e.errno != 17: # Already exists
+                raise
+        with open(filepath, "w") as template:
+            template.write(text.encode('utf-8'))
+    # Import rosters
+    members = set(config_dict.get("members", {}).keys()
+                + config_dict.get("digest_members", {}).keys())
+    import_roster(mlist, config_dict, members, MemberRole.member)
+    import_roster(mlist, config_dict, config_dict.get("owner", []),
+                  MemberRole.owner)
+    import_roster(mlist, config_dict, config_dict.get("moderator", []),
+                  MemberRole.moderator)
+
+
+
+def import_roster(mlist, config_dict, members, role):
+    """
+    Import members lists from a config.pck configuration dictionary to a
+    mailing list.
+
+    :param mlist: The mailing list.
+    :type  mlist: IMailingList
+    :param config_dict: The Mailman 2.1 configuration dictionary.
+    :type  config_dict: dict
+    :param members: The members list to import
+    :type  members: list
+    :param role: The MemberRole to import them as
+    :type  role: MemberRole enum
+    """
+    usermanager = getUtility(IUserManager)
+    for email in members:
+        email = unicode(email)
+        roster = mlist.get_roster(role)
+        if roster.get_member(email) is not None:
+            print("%s is already imported with role %s" % (email, role),
+                  file=sys.stderr)
+            continue
+        user = usermanager.get_user(email)
+        if user is None:
+            merged_members = {}
+            merged_members.update(config_dict.get("members", {}))
+            merged_members.update(config_dict.get("digest_members", {}))
+            if merged_members.get(email, 0) != 0:
+                original_email = merged_members[email]
+            else:
+                original_email = email
+            user = usermanager.create_user(original_email)
+        address = usermanager.get_address(email)
+        address.verified_on = datetime.datetime.now()
+        mlist.subscribe(address, role)
+        member = roster.get_member(email)
+        assert member is not None
+        prefs = config_dict.get("user_options", {}).get(email, 0)
+        if email in config_dict.get("members", {}):
+            member.preferences.delivery_mode = DeliveryMode.regular
+        elif email in config_dict.get("digest_members", {}):
+            if prefs & 8: # DisableMime
+                member.preferences.delivery_mode = DeliveryMode.plaintext_digests
+            else:
+                member.preferences.delivery_mode = DeliveryMode.mime_digests
+        else:
+            # probably not adding a member role here
+            pass
+        if email in config_dict.get("language", {}):
+            member.preferences.preferred_language = \
+                unicode(config_dict["language"][email])
+        # if the user already exists, display_name and password will be
+        # overwritten
+        if email in config_dict.get("usernames", {}):
+            address.display_name = unicode(config_dict["usernames"][email])
+            user.display_name    = unicode(config_dict["usernames"][email])
+        if email in config_dict.get("passwords", {}):
+            user.password = config.password_context.encrypt(
+                                    config_dict["passwords"][email])
+        # delivery_status
+        oldds = config_dict.get("delivery_status", {}).get(email, (0, 0))[0]
+        if oldds == 0:
+            member.preferences.delivery_status = DeliveryStatus.enabled
+        elif oldds == 1:
+            member.preferences.delivery_status = DeliveryStatus.unknown
+        elif oldds == 2:
+            member.preferences.delivery_status = DeliveryStatus.by_user
+        elif oldds == 3:
+            member.preferences.delivery_status = DeliveryStatus.by_moderator
+        elif oldds == 4:
+            member.preferences.delivery_status = DeliveryStatus.by_bounces
+        # moderation
+        if prefs & 128:
+            member.moderation_action = Action.hold
+        # other preferences
+        member.preferences.acknowledge_posts = bool(prefs & 4) # AcknowledgePosts
+        member.preferences.hide_address = bool(prefs & 16) # ConcealSubscription
+        member.preferences.receive_own_postings = not bool(prefs & 2) # DontReceiveOwnPosts
+        member.preferences.receive_list_copy = not bool(prefs & 256) # DontReceiveDuplicates
diff --git a/src/mailman/utilities/tests/test_import.py b/src/mailman/utilities/tests/test_import.py
index 45df80d80..b8eb3e280 100644
--- a/src/mailman/utilities/tests/test_import.py
+++ b/src/mailman/utilities/tests/test_import.py
@@ -27,14 +27,34 @@ __all__ = [
 
 import cPickle
 import unittest
+from datetime import timedelta, datetime
 
 from mailman.app.lifecycle import create_list, remove_list
 from mailman.testing.layers import ConfigLayer
 from mailman.utilities.importer import import_config_pck
 from mailman.interfaces.archiver import ArchivePolicy
+from mailman.interfaces.action import Action, FilterAction
+from mailman.interfaces.bounce import UnrecognizedBounceDisposition
+from mailman.interfaces.bans import IBanManager
+from mailman.interfaces.mailinglist import IAcceptableAliasSet
+from mailman.interfaces.nntp import NewsgroupModeration
+from mailman.interfaces.autorespond import ResponseAction
+from mailman.interfaces.templates import ITemplateLoader
+from mailman.interfaces.usermanager import IUserManager
+from mailman.interfaces.member import DeliveryMode, DeliveryStatus
+from mailman.interfaces.languages import ILanguageManager
+from mailman.handlers.decorate import decorate
+from mailman.utilities.string import expand
 from pkg_resources import resource_filename
+from enum import Enum
+from zope.component import getUtility
 
 
+
+class DummyEnum(Enum):
+    # For testing purposes
+    val = 42
+
 
 class TestBasicImport(unittest.TestCase):
     layer = ConfigLayer
@@ -70,6 +90,121 @@ class TestBasicImport(unittest.TestCase):
         self.assertTrue(self._mlist.allow_list_posts)
         self.assertTrue(self._mlist.include_rfc2369_headers)
 
+    def test_no_overwrite_rosters(self):
+        # The mlist.members and mlist.digest_members rosters must not be
+        # overwritten.
+        for rname in ("members", "digest_members"):
+            roster = getattr(self._mlist, rname)
+            self.assertFalse(isinstance(roster, dict))
+            self._import()
+            self.assertFalse(isinstance(roster, dict),
+                "The %s roster has been overwritten by the import" % rname)
+
+    def test_last_post_time(self):
+        # last_post_time -> last_post_at
+        self._pckdict["last_post_time"] = 1270420800.274485
+        self.assertEqual(self._mlist.last_post_at, None)
+        self._import()
+        # convert 1270420800.2744851 to datetime
+        expected = datetime(2010, 4, 4, 22, 40, 0, 274485)
+        self.assertEqual(self._mlist.last_post_at, expected)
+
+    def test_autoresponse_grace_period(self):
+        # autoresponse_graceperiod -> autoresponse_grace_period
+        # must be a timedelta, not an int
+        self._mlist.autoresponse_grace_period = timedelta(days=42)
+        self._import()
+        self.assertTrue(isinstance(
+                self._mlist.autoresponse_grace_period, timedelta))
+        self.assertEqual(self._mlist.autoresponse_grace_period,
+                         timedelta(days=90))
+
+    def test_autoresponse_admin_to_owner(self):
+        # admin -> owner
+        self._mlist.autorespond_owner = DummyEnum.val
+        self._mlist.autoresponse_owner_text = 'DUMMY'
+        self._import()
+        self.assertEqual(self._mlist.autorespond_owner, ResponseAction.none)
+        self.assertEqual(self._mlist.autoresponse_owner_text, '')
+
+    #def test_administrative(self):
+    #    # administrivia -> administrative
+    #    self._mlist.administrative = None
+    #    self._import()
+    #    self.assertTrue(self._mlist.administrative)
+
+    def test_filter_pass_renames(self):
+        # mime_types -> types
+        # filename_extensions -> extensions
+        self._mlist.filter_types = ["dummy"]
+        self._mlist.pass_types = ["dummy"]
+        self._mlist.filter_extensions = ["dummy"]
+        self._mlist.pass_extensions = ["dummy"]
+        self._import()
+        self.assertEqual(list(self._mlist.filter_types), [])
+        self.assertEqual(list(self._mlist.filter_extensions),
+                         ['exe', 'bat', 'cmd', 'com', 'pif',
+                          'scr', 'vbs', 'cpl'])
+        self.assertEqual(list(self._mlist.pass_types),
+                ['multipart/mixed', 'multipart/alternative', 'text/plain'])
+        self.assertEqual(list(self._mlist.pass_extensions), [])
+
+    def test_process_bounces(self):
+        # bounce_processing -> process_bounces
+        self._mlist.process_bounces = None
+        self._import()
+        self.assertTrue(self._mlist.process_bounces)
+
+    def test_forward_unrecognized_bounces_to(self):
+        # bounce_unrecognized_goes_to_list_owner -> forward_unrecognized_bounces_to
+        self._mlist.forward_unrecognized_bounces_to = DummyEnum.val
+        self._import()
+        self.assertEqual(self._mlist.forward_unrecognized_bounces_to,
+                         UnrecognizedBounceDisposition.administrators)
+
+    def test_moderator_password(self):
+        # mod_password -> moderator_password
+        self._mlist.moderator_password = str("TESTDATA")
+        self._import()
+        self.assertEqual(self._mlist.moderator_password, None)
+
+    def test_newsgroup_moderation(self):
+        # news_moderation -> newsgroup_moderation
+        # news_prefix_subject_too -> nntp_prefix_subject_too
+        self._mlist.newsgroup_moderation = DummyEnum.val
+        self._mlist.nntp_prefix_subject_too = None
+        self._import()
+        self.assertEqual(self._mlist.newsgroup_moderation,
+                         NewsgroupModeration.none)
+        self.assertTrue(self._mlist.nntp_prefix_subject_too)
+
+    def test_msg_to_message(self):
+        # send_welcome_msg -> send_welcome_message
+        # send_goodbye_msg -> send_goodbye_message
+        self._mlist.send_welcome_message = None
+        self._mlist.send_goodbye_message = None
+        self._import()
+        self.assertTrue(self._mlist.send_welcome_message)
+        self.assertTrue(self._mlist.send_goodbye_message)
+
+    def test_ban_list(self):
+        banned = [
+            ("anne@example.com", "anne@example.com"),
+            ("^.*@example.com", "bob@example.com")
+            ]
+        self._pckdict["ban_list"] = [ b[0] for b in banned ]
+        self._import()
+        for _pattern, addr in banned:
+            self.assertTrue(IBanManager(self._mlist).is_banned(addr))
+
+    def test_acceptable_aliases(self):
+        # it used to be a plain-text field (values are newline-separated)
+        aliases = ["alias1@example.com", "alias2@exemple.com"]
+        self._pckdict["acceptable_aliases"] = "\n".join(aliases)
+        self._import()
+        alias_set = IAcceptableAliasSet(self._mlist)
+        self.assertEqual(sorted(alias_set.aliases), aliases)
+
 
 
 class TestArchiveImport(unittest.TestCase):
@@ -80,7 +215,7 @@ class TestArchiveImport(unittest.TestCase):
 
     def setUp(self):
         self._mlist = create_list('blank@example.com')
-        self._mlist.archive_policy = "INITIAL-TEST-VALUE"
+        self._mlist.archive_policy = DummyEnum.val
 
     def tearDown(self):
         remove_list(self._mlist)
@@ -100,3 +235,434 @@ class TestArchiveImport(unittest.TestCase):
     def test_no_archive(self):
         self._do_test({ "archive": False, "archive_private": False },
                       ArchivePolicy.never)
+
+
+
+class TestFilterActionImport(unittest.TestCase):
+    # The mlist.filter_action enum values have changed. In Mailman 2.1 the
+    # order was 'Discard', 'Reject', 'Forward to List Owner', 'Preserve'.
+
+    layer = ConfigLayer
+
+    def setUp(self):
+        self._mlist = create_list('blank@example.com')
+        self._mlist.filter_action = DummyEnum.val
+
+    def tearDown(self):
+        remove_list(self._mlist)
+
+    def _do_test(self, original, expected):
+        import_config_pck(self._mlist, { "filter_action": original })
+        self.assertEqual(self._mlist.filter_action, expected)
+
+    def test_discard(self):
+        self._do_test(0, FilterAction.discard)
+
+    def test_reject(self):
+        self._do_test(1, FilterAction.reject)
+
+    def test_forward(self):
+        self._do_test(2, FilterAction.forward)
+
+    def test_preserve(self):
+        self._do_test(3, FilterAction.preserve)
+
+
+
+class TestMemberActionImport(unittest.TestCase):
+    # The mlist.default_member_action and mlist.default_nonmember_action enum
+    # values are different in Mailman 2.1, they have been merged into a
+    # single enum in Mailman 3
+    # For default_member_action, which used to be called
+    # member_moderation_action, the values were:
+    # 0==Hold, 1=Reject, 2==Discard
+    # For default_nonmember_action, which used to be called
+    # generic_nonmember_action, the values were:
+    # 0==Accept, 1==Hold, 2==Reject, 3==Discard
+
+    layer = ConfigLayer
+
+    def setUp(self):
+        self._mlist = create_list('blank@example.com')
+        self._mlist.default_member_action = DummyEnum.val
+        self._mlist.default_nonmember_action = DummyEnum.val
+        self._pckdict = {
+            "member_moderation_action": DummyEnum.val,
+            "generic_nonmember_action": DummyEnum.val,
+        }
+
+    def tearDown(self):
+        remove_list(self._mlist)
+
+    def _do_test(self, expected):
+        import_config_pck(self._mlist, self._pckdict)
+        for key, value in expected.iteritems():
+            self.assertEqual(getattr(self._mlist, key), value)
+
+    def test_member_hold(self):
+        self._pckdict["member_moderation_action"] = 0
+        self._do_test({"default_member_action": Action.hold})
+
+    def test_member_reject(self):
+        self._pckdict["member_moderation_action"] = 1
+        self._do_test({"default_member_action": Action.reject})
+
+    def test_member_discard(self):
+        self._pckdict["member_moderation_action"] = 2
+        self._do_test({"default_member_action": Action.discard})
+
+    def test_nonmember_accept(self):
+        self._pckdict["generic_nonmember_action"] = 0
+        self._do_test({"default_nonmember_action": Action.accept})
+
+    def test_nonmember_hold(self):
+        self._pckdict["generic_nonmember_action"] = 1
+        self._do_test({"default_nonmember_action": Action.hold})
+
+    def test_nonmember_reject(self):
+        self._pckdict["generic_nonmember_action"] = 2
+        self._do_test({"default_nonmember_action": Action.reject})
+
+    def test_nonmember_discard(self):
+        self._pckdict["generic_nonmember_action"] = 3
+        self._do_test({"default_nonmember_action": Action.discard})
+
+
+
+class TestConvertToURI(unittest.TestCase):
+    # The following values were plain text, and are now URIs in Mailman 3:
+    # - welcome_message_uri
+    # - goodbye_message_uri
+    # - header_uri
+    # - footer_uri
+    # - digest_header_uri
+    # - digest_footer_uri
+    #
+    # The templates contain variables that must be replaced:
+    # - %(real_name)s -> %(display_name)s
+    # - %(real_name)s@%(host_name)s -> %(fqdn_listname)s
+    # - %(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s -> %(listinfo_uri)s
+
+    layer = ConfigLayer
+
+    def setUp(self):
+        self._mlist = create_list('blank@example.com')
+        self._conf_mapping = {
+            "welcome_msg": "welcome_message_uri",
+            "goodbye_msg": "goodbye_message_uri",
+            "msg_header": "header_uri",
+            "msg_footer": "footer_uri",
+            "digest_header": "digest_header_uri",
+            "digest_footer": "digest_footer_uri",
+        }
+        self._pckdict = {}
+        #self._pckdict = {
+        #    "preferred_language": "XX", # templates are lang-specific
+        #}
+
+    def tearDown(self):
+        remove_list(self._mlist)
+
+    def test_text_to_uri(self):
+        for oldvar, newvar in self._conf_mapping.iteritems():
+            self._pckdict[oldvar] = "TEST VALUE"
+            import_config_pck(self._mlist, self._pckdict)
+            newattr = getattr(self._mlist, newvar)
+            text = decorate(self._mlist, newattr)
+            self.assertEqual(text, "TEST VALUE",
+                    "Old variable %s was not properly imported to %s"
+                    % (oldvar, newvar))
+
+    def test_substitutions(self):
+        test_text = ("UNIT TESTING %(real_name)s mailing list\n"
+                     "%(real_name)s@%(host_name)s\n"
+                     "%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s")
+        expected_text = ("UNIT TESTING $display_name mailing list\n"
+                         "$fqdn_listname\n"
+                         "$listinfo_uri")
+        for oldvar, newvar in self._conf_mapping.iteritems():
+            self._pckdict[oldvar] = test_text
+            import_config_pck(self._mlist, self._pckdict)
+            newattr = getattr(self._mlist, newvar)
+            template_uri = expand(newattr, dict(
+                listname=self._mlist.fqdn_listname,
+                language=self._mlist.preferred_language.code,
+                ))
+            loader = getUtility(ITemplateLoader)
+            text = loader.get(template_uri)
+            self.assertEqual(text, expected_text,
+                    "Old variables were not converted for %s" % newvar)
+
+    def test_keep_default(self):
+        # If the value was not changed from MM2.1's default, don't import it
+        default_msg_footer = (
+            "_______________________________________________\n"
+            "%(real_name)s mailing list\n"
+            "%(real_name)s@%(host_name)s\n"
+            "%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s"
+            )
+        for oldvar in ("msg_footer", "digest_footer"):
+            newvar = self._conf_mapping[oldvar]
+            self._pckdict[oldvar] = default_msg_footer
+            old_value = getattr(self._mlist, newvar)
+            import_config_pck(self._mlist, self._pckdict)
+            new_value = getattr(self._mlist, newvar)
+            self.assertEqual(old_value, new_value,
+                    "Default value was not preserved for %s" % newvar)
+
+    def test_keep_default_if_fqdn_changed(self):
+        # Use case: importing the old a@ex.com into b@ex.com
+        # We can't check if it changed from the default
+        # -> don't import, we may do more harm than good and it's easy to
+        # change if needed
+        test_value = "TEST-VALUE"
+        for oldvar, newvar in self._conf_mapping.iteritems():
+            self._mlist.mail_host = "example.com"
+            self._pckdict["mail_host"] = "test.example.com"
+            self._pckdict[oldvar] = test_value
+            old_value = getattr(self._mlist, newvar)
+            import_config_pck(self._mlist, self._pckdict)
+            new_value = getattr(self._mlist, newvar)
+            self.assertEqual(old_value, new_value,
+                    "Default value was not preserved for %s" % newvar)
+
+
+
+class TestRosterImport(unittest.TestCase):
+
+    layer = ConfigLayer
+
+    def setUp(self):
+        self._mlist = create_list('blank@example.com')
+        self._pckdict = {
+            "members": {
+                "anne@example.com": 0,
+                "bob@example.com": "bob@ExampLe.Com",
+            },
+            "digest_members": {
+                "cindy@example.com": 0,
+                "dave@example.com": "dave@ExampLe.Com",
+            },
+            "passwords": {
+                "anne@example.com" : "annepass",
+                "bob@example.com"  : "bobpass",
+                "cindy@example.com": "cindypass",
+                "dave@example.com" : "davepass",
+            },
+            "language": {
+                "anne@example.com" : "fr",
+                "bob@example.com"  : "de",
+                "cindy@example.com": "es",
+                "dave@example.com" : "it",
+            },
+            "usernames": {
+                "anne@example.com" : "Anne",
+                "bob@example.com"  : "Bob",
+                "cindy@example.com": "Cindy",
+                "dave@example.com" : "Dave",
+            },
+            "owner": [
+                "anne@example.com",
+                "emily@example.com",
+            ],
+            "moderator": [
+                "bob@example.com",
+                "fred@example.com",
+            ],
+        }
+        self._usermanager = getUtility(IUserManager)
+        language_manager = getUtility(ILanguageManager)
+        for code in self._pckdict["language"].values():
+            if code not in language_manager.codes:
+                language_manager.add(code, 'utf-8', code)
+
+    def tearDown(self):
+        remove_list(self._mlist)
+
+    def test_member(self):
+        import_config_pck(self._mlist, self._pckdict)
+        for name in ("anne", "bob", "cindy", "dave"):
+            addr = "%s@example.com" % name
+            self.assertTrue(
+                    addr in [ a.email for a in self._mlist.members.addresses],
+                    "Address %s was not imported" % addr)
+        self.assertTrue("anne@example.com" in [ a.email
+                        for a in self._mlist.regular_members.addresses])
+        self.assertTrue("bob@example.com" in [ a.email
+                        for a in self._mlist.regular_members.addresses])
+        self.assertTrue("cindy@example.com" in [ a.email
+                        for a in self._mlist.digest_members.addresses])
+        self.assertTrue("dave@example.com" in [ a.email
+                        for a in self._mlist.digest_members.addresses])
+
+    def test_original_email(self):
+        import_config_pck(self._mlist, self._pckdict)
+        bob = self._usermanager.get_address("bob@example.com")
+        self.assertEqual(bob.original_email, "bob@ExampLe.Com")
+        dave = self._usermanager.get_address("dave@example.com")
+        self.assertEqual(dave.original_email, "dave@ExampLe.Com")
+
+    def test_language(self):
+        import_config_pck(self._mlist, self._pckdict)
+        for name in ("anne", "bob", "cindy", "dave"):
+            addr = "%s@example.com" % name
+            member = self._mlist.members.get_member(addr)
+            self.assertTrue(member is not None,
+                            "Address %s was not imported" % addr)
+            print(self._pckdict["language"])
+            print(member.preferred_language, member.preferred_language.code)
+            self.assertEqual(member.preferred_language.code,
+                             self._pckdict["language"][addr])
+
+    def test_username(self):
+        import_config_pck(self._mlist, self._pckdict)
+        for name in ("anne", "bob", "cindy", "dave"):
+            addr = "%s@example.com" % name
+            user = self._usermanager.get_user(addr)
+            address = self._usermanager.get_address(addr)
+            self.assertTrue(user is not None,
+                    "User %s was not imported" % addr)
+            self.assertTrue(address is not None,
+                    "Address %s was not imported" % addr)
+            display_name = self._pckdict["usernames"][addr]
+            self.assertEqual(user.display_name, display_name,
+                    "The display name was not set for User %s" % addr)
+            self.assertEqual(address.display_name, display_name,
+                    "The display name was not set for Address %s" % addr)
+
+    def test_owner(self):
+        import_config_pck(self._mlist, self._pckdict)
+        for name in ("anne", "emily"):
+            addr = "%s@example.com" % name
+            self.assertTrue(
+                    addr in [ a.email for a in self._mlist.owners.addresses ],
+                    "Address %s was not imported as owner" % addr)
+        self.assertFalse("emily@example.com" in
+                [ a.email for a in self._mlist.members.addresses ],
+                "Address emily@ was wrongly added to the members list")
+
+    def test_moderator(self):
+        import_config_pck(self._mlist, self._pckdict)
+        for name in ("bob", "fred"):
+            addr = "%s@example.com" % name
+            self.assertTrue(
+                    addr in [ a.email for a in self._mlist.moderators.addresses ],
+                    "Address %s was not imported as moderator" % addr)
+        self.assertFalse("fred@example.com" in
+                [ a.email for a in self._mlist.members.addresses ],
+                "Address fred@ was wrongly added to the members list")
+
+    def test_password(self):
+        #self.anne.password = config.password_context.encrypt('abc123')
+        import_config_pck(self._mlist, self._pckdict)
+        for name in ("anne", "bob", "cindy", "dave"):
+            addr = "%s@example.com" % name
+            user = self._usermanager.get_user(addr)
+            self.assertTrue(user is not None,
+                    "Address %s was not imported" % addr)
+            self.assertEqual(user.password, '{plaintext}%spass' % name,
+                    "Password for %s was not imported" % addr)
+
+    def test_same_user(self):
+        # Adding the address of an existing User must not create another user
+        user = self._usermanager.create_user('anne@example.com', 'Anne')
+        user.register("bob@example.com") # secondary email
+        import_config_pck(self._mlist, self._pckdict)
+        member = self._mlist.members.get_member('bob@example.com')
+        self.assertEqual(member.user, user)
+
+
+
+class TestPreferencesImport(unittest.TestCase):
+
+    layer = ConfigLayer
+
+    def setUp(self):
+        self._mlist = create_list('blank@example.com')
+        self._pckdict = {
+            "members": { "anne@example.com": 0 },
+            "user_options": {},
+            "delivery_status": {},
+        }
+        self._usermanager = getUtility(IUserManager)
+
+    def tearDown(self):
+        remove_list(self._mlist)
+
+    def _do_test(self, oldvalue, expected):
+        self._pckdict["user_options"]["anne@example.com"] = oldvalue
+        import_config_pck(self._mlist, self._pckdict)
+        user = self._usermanager.get_user("anne@example.com")
+        self.assertTrue(user is not None, "User was not imported")
+        member = self._mlist.members.get_member("anne@example.com")
+        self.assertTrue(member is not None, "Address was not subscribed")
+        for exp_name, exp_val in expected.iteritems():
+            try:
+                currentval = getattr(member, exp_name)
+            except AttributeError:
+                # hide_address has no direct getter
+                currentval = getattr(member.preferences, exp_name)
+            self.assertEqual(currentval, exp_val,
+                    "Preference %s was not imported" % exp_name)
+        # XXX: should I check that other params are still equal to
+        # mailman.core.constants.system_preferences ?
+
+    def test_acknowledge_posts(self):
+        # AcknowledgePosts
+        self._do_test(4, {"acknowledge_posts": True})
+
+    def test_hide_address(self):
+        # ConcealSubscription
+        self._do_test(16, {"hide_address": True})
+
+    def test_receive_own_postings(self):
+        # DontReceiveOwnPosts
+        self._do_test(2, {"receive_own_postings": False})
+
+    def test_receive_list_copy(self):
+        # DontReceiveDuplicates
+        self._do_test(256, {"receive_list_copy": False})
+
+    def test_digest_plain(self):
+        # Digests & DisableMime
+        self._pckdict["digest_members"] = self._pckdict["members"].copy()
+        self._pckdict["members"] = {}
+        self._do_test(8, {"delivery_mode": DeliveryMode.plaintext_digests})
+
+    def test_digest_mime(self):
+        # Digests & not DisableMime
+        self._pckdict["digest_members"] = self._pckdict["members"].copy()
+        self._pckdict["members"] = {}
+        self._do_test(0, {"delivery_mode": DeliveryMode.mime_digests})
+
+    def test_delivery_status(self):
+        # look for the pckdict["delivery_status"] key which will look like
+        # (status, time) where status is among the following:
+        # ENABLED  = 0 # enabled
+        # UNKNOWN  = 1 # legacy disabled
+        # BYUSER   = 2 # disabled by user choice
+        # BYADMIN  = 3 # disabled by admin choice
+        # BYBOUNCE = 4 # disabled by bounces
+        for oldval, expected in enumerate((DeliveryStatus.enabled,
+                DeliveryStatus.unknown, DeliveryStatus.by_user,
+                DeliveryStatus.by_moderator, DeliveryStatus.by_bounces)):
+            self._pckdict["delivery_status"]["anne@example.com"] = (oldval, 0)
+            import_config_pck(self._mlist, self._pckdict)
+            member = self._mlist.members.get_member("anne@example.com")
+            self.assertTrue(member is not None, "Address was not subscribed")
+            self.assertEqual(member.delivery_status, expected)
+            member.unsubscribe()
+
+    def test_moderate(self):
+        # Option flag Moderate is translated to
+        # member.moderation_action = Action.hold
+        self._do_test(128, {"moderation_action": Action.hold})
+
+    def test_multiple_options(self):
+        # DontReceiveDuplicates & DisableMime & SuppressPasswordReminder
+        self._pckdict["digest_members"] = self._pckdict["members"].copy()
+        self._pckdict["members"] = {}
+        self._do_test(296, {
+                "receive_list_copy": False,
+                "delivery_mode": DeliveryMode.plaintext_digests,
+                })
-- 
cgit v1.3.1


From 5e7a56fe2dee15d5ea1be7942774b20560231821 Mon Sep 17 00:00:00 2001
From: Aurélien Bompard
Date: Thu, 3 Oct 2013 17:14:57 +0200
Subject: Strings are almost always bytestrings in the pickle, test for that

---
 src/mailman/utilities/importer.py          |   6 +-
 src/mailman/utilities/tests/test_import.py | 110 ++++++++++++++---------------
 2 files changed, 58 insertions(+), 58 deletions(-)

(limited to 'src')

diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py
index 50d2e7640..0822f3362 100644
--- a/src/mailman/utilities/importer.py
+++ b/src/mailman/utilities/importer.py
@@ -197,13 +197,13 @@ def import_config_pck(mlist, config_dict):
         mlist.archive_policy = ArchivePolicy.never
     # Handle ban list
     for addr in config_dict.get('ban_list', []):
-        IBanManager(mlist).ban(addr)
+        IBanManager(mlist).ban(unicode(addr))
     # Handle acceptable aliases
     for addr in config_dict.get('acceptable_aliases', '').splitlines():
         addr = addr.strip()
         if not addr:
             continue
-        IAcceptableAliasSet(mlist).add(addr)
+        IAcceptableAliasSet(mlist).add(unicode(addr))
     # Handle conversion to URIs
     convert_to_uri = {
         "welcome_msg": "welcome_message_uri",
@@ -299,7 +299,7 @@ def import_roster(mlist, config_dict, members, role):
                 original_email = merged_members[email]
             else:
                 original_email = email
-            user = usermanager.create_user(original_email)
+            user = usermanager.create_user(unicode(original_email))
         address = usermanager.get_address(email)
         address.verified_on = datetime.datetime.now()
         mlist.subscribe(address, role)
diff --git a/src/mailman/utilities/tests/test_import.py b/src/mailman/utilities/tests/test_import.py
index b8eb3e280..bc018f33d 100644
--- a/src/mailman/utilities/tests/test_import.py
+++ b/src/mailman/utilities/tests/test_import.py
@@ -192,7 +192,7 @@ class TestBasicImport(unittest.TestCase):
             ("anne@example.com", "anne@example.com"),
             ("^.*@example.com", "bob@example.com")
             ]
-        self._pckdict["ban_list"] = [ b[0] for b in banned ]
+        self._pckdict["ban_list"] = [ str(b[0]) for b in banned ]
         self._import()
         for _pattern, addr in banned:
             self.assertTrue(IBanManager(self._mlist).is_banned(addr))
@@ -200,7 +200,7 @@ class TestBasicImport(unittest.TestCase):
     def test_acceptable_aliases(self):
         # it used to be a plain-text field (values are newline-separated)
         aliases = ["alias1@example.com", "alias2@exemple.com"]
-        self._pckdict["acceptable_aliases"] = "\n".join(aliases)
+        self._pckdict[b"acceptable_aliases"] = str("\n".join(aliases))
         self._import()
         alias_set = IAcceptableAliasSet(self._mlist)
         self.assertEqual(sorted(alias_set.aliases), aliases)
@@ -287,8 +287,8 @@ class TestMemberActionImport(unittest.TestCase):
         self._mlist.default_member_action = DummyEnum.val
         self._mlist.default_nonmember_action = DummyEnum.val
         self._pckdict = {
-            "member_moderation_action": DummyEnum.val,
-            "generic_nonmember_action": DummyEnum.val,
+            b"member_moderation_action": DummyEnum.val,
+            b"generic_nonmember_action": DummyEnum.val,
         }
 
     def tearDown(self):
@@ -300,31 +300,31 @@ class TestMemberActionImport(unittest.TestCase):
             self.assertEqual(getattr(self._mlist, key), value)
 
     def test_member_hold(self):
-        self._pckdict["member_moderation_action"] = 0
+        self._pckdict[b"member_moderation_action"] = 0
         self._do_test({"default_member_action": Action.hold})
 
     def test_member_reject(self):
-        self._pckdict["member_moderation_action"] = 1
+        self._pckdict[b"member_moderation_action"] = 1
         self._do_test({"default_member_action": Action.reject})
 
     def test_member_discard(self):
-        self._pckdict["member_moderation_action"] = 2
+        self._pckdict[b"member_moderation_action"] = 2
         self._do_test({"default_member_action": Action.discard})
 
     def test_nonmember_accept(self):
-        self._pckdict["generic_nonmember_action"] = 0
+        self._pckdict[b"generic_nonmember_action"] = 0
         self._do_test({"default_nonmember_action": Action.accept})
 
     def test_nonmember_hold(self):
-        self._pckdict["generic_nonmember_action"] = 1
+        self._pckdict[b"generic_nonmember_action"] = 1
         self._do_test({"default_nonmember_action": Action.hold})
 
     def test_nonmember_reject(self):
-        self._pckdict["generic_nonmember_action"] = 2
+        self._pckdict[b"generic_nonmember_action"] = 2
         self._do_test({"default_nonmember_action": Action.reject})
 
     def test_nonmember_discard(self):
-        self._pckdict["generic_nonmember_action"] = 3
+        self._pckdict[b"generic_nonmember_action"] = 3
         self._do_test({"default_nonmember_action": Action.discard})
 
 
@@ -365,7 +365,7 @@ class TestConvertToURI(unittest.TestCase):
 
     def test_text_to_uri(self):
         for oldvar, newvar in self._conf_mapping.iteritems():
-            self._pckdict[oldvar] = "TEST VALUE"
+            self._pckdict[str(oldvar)] = b"TEST VALUE"
             import_config_pck(self._mlist, self._pckdict)
             newattr = getattr(self._mlist, newvar)
             text = decorate(self._mlist, newattr)
@@ -381,7 +381,7 @@ class TestConvertToURI(unittest.TestCase):
                          "$fqdn_listname\n"
                          "$listinfo_uri")
         for oldvar, newvar in self._conf_mapping.iteritems():
-            self._pckdict[oldvar] = test_text
+            self._pckdict[str(oldvar)] = str(test_text)
             import_config_pck(self._mlist, self._pckdict)
             newattr = getattr(self._mlist, newvar)
             template_uri = expand(newattr, dict(
@@ -403,7 +403,7 @@ class TestConvertToURI(unittest.TestCase):
             )
         for oldvar in ("msg_footer", "digest_footer"):
             newvar = self._conf_mapping[oldvar]
-            self._pckdict[oldvar] = default_msg_footer
+            self._pckdict[str(oldvar)] = str(default_msg_footer)
             old_value = getattr(self._mlist, newvar)
             import_config_pck(self._mlist, self._pckdict)
             new_value = getattr(self._mlist, newvar)
@@ -418,8 +418,8 @@ class TestConvertToURI(unittest.TestCase):
         test_value = "TEST-VALUE"
         for oldvar, newvar in self._conf_mapping.iteritems():
             self._mlist.mail_host = "example.com"
-            self._pckdict["mail_host"] = "test.example.com"
-            self._pckdict[oldvar] = test_value
+            self._pckdict[b"mail_host"] = b"test.example.com"
+            self._pckdict[str(oldvar)] = test_value
             old_value = getattr(self._mlist, newvar)
             import_config_pck(self._mlist, self._pckdict)
             new_value = getattr(self._mlist, newvar)
@@ -435,44 +435,44 @@ class TestRosterImport(unittest.TestCase):
     def setUp(self):
         self._mlist = create_list('blank@example.com')
         self._pckdict = {
-            "members": {
-                "anne@example.com": 0,
-                "bob@example.com": "bob@ExampLe.Com",
+            b"members": {
+                b"anne@example.com": 0,
+                b"bob@example.com": b"bob@ExampLe.Com",
             },
-            "digest_members": {
-                "cindy@example.com": 0,
-                "dave@example.com": "dave@ExampLe.Com",
+            b"digest_members": {
+                b"cindy@example.com": 0,
+                b"dave@example.com": b"dave@ExampLe.Com",
             },
-            "passwords": {
-                "anne@example.com" : "annepass",
-                "bob@example.com"  : "bobpass",
-                "cindy@example.com": "cindypass",
-                "dave@example.com" : "davepass",
+            b"passwords": {
+                b"anne@example.com" : b"annepass",
+                b"bob@example.com"  : b"bobpass",
+                b"cindy@example.com": b"cindypass",
+                b"dave@example.com" : b"davepass",
             },
-            "language": {
-                "anne@example.com" : "fr",
-                "bob@example.com"  : "de",
-                "cindy@example.com": "es",
-                "dave@example.com" : "it",
+            b"language": {
+                b"anne@example.com" : b"fr",
+                b"bob@example.com"  : b"de",
+                b"cindy@example.com": b"es",
+                b"dave@example.com" : b"it",
             },
-            "usernames": {
-                "anne@example.com" : "Anne",
-                "bob@example.com"  : "Bob",
-                "cindy@example.com": "Cindy",
-                "dave@example.com" : "Dave",
+            b"usernames": { # Usernames are unicode strings in the pickle
+                b"anne@example.com" : "Anne",
+                b"bob@example.com"  : "Bob",
+                b"cindy@example.com": "Cindy",
+                b"dave@example.com" : "Dave",
             },
-            "owner": [
-                "anne@example.com",
-                "emily@example.com",
+            b"owner": [
+                b"anne@example.com",
+                b"emily@example.com",
             ],
-            "moderator": [
-                "bob@example.com",
-                "fred@example.com",
+            b"moderator": [
+                b"bob@example.com",
+                b"fred@example.com",
             ],
         }
         self._usermanager = getUtility(IUserManager)
         language_manager = getUtility(ILanguageManager)
-        for code in self._pckdict["language"].values():
+        for code in self._pckdict[b"language"].values():
             if code not in language_manager.codes:
                 language_manager.add(code, 'utf-8', code)
 
@@ -580,9 +580,9 @@ class TestPreferencesImport(unittest.TestCase):
     def setUp(self):
         self._mlist = create_list('blank@example.com')
         self._pckdict = {
-            "members": { "anne@example.com": 0 },
-            "user_options": {},
-            "delivery_status": {},
+            b"members": { b"anne@example.com": 0 },
+            b"user_options": {},
+            b"delivery_status": {},
         }
         self._usermanager = getUtility(IUserManager)
 
@@ -590,7 +590,7 @@ class TestPreferencesImport(unittest.TestCase):
         remove_list(self._mlist)
 
     def _do_test(self, oldvalue, expected):
-        self._pckdict["user_options"]["anne@example.com"] = oldvalue
+        self._pckdict[b"user_options"][b"anne@example.com"] = oldvalue
         import_config_pck(self._mlist, self._pckdict)
         user = self._usermanager.get_user("anne@example.com")
         self.assertTrue(user is not None, "User was not imported")
@@ -625,14 +625,14 @@ class TestPreferencesImport(unittest.TestCase):
 
     def test_digest_plain(self):
         # Digests & DisableMime
-        self._pckdict["digest_members"] = self._pckdict["members"].copy()
-        self._pckdict["members"] = {}
+        self._pckdict[b"digest_members"] = self._pckdict[b"members"].copy()
+        self._pckdict[b"members"] = {}
         self._do_test(8, {"delivery_mode": DeliveryMode.plaintext_digests})
 
     def test_digest_mime(self):
         # Digests & not DisableMime
-        self._pckdict["digest_members"] = self._pckdict["members"].copy()
-        self._pckdict["members"] = {}
+        self._pckdict[b"digest_members"] = self._pckdict[b"members"].copy()
+        self._pckdict[b"members"] = {}
         self._do_test(0, {"delivery_mode": DeliveryMode.mime_digests})
 
     def test_delivery_status(self):
@@ -646,7 +646,7 @@ class TestPreferencesImport(unittest.TestCase):
         for oldval, expected in enumerate((DeliveryStatus.enabled,
                 DeliveryStatus.unknown, DeliveryStatus.by_user,
                 DeliveryStatus.by_moderator, DeliveryStatus.by_bounces)):
-            self._pckdict["delivery_status"]["anne@example.com"] = (oldval, 0)
+            self._pckdict[b"delivery_status"][b"anne@example.com"] = (oldval, 0)
             import_config_pck(self._mlist, self._pckdict)
             member = self._mlist.members.get_member("anne@example.com")
             self.assertTrue(member is not None, "Address was not subscribed")
@@ -660,8 +660,8 @@ class TestPreferencesImport(unittest.TestCase):
 
     def test_multiple_options(self):
         # DontReceiveDuplicates & DisableMime & SuppressPasswordReminder
-        self._pckdict["digest_members"] = self._pckdict["members"].copy()
-        self._pckdict["members"] = {}
+        self._pckdict[b"digest_members"] = self._pckdict[b"members"].copy()
+        self._pckdict[b"members"] = {}
         self._do_test(296, {
                 "receive_list_copy": False,
                 "delivery_mode": DeliveryMode.plaintext_digests,
-- 
cgit v1.3.1


From 0fd9307139c0c5f2245d5df39e304b5b9d06df3e Mon Sep 17 00:00:00 2001
From: Aurélien Bompard
Date: Thu, 3 Oct 2013 18:37:04 +0200
Subject: More unicode fixes in the import script

---
 src/mailman/utilities/importer.py          | 15 ++++++++++++++-
 src/mailman/utilities/tests/test_import.py | 22 +++++++++++++++++++++-
 2 files changed, 35 insertions(+), 2 deletions(-)

(limited to 'src')

diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py
index 0822f3362..14214438b 100644
--- a/src/mailman/utilities/importer.py
+++ b/src/mailman/utilities/importer.py
@@ -103,6 +103,10 @@ def nonmember_action_mapping(value):
     elif value == 3:
         return Action.discard
 
+
+def unicode_to_string(value):
+    return str(value) if value is not None else None
+
 
 # Attributes in Mailman 2 which have a different type in Mailman 3.
 TYPES = dict(
@@ -124,6 +128,7 @@ TYPES = dict(
     forward_unrecognized_bounces_to=UnrecognizedBounceDisposition,
     default_member_action=member_action_mapping,
     default_nonmember_action=nonmember_action_mapping,
+    moderator_password=unicode_to_string,
     )
 
 
@@ -177,7 +182,15 @@ def import_config_pck(mlist, config_dict):
         # strings).
         if hasattr(mlist, key):
             if isinstance(value, str):
-                value = unicode(value, 'ascii')
+                for encoding in ("ascii", "utf-8"):
+                    try:
+                        value = unicode(value, encoding)
+                    except UnicodeDecodeError, e:
+                        continue
+                    else:
+                        break
+                if isinstance(value, str): # we did our best
+                    value = unicode(value, 'ascii', 'replace')
             # Some types require conversion.
             converter = TYPES.get(key)
             if converter is not None:
diff --git a/src/mailman/utilities/tests/test_import.py b/src/mailman/utilities/tests/test_import.py
index bc018f33d..d4ae72fbf 100644
--- a/src/mailman/utilities/tests/test_import.py
+++ b/src/mailman/utilities/tests/test_import.py
@@ -168,6 +168,12 @@ class TestBasicImport(unittest.TestCase):
         self._import()
         self.assertEqual(self._mlist.moderator_password, None)
 
+    def test_moderator_password_str(self):
+        # moderator_password must not be unicode
+        self._pckdict[b"mod_password"] = b'TESTVALUE'
+        self._import()
+        self.assertEqual(self._mlist.moderator_password, b'TESTVALUE')
+
     def test_newsgroup_moderation(self):
         # news_moderation -> newsgroup_moderation
         # news_prefix_subject_too -> nntp_prefix_subject_too
@@ -205,6 +211,20 @@ class TestBasicImport(unittest.TestCase):
         alias_set = IAcceptableAliasSet(self._mlist)
         self.assertEqual(sorted(alias_set.aliases), aliases)
 
+    def test_info_non_ascii(self):
+        # info can contain non-ascii chars
+        info = 'O idioma aceito \xe9 somente Portugu\xeas do Brasil'
+        self._pckdict[b"info"] = info.encode("utf-8")
+        self._import()
+        self.assertEqual(self._mlist.info, info,
+                         "Encoding to UTF-8 is not handled")
+        # test fallback to ascii with replace
+        self._pckdict[b"info"] = info.encode("iso-8859-1")
+        self._import()
+        self.assertEqual(self._mlist.info,
+                         unicode(self._pckdict[b"info"], "ascii", "replace"),
+                         "We don't fall back to replacing non-ascii chars")
+
 
 
 class TestArchiveImport(unittest.TestCase):
@@ -560,7 +580,7 @@ class TestRosterImport(unittest.TestCase):
             user = self._usermanager.get_user(addr)
             self.assertTrue(user is not None,
                     "Address %s was not imported" % addr)
-            self.assertEqual(user.password, '{plaintext}%spass' % name,
+            self.assertEqual(user.password, b'{plaintext}%spass' % name,
                     "Password for %s was not imported" % addr)
 
     def test_same_user(self):
-- 
cgit v1.3.1


From 878f0324648e32c15e728497e1fc82ac74f35509 Mon Sep 17 00:00:00 2001
From: Aurélien Bompard
Date: Thu, 3 Oct 2013 18:54:57 +0200
Subject: Content filter must be removed on list deletion

---
 src/mailman/model/listmanager.py            |  2 ++
 src/mailman/model/tests/test_listmanager.py | 15 +++++++++++++++
 2 files changed, 17 insertions(+)

(limited to 'src')

diff --git a/src/mailman/model/listmanager.py b/src/mailman/model/listmanager.py
index 5e260a6cd..f1c2941e0 100644
--- a/src/mailman/model/listmanager.py
+++ b/src/mailman/model/listmanager.py
@@ -34,6 +34,7 @@ from mailman.interfaces.listmanager import (
     IListManager, ListAlreadyExistsError, ListCreatedEvent, ListCreatingEvent,
     ListDeletedEvent, ListDeletingEvent)
 from mailman.model.mailinglist import MailingList
+from mailman.model.mime import ContentFilter
 from mailman.utilities.datetime import now
 
 
@@ -79,6 +80,7 @@ class ListManager:
         """See `IListManager`."""
         fqdn_listname = mlist.fqdn_listname
         notify(ListDeletingEvent(mlist))
+        store.find(ContentFilter, ContentFilter.mailing_list == mlist).remove()
         store.remove(mlist)
         notify(ListDeletedEvent(fqdn_listname))
 
diff --git a/src/mailman/model/tests/test_listmanager.py b/src/mailman/model/tests/test_listmanager.py
index 152d96b9f..32f16993f 100644
--- a/src/mailman/model/tests/test_listmanager.py
+++ b/src/mailman/model/tests/test_listmanager.py
@@ -27,6 +27,7 @@ __all__ = [
 import unittest
 
 from zope.component import getUtility
+from storm.locals import Store
 
 from mailman.app.lifecycle import create_list
 from mailman.app.moderator import hold_message
@@ -37,6 +38,7 @@ from mailman.interfaces.messages import IMessageStore
 from mailman.interfaces.requests import IListRequests
 from mailman.interfaces.subscriptions import ISubscriptionService
 from mailman.interfaces.usermanager import IUserManager
+from mailman.model.mime import ContentFilter
 from mailman.testing.helpers import (
     event_subscribers, specialized_message_from_string)
 from mailman.testing.layers import ConfigLayer
@@ -126,6 +128,19 @@ Message-ID: <argon>
         saved_message = getUtility(IMessageStore).get_message_by_id('<argon>')
         self.assertEqual(saved_message.as_string(), msg.as_string())
 
+    def test_content_filters_are_deleted_when_mailing_list_is_deleted(self):
+        # When a mailing list with content filters is deleted, the filters must
+        # be deleted fist or an IntegrityError will be raised
+        filter_names = ("filter_types", "pass_types",
+                        "filter_extensions", "pass_extensions")
+        for fname in filter_names:
+            setattr(self._ant, fname, ["test-filter-1", "test-filter-2"])
+        getUtility(IListManager).delete(self._ant)
+        store = Store.of(self._ant)
+        filters = store.find(ContentFilter,
+                             ContentFilter.mailing_list == self._ant)
+        self.assertEqual(filters.count(), 0)
+
 
 
 class TestListCreation(unittest.TestCase):
-- 
cgit v1.3.1


From 004915c1fa7eaa8957101d41eeef0c48ff621eeb Mon Sep 17 00:00:00 2001
From: Aurélien Bompard
Date: Tue, 8 Oct 2013 10:55:24 +0200
Subject: Some more unicode checks when importing

---
 src/mailman/utilities/importer.py          | 31 ++++++++++++++++++++++--------
 src/mailman/utilities/tests/test_import.py | 17 +++++++++++++++-
 2 files changed, 39 insertions(+), 9 deletions(-)

(limited to 'src')

diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py
index 14214438b..9ada4c75e 100644
--- a/src/mailman/utilities/importer.py
+++ b/src/mailman/utilities/importer.py
@@ -231,28 +231,43 @@ def import_config_pck(mlist, config_dict):
         "%(real_name)s@%(host_name)s": "$fqdn_listname",
         "%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s": "$listinfo_uri",
     }
+    # Collect defaults
+    defaults = {}
+    for oldvar, newvar in convert_to_uri.iteritems():
+        default_value = getattr(mlist, newvar)
+        if not default_value:
+            continue
+        # Check if the value changed from the default
+        try:
+            default_text = decorate(mlist, default_value)
+        except (URLError, KeyError):
+            # Use case: importing the old a@ex.com into b@ex.com
+            # We can't check if it changed from the default
+            # -> don't import, we may do more harm than good and it's easy to
+            # change if needed
+            continue
+        defaults[newvar] = (default_value, default_text)
     for oldvar, newvar in convert_to_uri.iteritems():
         if oldvar not in config_dict:
             continue
         text = config_dict[oldvar]
+        text = unicode(text, "utf-8", "replace")
         for oldph, newph in convert_placeholders.iteritems():
             text = text.replace(oldph, newph)
-        default_value = getattr(mlist, newvar)
-        if not text and not default_value:
-            continue
+        default_value, default_text  = defaults.get(newvar, (None, None))
+        if not text and not (default_value or default_text):
+            continue # both are empty, leave it
         # Check if the value changed from the default
         try:
-            default_text = decorate(mlist, default_value)
             expanded_text = decorate_template(mlist, text)
-        except (URLError, KeyError):
+        except KeyError:
             # Use case: importing the old a@ex.com into b@ex.com
             # We can't check if it changed from the default
             # -> don't import, we may do more harm than good and it's easy to
             # change if needed
             continue
-        if not text and not default_text:
-            continue # both are empty, leave it
-        if expanded_text.strip() == default_text.strip():
+        if expanded_text and default_text \
+                and expanded_text.strip() == default_text.strip():
             continue # keep the default
         # Write the custom value to the right file
         base_uri = "mailman:///$listname/$language/"
diff --git a/src/mailman/utilities/tests/test_import.py b/src/mailman/utilities/tests/test_import.py
index d4ae72fbf..2e6a4c0be 100644
--- a/src/mailman/utilities/tests/test_import.py
+++ b/src/mailman/utilities/tests/test_import.py
@@ -172,6 +172,7 @@ class TestBasicImport(unittest.TestCase):
         # moderator_password must not be unicode
         self._pckdict[b"mod_password"] = b'TESTVALUE'
         self._import()
+        self.assertFalse(isinstance(self._mlist.moderator_password, unicode))
         self.assertEqual(self._mlist.moderator_password, b'TESTVALUE')
 
     def test_newsgroup_moderation(self):
@@ -435,7 +436,7 @@ class TestConvertToURI(unittest.TestCase):
         # We can't check if it changed from the default
         # -> don't import, we may do more harm than good and it's easy to
         # change if needed
-        test_value = "TEST-VALUE"
+        test_value = b"TEST-VALUE"
         for oldvar, newvar in self._conf_mapping.iteritems():
             self._mlist.mail_host = "example.com"
             self._pckdict[b"mail_host"] = b"test.example.com"
@@ -446,6 +447,20 @@ class TestConvertToURI(unittest.TestCase):
             self.assertEqual(old_value, new_value,
                     "Default value was not preserved for %s" % newvar)
 
+    def test_unicode(self):
+        for oldvar in self._conf_mapping:
+            self._pckdict[str(oldvar)] = b"Ol\xe1!"
+        try:
+            import_config_pck(self._mlist, self._pckdict)
+        except UnicodeDecodeError, e:
+            self.fail(e)
+        for oldvar, newvar in self._conf_mapping.iteritems():
+            newattr = getattr(self._mlist, newvar)
+            text = decorate(self._mlist, newattr)
+            expected = u'Ol\ufffd!'.encode("utf-8")
+            # we get bytestrings because the text is stored in a file
+            self.assertEqual(text, expected)
+
 
 
 class TestRosterImport(unittest.TestCase):
-- 
cgit v1.3.1


From 228e1c57fb79411381fc0da3bcbd69fbaf3cbf9a Mon Sep 17 00:00:00 2001
From: Aurélien Bompard
Date: Tue, 8 Oct 2013 15:45:36 +0200
Subject: Make sure the imported preferred_language exists, and work around
 Python issue 9666

---
 src/mailman/commands/cli_import.py         |  8 ++++--
 src/mailman/utilities/importer.py          | 34 ++++++++++++++++++++++++--
 src/mailman/utilities/tests/test_import.py | 39 +++++++++++++++++++++++++++++-
 3 files changed, 76 insertions(+), 5 deletions(-)

(limited to 'src')

diff --git a/src/mailman/commands/cli_import.py b/src/mailman/commands/cli_import.py
index c8145429d..880457334 100644
--- a/src/mailman/commands/cli_import.py
+++ b/src/mailman/commands/cli_import.py
@@ -35,7 +35,7 @@ from mailman.core.i18n import _
 from mailman.database.transaction import transactional
 from mailman.interfaces.command import ICLISubCommand
 from mailman.interfaces.listmanager import IListManager
-from mailman.utilities.importer import import_config_pck
+from mailman.utilities.importer import import_config_pck, Import21Error
 
 
 
@@ -93,4 +93,8 @@ class Import21:
                         print(_('Ignoring non-dictionary: {0!r}').format(
                             config_dict), file=sys.stderr)
                         continue
-                    import_config_pck(mlist, config_dict)
+                    try:
+                        import_config_pck(mlist, config_dict)
+                    except Import21Error, e:
+                        print(e, file=sys.stderr)
+                        sys.exit(1)
diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py
index 9ada4c75e..dcdc0e4c3 100644
--- a/src/mailman/utilities/importer.py
+++ b/src/mailman/utilities/importer.py
@@ -22,6 +22,7 @@ from __future__ import absolute_import, print_function, unicode_literals
 __metaclass__ = type
 __all__ = [
     'import_config_pck',
+    'ImportError',
     ]
 
 
@@ -31,6 +32,7 @@ import os
 from urllib2 import URLError
 
 from mailman.config import config
+from mailman.core.errors import MailmanError
 from mailman.interfaces.action import FilterAction, Action
 from mailman.interfaces.autorespond import ResponseAction
 from mailman.interfaces.digests import DigestFrequency
@@ -42,11 +44,16 @@ from mailman.interfaces.mailinglist import IAcceptableAliasSet
 from mailman.interfaces.bounce import UnrecognizedBounceDisposition
 from mailman.interfaces.usermanager import IUserManager
 from mailman.interfaces.member import DeliveryMode, DeliveryStatus, MemberRole
+from mailman.interfaces.languages import ILanguageManager
 from mailman.handlers.decorate import decorate, decorate_template
 from mailman.utilities.i18n import search
 from zope.component import getUtility
 
 
+
+class Import21Error(MailmanError):
+    pass
+
 
 def seconds_to_delta(value):
     return datetime.timedelta(seconds=value)
@@ -107,6 +114,26 @@ def nonmember_action_mapping(value):
 def unicode_to_string(value):
     return str(value) if value is not None else None
 
+
+def check_language_code(code):
+    if code is None:
+        return None
+    code = unicode(code)
+    if code not in getUtility(ILanguageManager):
+        msg = """Missing language: {0}
+You must add a section describing this language in your mailman.cfg file.
+This section should look like this:
+[language.{1}]
+# The English name for the language.
+description: CHANGE ME
+# And the default character set for the language.
+charset: utf-8
+# Whether the language is enabled or not.
+enabled: yes
+""".format(code, code[:2])
+        raise Import21Error(msg)
+    return code
+
 
 # Attributes in Mailman 2 which have a different type in Mailman 3.
 TYPES = dict(
@@ -129,6 +156,7 @@ TYPES = dict(
     default_member_action=member_action_mapping,
     default_nonmember_action=nonmember_action_mapping,
     moderator_password=unicode_to_string,
+    preferred_language=check_language_code,
     )
 
 
@@ -180,7 +208,9 @@ def import_config_pck(mlist, config_dict):
         # Handle the simple case where the key is an attribute of the
         # IMailingList and the types are the same (modulo 8-bit/unicode
         # strings).
-        if hasattr(mlist, key):
+        # When attributes raise an exception, hasattr may think they don't
+        # exist (see python issue 9666). Add them here.
+        if hasattr(mlist, key) or key in ("preferred_language", ):
             if isinstance(value, str):
                 for encoding in ("ascii", "utf-8"):
                     try:
@@ -346,7 +376,7 @@ def import_roster(mlist, config_dict, members, role):
             pass
         if email in config_dict.get("language", {}):
             member.preferences.preferred_language = \
-                unicode(config_dict["language"][email])
+                check_language_code(config_dict["language"][email])
         # if the user already exists, display_name and password will be
         # overwritten
         if email in config_dict.get("usernames", {}):
diff --git a/src/mailman/utilities/tests/test_import.py b/src/mailman/utilities/tests/test_import.py
index 2e6a4c0be..2cd737f08 100644
--- a/src/mailman/utilities/tests/test_import.py
+++ b/src/mailman/utilities/tests/test_import.py
@@ -31,7 +31,7 @@ from datetime import timedelta, datetime
 
 from mailman.app.lifecycle import create_list, remove_list
 from mailman.testing.layers import ConfigLayer
-from mailman.utilities.importer import import_config_pck
+from mailman.utilities.importer import import_config_pck, Import21Error
 from mailman.interfaces.archiver import ArchivePolicy
 from mailman.interfaces.action import Action, FilterAction
 from mailman.interfaces.bounce import UnrecognizedBounceDisposition
@@ -226,6 +226,32 @@ class TestBasicImport(unittest.TestCase):
                          unicode(self._pckdict[b"info"], "ascii", "replace"),
                          "We don't fall back to replacing non-ascii chars")
 
+    def test_preferred_language(self):
+        self._pckdict[b"preferred_language"] = b'ja'
+        english = getUtility(ILanguageManager).get('en')
+        japanese = getUtility(ILanguageManager).get('ja')
+        self.assertEqual(self._mlist.preferred_language, english)
+        self._import()
+        self.assertEqual(self._mlist.preferred_language, japanese)
+
+    def test_preferred_language_unknown_previous(self):
+        # when the previous language is unknown, it should not fail
+        self._mlist._preferred_language = 'xx' # non-existant
+        self._import()
+        english = getUtility(ILanguageManager).get('en')
+        self.assertEqual(self._mlist.preferred_language, english)
+
+    def test_new_language(self):
+        self._pckdict[b"preferred_language"] = b'xx_XX'
+        try:
+            self._import()
+        except Import21Error, e:
+            # check the message
+            self.assertTrue("xx_XX" in str(e))
+            self.assertTrue("[language.xx]" in str(e))
+        else:
+            self.fail("Import21Error was not raised")
+
 
 
 class TestArchiveImport(unittest.TestCase):
@@ -549,6 +575,17 @@ class TestRosterImport(unittest.TestCase):
             self.assertEqual(member.preferred_language.code,
                              self._pckdict["language"][addr])
 
+    def test_new_language(self):
+        self._pckdict[b"language"][b"anne@example.com"] = b'xx_XX'
+        try:
+            import_config_pck(self._mlist, self._pckdict)
+        except Import21Error, e:
+            # check the message
+            self.assertTrue("xx_XX" in str(e))
+            self.assertTrue("[language.xx]" in str(e))
+        else:
+            self.fail("Import21Error was not raised")
+
     def test_username(self):
         import_config_pck(self._mlist, self._pckdict)
         for name in ("anne", "bob", "cindy", "dave"):
-- 
cgit v1.3.1


From 57dda2005b0074d083ccb309fea61eed9a8ecb18 Mon Sep 17 00:00:00 2001
From: Aurélien Bompard
Date: Wed, 9 Oct 2013 15:55:22 +0200
Subject: Deal with non-ascii bans and aliases

---
 src/mailman/utilities/importer.py          | 37 ++++++++++++++++++------------
 src/mailman/utilities/tests/test_import.py | 17 ++++++++++----
 2 files changed, 34 insertions(+), 20 deletions(-)

(limited to 'src')

diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py
index dcdc0e4c3..3ee2951bb 100644
--- a/src/mailman/utilities/importer.py
+++ b/src/mailman/utilities/importer.py
@@ -54,6 +54,19 @@ from zope.component import getUtility
 class Import21Error(MailmanError):
     pass
 
+
+def str_to_unicode(value):
+    # Convert a string to unicode when the encoding is not declared
+    if isinstance(value, unicode):
+        return value
+    for encoding in ("ascii", "utf-8"):
+        try:
+            return unicode(value, encoding)
+        except UnicodeDecodeError, e:
+            continue
+    # we did our best, use replace
+    return unicode(value, 'ascii', 'replace')
+
 
 def seconds_to_delta(value):
     return datetime.timedelta(seconds=value)
@@ -212,15 +225,7 @@ def import_config_pck(mlist, config_dict):
         # exist (see python issue 9666). Add them here.
         if hasattr(mlist, key) or key in ("preferred_language", ):
             if isinstance(value, str):
-                for encoding in ("ascii", "utf-8"):
-                    try:
-                        value = unicode(value, encoding)
-                    except UnicodeDecodeError, e:
-                        continue
-                    else:
-                        break
-                if isinstance(value, str): # we did our best
-                    value = unicode(value, 'ascii', 'replace')
+                value = str_to_unicode(value)
             # Some types require conversion.
             converter = TYPES.get(key)
             if converter is not None:
@@ -240,13 +245,13 @@ def import_config_pck(mlist, config_dict):
         mlist.archive_policy = ArchivePolicy.never
     # Handle ban list
     for addr in config_dict.get('ban_list', []):
-        IBanManager(mlist).ban(unicode(addr))
+        IBanManager(mlist).ban(str_to_unicode(addr))
     # Handle acceptable aliases
     for addr in config_dict.get('acceptable_aliases', '').splitlines():
         addr = addr.strip()
         if not addr:
             continue
-        IAcceptableAliasSet(mlist).add(unicode(addr))
+        IAcceptableAliasSet(mlist).add(str_to_unicode(addr))
     # Handle conversion to URIs
     convert_to_uri = {
         "welcome_msg": "welcome_message_uri",
@@ -342,7 +347,7 @@ def import_roster(mlist, config_dict, members, role):
     """
     usermanager = getUtility(IUserManager)
     for email in members:
-        email = unicode(email)
+        email = str_to_unicode(email)
         roster = mlist.get_roster(role)
         if roster.get_member(email) is not None:
             print("%s is already imported with role %s" % (email, role),
@@ -357,7 +362,7 @@ def import_roster(mlist, config_dict, members, role):
                 original_email = merged_members[email]
             else:
                 original_email = email
-            user = usermanager.create_user(unicode(original_email))
+            user = usermanager.create_user(str_to_unicode(original_email))
         address = usermanager.get_address(email)
         address.verified_on = datetime.datetime.now()
         mlist.subscribe(address, role)
@@ -380,8 +385,10 @@ def import_roster(mlist, config_dict, members, role):
         # if the user already exists, display_name and password will be
         # overwritten
         if email in config_dict.get("usernames", {}):
-            address.display_name = unicode(config_dict["usernames"][email])
-            user.display_name    = unicode(config_dict["usernames"][email])
+            address.display_name = \
+                    str_to_unicode(config_dict["usernames"][email])
+            user.display_name    = \
+                    str_to_unicode(config_dict["usernames"][email])
         if email in config_dict.get("passwords", {}):
             user.password = config.password_context.encrypt(
                                     config_dict["passwords"][email])
diff --git a/src/mailman/utilities/tests/test_import.py b/src/mailman/utilities/tests/test_import.py
index 2cd737f08..5a317fedb 100644
--- a/src/mailman/utilities/tests/test_import.py
+++ b/src/mailman/utilities/tests/test_import.py
@@ -197,17 +197,24 @@ class TestBasicImport(unittest.TestCase):
     def test_ban_list(self):
         banned = [
             ("anne@example.com", "anne@example.com"),
-            ("^.*@example.com", "bob@example.com")
+            ("^.*@example.com", "bob@example.com"),
+            ("non-ascii-\xe8@example.com", "non-ascii-\ufffd@example.com"),
             ]
-        self._pckdict["ban_list"] = [ str(b[0]) for b in banned ]
-        self._import()
+        self._pckdict["ban_list"] = [ b[0].encode("iso-8859-1") for b in banned ]
+        try:
+            self._import()
+        except UnicodeDecodeError, e:
+            self.fail(e)
         for _pattern, addr in banned:
             self.assertTrue(IBanManager(self._mlist).is_banned(addr))
 
     def test_acceptable_aliases(self):
         # it used to be a plain-text field (values are newline-separated)
-        aliases = ["alias1@example.com", "alias2@exemple.com"]
-        self._pckdict[b"acceptable_aliases"] = str("\n".join(aliases))
+        aliases = ["alias1@example.com",
+                   "alias2@exemple.com",
+                   "non-ascii-\xe8@example.com"]
+        self._pckdict[b"acceptable_aliases"] = \
+                ("\n".join(aliases)).encode("utf-8")
         self._import()
         alias_set = IAcceptableAliasSet(self._mlist)
         self.assertEqual(sorted(alias_set.aliases), aliases)
-- 
cgit v1.3.1


From 8e6cad997c026cfa81e23043531a2a0ac8c1c6a0 Mon Sep 17 00:00:00 2001
From: Aurélien Bompard
Date: Fri, 11 Oct 2013 17:37:37 +0200
Subject: Use the full language code in the configuration hint

---
 src/mailman/utilities/importer.py          | 4 ++--
 src/mailman/utilities/tests/test_import.py | 6 ++----
 2 files changed, 4 insertions(+), 6 deletions(-)

(limited to 'src')

diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py
index 3ee2951bb..9eb0fdf35 100644
--- a/src/mailman/utilities/importer.py
+++ b/src/mailman/utilities/importer.py
@@ -136,14 +136,14 @@ def check_language_code(code):
         msg = """Missing language: {0}
 You must add a section describing this language in your mailman.cfg file.
 This section should look like this:
-[language.{1}]
+[language.{0}]
 # The English name for the language.
 description: CHANGE ME
 # And the default character set for the language.
 charset: utf-8
 # Whether the language is enabled or not.
 enabled: yes
-""".format(code, code[:2])
+""".format(code)
         raise Import21Error(msg)
     return code
 
diff --git a/src/mailman/utilities/tests/test_import.py b/src/mailman/utilities/tests/test_import.py
index 5a317fedb..23cf74b36 100644
--- a/src/mailman/utilities/tests/test_import.py
+++ b/src/mailman/utilities/tests/test_import.py
@@ -254,8 +254,7 @@ class TestBasicImport(unittest.TestCase):
             self._import()
         except Import21Error, e:
             # check the message
-            self.assertTrue("xx_XX" in str(e))
-            self.assertTrue("[language.xx]" in str(e))
+            self.assertTrue("[language.xx_XX]" in str(e))
         else:
             self.fail("Import21Error was not raised")
 
@@ -588,8 +587,7 @@ class TestRosterImport(unittest.TestCase):
             import_config_pck(self._mlist, self._pckdict)
         except Import21Error, e:
             # check the message
-            self.assertTrue("xx_XX" in str(e))
-            self.assertTrue("[language.xx]" in str(e))
+            self.assertTrue("[language.xx_XX]" in str(e))
         else:
             self.fail("Import21Error was not raised")
 
-- 
cgit v1.3.1


From 000a025d0d649353908ac0290cf46c1479233489 Mon Sep 17 00:00:00 2001
From: Aurélien Bompard
Date: Fri, 11 Oct 2013 17:40:39 +0200
Subject: Don't change the mail_host when importing

---
 src/mailman/utilities/importer.py          | 1 -
 src/mailman/utilities/tests/test_import.py | 7 ++++---
 2 files changed, 4 insertions(+), 4 deletions(-)

(limited to 'src')

diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py
index 9eb0fdf35..03ac94870 100644
--- a/src/mailman/utilities/importer.py
+++ b/src/mailman/utilities/importer.py
@@ -175,7 +175,6 @@ TYPES = dict(
 
 # Attribute names in Mailman 2 which are renamed in Mailman 3.
 NAME_MAPPINGS = dict(
-    host_name='mail_host',
     include_list_post_header='allow_list_posts',
     real_name='display_name',
     last_post_time='last_post_at',
diff --git a/src/mailman/utilities/tests/test_import.py b/src/mailman/utilities/tests/test_import.py
index 23cf74b36..76c8174d5 100644
--- a/src/mailman/utilities/tests/test_import.py
+++ b/src/mailman/utilities/tests/test_import.py
@@ -77,11 +77,12 @@ class TestBasicImport(unittest.TestCase):
         self._import()
         self.assertEqual(self._mlist.display_name, 'Test')
 
-    def test_mail_host(self):
-        # The mlist.mail_host gets set.
+    def test_mail_host_invariant(self):
+        # The mlist.mail_host must not be updated when importing (it will
+        # change the list_id property, which is supposed to be read-only)
         self.assertEqual(self._mlist.mail_host, 'example.com')
         self._import()
-        self.assertEqual(self._mlist.mail_host, 'heresy.example.org')
+        self.assertEqual(self._mlist.mail_host, 'example.com')
 
     def test_rfc2369_headers(self):
         self._mlist.allow_list_posts = False
-- 
cgit v1.3.1


From 58b998361fa7985b8fbeb369ae9aff553a927132 Mon Sep 17 00:00:00 2001
From: Aurélien Bompard
Date: Fri, 11 Oct 2013 17:48:05 +0200
Subject: Handle 'local-part-only' acceptable aliases

---
 src/mailman/utilities/importer.py          |  6 +++++-
 src/mailman/utilities/tests/test_import.py | 18 +++++++++++++++++-
 2 files changed, 22 insertions(+), 2 deletions(-)

(limited to 'src')

diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py
index 03ac94870..68e0274fa 100644
--- a/src/mailman/utilities/importer.py
+++ b/src/mailman/utilities/importer.py
@@ -250,7 +250,11 @@ def import_config_pck(mlist, config_dict):
         addr = addr.strip()
         if not addr:
             continue
-        IAcceptableAliasSet(mlist).add(str_to_unicode(addr))
+        addr = str_to_unicode(addr)
+        try:
+            IAcceptableAliasSet(mlist).add(addr)
+        except ValueError:
+            IAcceptableAliasSet(mlist).add("^" + addr)
     # Handle conversion to URIs
     convert_to_uri = {
         "welcome_msg": "welcome_message_uri",
diff --git a/src/mailman/utilities/tests/test_import.py b/src/mailman/utilities/tests/test_import.py
index 76c8174d5..3a909e857 100644
--- a/src/mailman/utilities/tests/test_import.py
+++ b/src/mailman/utilities/tests/test_import.py
@@ -213,13 +213,29 @@ class TestBasicImport(unittest.TestCase):
         # it used to be a plain-text field (values are newline-separated)
         aliases = ["alias1@example.com",
                    "alias2@exemple.com",
-                   "non-ascii-\xe8@example.com"]
+                   "non-ascii-\xe8@example.com",
+                   ]
         self._pckdict[b"acceptable_aliases"] = \
                 ("\n".join(aliases)).encode("utf-8")
         self._import()
         alias_set = IAcceptableAliasSet(self._mlist)
         self.assertEqual(sorted(alias_set.aliases), aliases)
 
+    def test_acceptable_aliases_invalid(self):
+        # values without an '@' sign used to be matched against the local part,
+        # now we need to add the '^' sign
+        aliases = ["invalid-value", ]
+        self._pckdict[b"acceptable_aliases"] = \
+                ("\n".join(aliases)).encode("utf-8")
+        try:
+            self._import()
+        except ValueError, e:
+            print(format_exc())
+            self.fail("Invalid value '%s' caused a crash" % e)
+        alias_set = IAcceptableAliasSet(self._mlist)
+        self.assertEqual(sorted(alias_set.aliases),
+                         [ ("^" + a) for a in aliases ])
+
     def test_info_non_ascii(self):
         # info can contain non-ascii chars
         info = 'O idioma aceito \xe9 somente Portugu\xeas do Brasil'
-- 
cgit v1.3.1


From c56a83d5eb2ca8aa7e384e27825f2f34faeabebe Mon Sep 17 00:00:00 2001
From: Aurélien Bompard
Date: Fri, 11 Oct 2013 17:49:09 +0200
Subject: Better handling of mixed case email addresses

---
 src/mailman/utilities/importer.py          | 27 +++++++++-------
 src/mailman/utilities/tests/test_import.py | 50 +++++++++++++++++++++++++++++-
 2 files changed, 65 insertions(+), 12 deletions(-)

(limited to 'src')

diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py
index 68e0274fa..e97db12dc 100644
--- a/src/mailman/utilities/importer.py
+++ b/src/mailman/utilities/importer.py
@@ -350,24 +350,29 @@ def import_roster(mlist, config_dict, members, role):
     """
     usermanager = getUtility(IUserManager)
     for email in members:
-        email = str_to_unicode(email)
+        # for owners and members, the emails can have a mixed case, so
+        # lowercase them all
+        email = str_to_unicode(email).lower()
         roster = mlist.get_roster(role)
         if roster.get_member(email) is not None:
             print("%s is already imported with role %s" % (email, role),
                   file=sys.stderr)
             continue
+        address = usermanager.get_address(email)
         user = usermanager.get_user(email)
         if user is None:
-            merged_members = {}
-            merged_members.update(config_dict.get("members", {}))
-            merged_members.update(config_dict.get("digest_members", {}))
-            if merged_members.get(email, 0) != 0:
-                original_email = merged_members[email]
-            else:
-                original_email = email
-            user = usermanager.create_user(str_to_unicode(original_email))
-        address = usermanager.get_address(email)
-        address.verified_on = datetime.datetime.now()
+            user = usermanager.create_user()
+            if address is None:
+                merged_members = {}
+                merged_members.update(config_dict.get("members", {}))
+                merged_members.update(config_dict.get("digest_members", {}))
+                if merged_members.get(email, 0) != 0:
+                    original_email = str_to_unicode(merged_members[email])
+                else:
+                    original_email = email
+                address = usermanager.create_address(original_email)
+                address.verified_on = datetime.datetime.now()
+            user.link(address)
         mlist.subscribe(address, role)
         member = roster.get_member(email)
         assert member is not None
diff --git a/src/mailman/utilities/tests/test_import.py b/src/mailman/utilities/tests/test_import.py
index 3a909e857..a29c560dd 100644
--- a/src/mailman/utilities/tests/test_import.py
+++ b/src/mailman/utilities/tests/test_import.py
@@ -28,12 +28,14 @@ __all__ = [
 import cPickle
 import unittest
 from datetime import timedelta, datetime
+from traceback import format_exc
 
 from mailman.app.lifecycle import create_list, remove_list
 from mailman.testing.layers import ConfigLayer
 from mailman.utilities.importer import import_config_pck, Import21Error
 from mailman.interfaces.archiver import ArchivePolicy
 from mailman.interfaces.action import Action, FilterAction
+from mailman.interfaces.address import ExistingAddressError
 from mailman.interfaces.bounce import UnrecognizedBounceDisposition
 from mailman.interfaces.bans import IBanManager
 from mailman.interfaces.mailinglist import IAcceptableAliasSet
@@ -41,13 +43,15 @@ from mailman.interfaces.nntp import NewsgroupModeration
 from mailman.interfaces.autorespond import ResponseAction
 from mailman.interfaces.templates import ITemplateLoader
 from mailman.interfaces.usermanager import IUserManager
-from mailman.interfaces.member import DeliveryMode, DeliveryStatus
+from mailman.interfaces.member import DeliveryMode, DeliveryStatus, MemberRole
 from mailman.interfaces.languages import ILanguageManager
+from mailman.model.address import Address
 from mailman.handlers.decorate import decorate
 from mailman.utilities.string import expand
 from pkg_resources import resource_filename
 from enum import Enum
 from zope.component import getUtility
+from storm.locals import Store
 
 
 
@@ -205,6 +209,7 @@ class TestBasicImport(unittest.TestCase):
         try:
             self._import()
         except UnicodeDecodeError, e:
+            print(format_exc())
             self.fail(e)
         for _pattern, addr in banned:
             self.assertTrue(IBanManager(self._mlist).is_banned(addr))
@@ -502,6 +507,7 @@ class TestConvertToURI(unittest.TestCase):
         try:
             import_config_pck(self._mlist, self._pckdict)
         except UnicodeDecodeError, e:
+            print(format_exc())
             self.fail(e)
         for oldvar, newvar in self._conf_mapping.iteritems():
             newattr = getattr(self._mlist, newvar)
@@ -665,6 +671,48 @@ class TestRosterImport(unittest.TestCase):
         member = self._mlist.members.get_member('bob@example.com')
         self.assertEqual(member.user, user)
 
+    def test_owner_and_moderator_not_lowercase(self):
+        # In the v2.1 pickled dict, the owner and moderator lists are not
+        # necessarily lowercased already
+        self._pckdict[b"owner"] = [b"Anne@example.com"]
+        self._pckdict[b"moderator"] = [b"Anne@example.com"]
+        try:
+            import_config_pck(self._mlist, self._pckdict)
+        except AssertionError:
+            print(format_exc())
+            self.fail("The address was not lowercased")
+        self.assertTrue("anne@example.com" in
+                [ a.email for a in self._mlist.owners.addresses ])
+        self.assertTrue("anne@example.com" in
+                [ a.email for a in self._mlist.moderators.addresses])
+
+    def test_address_already_exists_but_no_user(self):
+        # An address already exists, but it is not linked to a user nor
+        # subscribed
+        anne_addr = Address("anne@example.com", "Anne")
+        Store.of(self._mlist).add(anne_addr)
+        try:
+            import_config_pck(self._mlist, self._pckdict)
+        except ExistingAddressError:
+            print(format_exc())
+            self.fail("existing address was not checked")
+        anne = self._usermanager.get_user("anne@example.com")
+        self.assertTrue(anne.controls("anne@example.com"))
+        self.assertTrue(anne_addr in self._mlist.regular_members.addresses)
+
+    def test_address_already_subscribed_but_no_user(self):
+        # An address is already subscribed, but it is not linked to a user
+        anne_addr = Address("anne@example.com", "Anne")
+        self._mlist.subscribe(anne_addr)
+        try:
+            import_config_pck(self._mlist, self._pckdict)
+        except ExistingAddressError:
+            print(format_exc())
+            self.fail("existing address was not checked")
+        anne = self._usermanager.get_user("anne@example.com")
+        self.assertTrue(anne.controls("anne@example.com"))
+
+
 
 
 class TestPreferencesImport(unittest.TestCase):
-- 
cgit v1.3.1


From e3f8ad0d49104964ab14a9dfd0d60253d7d808c3 Mon Sep 17 00:00:00 2001
From: Aurélien Bompard
Date: Tue, 15 Oct 2013 09:27:18 +0200
Subject: Handle acceptable_aliases being a list in the pickle

---
 src/mailman/utilities/importer.py          |  5 ++++-
 src/mailman/utilities/tests/test_import.py | 13 +++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

(limited to 'src')

diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py
index e97db12dc..5c77a45d8 100644
--- a/src/mailman/utilities/importer.py
+++ b/src/mailman/utilities/importer.py
@@ -246,7 +246,10 @@ def import_config_pck(mlist, config_dict):
     for addr in config_dict.get('ban_list', []):
         IBanManager(mlist).ban(str_to_unicode(addr))
     # Handle acceptable aliases
-    for addr in config_dict.get('acceptable_aliases', '').splitlines():
+    acceptable_aliases = config_dict.get('acceptable_aliases', '')
+    if isinstance(acceptable_aliases, basestring):
+        acceptable_aliases = acceptable_aliases.splitlines()
+    for addr in acceptable_aliases:
         addr = addr.strip()
         if not addr:
             continue
diff --git a/src/mailman/utilities/tests/test_import.py b/src/mailman/utilities/tests/test_import.py
index a29c560dd..bf757ba4b 100644
--- a/src/mailman/utilities/tests/test_import.py
+++ b/src/mailman/utilities/tests/test_import.py
@@ -241,6 +241,19 @@ class TestBasicImport(unittest.TestCase):
         self.assertEqual(sorted(alias_set.aliases),
                          [ ("^" + a) for a in aliases ])
 
+    def test_acceptable_aliases_as_list(self):
+        # in some versions of the pickle, it can be a list, not a string
+        # (seen in the wild)
+        aliases = [b"alias1@example.com", b"alias2@exemple.com" ]
+        self._pckdict[b"acceptable_aliases"] = aliases
+        try:
+            self._import()
+        except AttributeError:
+            print(format_exc())
+            self.fail("Import does not handle acceptable_aliases as list")
+        alias_set = IAcceptableAliasSet(self._mlist)
+        self.assertEqual(sorted(alias_set.aliases), aliases)
+
     def test_info_non_ascii(self):
         # info can contain non-ascii chars
         info = 'O idioma aceito \xe9 somente Portugu\xeas do Brasil'
-- 
cgit v1.3.1


From a2687bb2f937410a5e16185745e315e94fa03d9e Mon Sep 17 00:00:00 2001
From: Aurélien Bompard
Date: Mon, 21 Oct 2013 16:00:59 +0200
Subject: Text templates should be in UTF-8

---
 src/mailman/app/templates.py               |  2 +-
 src/mailman/app/tests/test_templates.py    | 11 +++++++++++
 src/mailman/utilities/tests/test_import.py | 21 +++++++++++++++++++--
 3 files changed, 31 insertions(+), 3 deletions(-)

(limited to 'src')

diff --git a/src/mailman/app/templates.py b/src/mailman/app/templates.py
index 707e7c256..f6b26811c 100644
--- a/src/mailman/app/templates.py
+++ b/src/mailman/app/templates.py
@@ -103,4 +103,4 @@ class TemplateLoader:
     def get(self, uri):
         """See `ITemplateLoader`."""
         with closing(urllib2.urlopen(uri)) as fp:
-            return fp.read()
+            return unicode(fp.read(), "utf-8")
diff --git a/src/mailman/app/tests/test_templates.py b/src/mailman/app/tests/test_templates.py
index 77a0eb381..634d00907 100644
--- a/src/mailman/app/tests/test_templates.py
+++ b/src/mailman/app/tests/test_templates.py
@@ -126,3 +126,14 @@ class TestTemplateLoader(unittest.TestCase):
         with self.assertRaises(urllib2.URLError) as cm:
             self._loader.get('mailman:///missing@example.com/en/foo/demo.txt')
         self.assertEqual(cm.exception.reason, 'No such file')
+
+    def test_non_ascii(self):
+        # mailman://demo.txt with non-ascii content
+        test_text = b'\xe4\xb8\xad'
+        path = os.path.join(self.var_dir, 'templates', 'site', 'it')
+        os.makedirs(path)
+        with open(os.path.join(path, 'demo.txt'), 'w') as fp:
+            print(test_text, end='', file=fp)
+        content = self._loader.get('mailman:///it/demo.txt')
+        self.assertTrue(isinstance(content, unicode))
+        self.assertEqual(content, test_text.decode("utf-8"))
diff --git a/src/mailman/utilities/tests/test_import.py b/src/mailman/utilities/tests/test_import.py
index bf757ba4b..f3e01c1a6 100644
--- a/src/mailman/utilities/tests/test_import.py
+++ b/src/mailman/utilities/tests/test_import.py
@@ -25,11 +25,13 @@ __all__ = [
     ]
 
 
+import os
 import cPickle
 import unittest
 from datetime import timedelta, datetime
 from traceback import format_exc
 
+from mailman.config import config
 from mailman.app.lifecycle import create_list, remove_list
 from mailman.testing.layers import ConfigLayer
 from mailman.utilities.importer import import_config_pck, Import21Error
@@ -515,6 +517,7 @@ class TestConvertToURI(unittest.TestCase):
                     "Default value was not preserved for %s" % newvar)
 
     def test_unicode(self):
+        # non-ascii templates
         for oldvar in self._conf_mapping:
             self._pckdict[str(oldvar)] = b"Ol\xe1!"
         try:
@@ -525,10 +528,24 @@ class TestConvertToURI(unittest.TestCase):
         for oldvar, newvar in self._conf_mapping.iteritems():
             newattr = getattr(self._mlist, newvar)
             text = decorate(self._mlist, newattr)
-            expected = u'Ol\ufffd!'.encode("utf-8")
-            # we get bytestrings because the text is stored in a file
+            expected = u'Ol\ufffd!'
             self.assertEqual(text, expected)
 
+    def test_unicode_in_default(self):
+        # What if the default template is already in UTF-8? (like if you import twice)
+        footer = b'\xe4\xb8\xad $listinfo_uri'
+        footer_path = os.path.join(config.VAR_DIR, "templates", "lists",
+                                   "blank@example.com", "en", "footer-generic.txt")
+        try:
+            os.makedirs(os.path.dirname(footer_path))
+        except OSError:
+            pass
+        with open(footer_path, "w") as footer_file:
+            footer_file.write(footer)
+        self._pckdict[b"msg_footer"] = b"NEW-VALUE"
+        import_config_pck(self._mlist, self._pckdict)
+        text = decorate(self._mlist, self._mlist.footer_uri)
+        self.assertEqual(text, 'NEW-VALUE')
 
 
 class TestRosterImport(unittest.TestCase):
-- 
cgit v1.3.1


From acc302099df53474e631117351f8116727c1ceb6 Mon Sep 17 00:00:00 2001
From: Aurélien Bompard
Date: Tue, 22 Oct 2013 15:33:55 +0200
Subject: Minor pyflakes fixes

---
 src/mailman/utilities/importer.py          | 4 ++--
 src/mailman/utilities/tests/test_import.py | 3 +--
 2 files changed, 3 insertions(+), 4 deletions(-)

(limited to 'src')

diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py
index 333cbd1bd..1c48f257f 100644
--- a/src/mailman/utilities/importer.py
+++ b/src/mailman/utilities/importer.py
@@ -22,7 +22,7 @@ from __future__ import absolute_import, print_function, unicode_literals
 __metaclass__ = type
 __all__ = [
     'import_config_pck',
-    'ImportError',
+    'Import21Error',
     ]
 
 
@@ -62,7 +62,7 @@ def str_to_unicode(value):
     for encoding in ("ascii", "utf-8"):
         try:
             return unicode(value, encoding)
-        except UnicodeDecodeError, e:
+        except UnicodeDecodeError:
             continue
     # we did our best, use replace
     return unicode(value, 'ascii', 'replace')
diff --git a/src/mailman/utilities/tests/test_import.py b/src/mailman/utilities/tests/test_import.py
index 34ff7eb3c..4abdd7dc0 100644
--- a/src/mailman/utilities/tests/test_import.py
+++ b/src/mailman/utilities/tests/test_import.py
@@ -34,7 +34,6 @@ from traceback import format_exc
 
 from mailman.config import config
 from mailman.app.lifecycle import create_list, remove_list
-from mailman.interfaces.archiver import ArchivePolicy
 from mailman.testing.layers import ConfigLayer
 from mailman.utilities.importer import import_config_pck, Import21Error
 from mailman.interfaces.archiver import ArchivePolicy
@@ -47,7 +46,7 @@ from mailman.interfaces.nntp import NewsgroupModeration
 from mailman.interfaces.autorespond import ResponseAction
 from mailman.interfaces.templates import ITemplateLoader
 from mailman.interfaces.usermanager import IUserManager
-from mailman.interfaces.member import DeliveryMode, DeliveryStatus, MemberRole
+from mailman.interfaces.member import DeliveryMode, DeliveryStatus
 from mailman.interfaces.languages import ILanguageManager
 from mailman.model.address import Address
 from mailman.handlers.decorate import decorate
-- 
cgit v1.3.1