10 files changed, 64 insertions, 50 deletions

diff --git a/Mailman/Cgi/Auth.py b/Mailman/Cgi/Auth.py
index 03b61b1ce..37bb81b13 100644
--- a/Mailman/Cgi/Auth.py
+++ b/Mailman/Cgi/Auth.py
@@ -56,5 +56,5 @@ def loginpage(mlist, scriptname, msg='', frontpage=False):
          'path'    : actionurl,
          'message' : msg,
          'who'     : who,
-         }, mlist=mlist)
-    print mlist.GetMailmanFooter()
+         }, mlist=mlist).encode(charset)
+    print mlist.GetMailmanFooter().encode(charset)
diff --git a/Mailman/Cgi/admin.py b/Mailman/Cgi/admin.py
index 5cbf1affc..ae9850fba 100644
--- a/Mailman/Cgi/admin.py
+++ b/Mailman/Cgi/admin.py
@@ -497,7 +497,7 @@ def show_variables(mlist, category, subcat, cgidata, doc):
     # The very first item in the config info will be treated as a general
     # description if it is a string
     description = options[0]
-    if isinstance(description, str):
+    if isinstance(description, basestring):
         table.AddRow([description])
         table.AddCellInfo(table.GetCurrentRowIndex(), 0, colspan=2)
         options = options[1:]
@@ -514,7 +514,7 @@ def show_variables(mlist, category, subcat, cgidata, doc):
                       width='85%')
 
     for item in options:
-        if isinstance(item, str):
+        if isinstance(item, basestring):
             # The very first banner option (string in an options list) is
             # treated as a general description, while any others are
             # treated as section headers - centered and italicized...
@@ -937,8 +937,7 @@ def membership_options(mlist, subcat, cgidata, doc, form):
     for addr in members:
         link = Link(mlist.GetOptionsURL(addr, obscure=1),
                     mlist.getMemberCPAddress(addr))
-        fullname = Utils.uncanonstr(mlist.getMemberName(addr),
-                                    mlist.preferred_language)
+        fullname = mlist.getMemberName(addr)
         name = TextBox(addr + '_realname', fullname, size=longest).Format()
         cells = [Center(CheckBox(addr + '_unsub', 'off', 0).Format()),
                  link.Format() + '<br>' +
diff --git a/Mailman/Gui/GUIBase.py b/Mailman/Gui/GUIBase.py
index e12ac5984..3bb558dce 100644
--- a/Mailman/Gui/GUIBase.py
+++ b/Mailman/Gui/GUIBase.py
@@ -42,8 +42,9 @@ class GUIBase:
             # Let ValueErrors propagate
             return int(val)
         # String and Text widgets both just return their values verbatim
+        # but convert into unicode (for 2.2)
         if wtype in (Defaults.String, Defaults.Text):
-            return val
+            return unicode(val, Utils.GetCharSet(mlist.preferred_language))
         # This widget contains a single email address
         if wtype == Defaults.Email:
             # BAW: We must allow blank values otherwise reply_to_address can't
diff --git a/Mailman/HTMLFormatter.py b/Mailman/HTMLFormatter.py
index 2283f84ed..ab4c03a32 100644
--- a/Mailman/HTMLFormatter.py
+++ b/Mailman/HTMLFormatter.py
@@ -361,8 +361,8 @@ class HTMLFormatter:
             tag = parts[i].lower()
             if replacements.has_key(tag):
                 repl = replacements[tag]
-                if isinstance(repl, type(u'')):
-                    repl = repl.encode(charset, 'replace')
+                if isinstance(repl, str):
+                    repl = unicode(repl, charset, 'replace')
                 parts[i] = repl
             else:
                 parts[i] = ''
diff --git a/Mailman/Handlers/Decorate.py b/Mailman/Handlers/Decorate.py
index ee1a630ba..a5712947c 100644
--- a/Mailman/Handlers/Decorate.py
+++ b/Mailman/Handlers/Decorate.py
@@ -50,11 +50,11 @@ def process(mlist, msg, msgdata):
             d['user_password'] = mlist.getMemberPassword(member)
             d['user_language'] = mlist.getMemberLanguage(member)
             username = mlist.getMemberName(member) or None
-            try:
-                username = username.encode(Utils.GetCharSet(d['user_language']))
-            except (AttributeError, UnicodeError):
-                username = member
-            d['user_name'] = username
+            #try:
+            #    username = username.encode(Utils.GetCharSet(d['user_language']))
+            #except (AttributeError, UnicodeError):
+            #    username = member
+            d['user_name'] = username or d['user_delivered_to']
             d['user_optionsurl'] = mlist.GetOptionsURL(member)
         except Errors.NotAMemberError:
             pass
@@ -88,11 +88,9 @@ def process(mlist, msg, msgdata):
     # MIME multipart chroming the message?
     wrap = True
     if not msg.is_multipart() and msgtype == 'text/plain':
-        # TK: Try to keep the message plain by converting the header/
-        # footer/oldpayload into unicode and encode with mcset/lcset.
-        # Try to decode qp/base64 also.
-        uheader = unicode(header, lcset, 'ignore')
-        ufooter = unicode(footer, lcset, 'ignore')
+        # header/footer is now in unicode (2.2)
+        uheader = header
+        ufooter = footer
         try:
             oldpayload = unicode(msg.get_payload(decode=True), mcset)
             frontsep = endsep = u''
@@ -101,20 +99,23 @@ def process(mlist, msg, msgdata):
             if footer and not oldpayload.endswith('\n'):
                 endsep = u'\n'
             payload = uheader + frontsep + oldpayload + endsep + ufooter
-            try:
-                # first, try encode with list charset
-                payload = payload.encode(lcset)
-                newcset = lcset
-            except UnicodeError:
-                if lcset != mcset:
-                    # if fail, encode with message charset (if different)
-                    payload = payload.encode(mcset)
-                    newcset = mcset
-                    # if this fails, fallback to outer try and wrap=true
-            del msg['content-transfer-encoding']
-            del msg['content-type']
-            msg.set_payload(payload, newcset)
-            wrap = False
+            # Try to set message in list charset then message charset.
+            # Fall back to 'utf-8' if both doesn't work.
+            csets = [lcset,]
+            if mcset != lcset:
+                csets.append(mcset)
+            if 'utf-8' not in csets:
+                csets.append('utf-8')
+            for cset in csets:
+                try:
+                    payload = payload.encode(cset)
+                    del msg['content-transfer-encoding']
+                    del msg['content-type']
+                    msg.set_payload(payload, cset)
+                    wrap = False
+                    break
+                except UnicodeError:
+                    continue
         except (LookupError, UnicodeError):
             pass
     elif msg.get_content_type() == 'multipart/mixed':
@@ -124,11 +125,11 @@ def process(mlist, msg, msgdata):
         if not isinstance(payload, list):
             payload = [payload]
         if footer:
-            mimeftr = MIMEText(footer, 'plain', lcset)
+            mimeftr = MIMEText(footer.encode(lcset), 'plain', lcset)
             mimeftr['Content-Disposition'] = 'inline'
             payload.append(mimeftr)
         if header:
-            mimehdr = MIMEText(header, 'plain', lcset)
+            mimehdr = MIMEText(header.encode(lcset), 'plain', lcset)
             mimehdr['Content-Disposition'] = 'inline'
             payload.insert(0, mimehdr)
         msg.set_payload(payload)
@@ -166,11 +167,11 @@ def process(mlist, msg, msgdata):
     # any).
     payload = [inner]
     if header:
-        mimehdr = MIMEText(header, 'plain', lcset)
+        mimehdr = MIMEText(header.encode(lcset), 'plain', lcset)
         mimehdr['Content-Disposition'] = 'inline'
         payload.insert(0, mimehdr)
     if footer:
-        mimeftr = MIMEText(footer, 'plain', lcset)
+        mimeftr = MIMEText(footer.encode(lcset), 'plain', lcset)
         mimeftr['Content-Disposition'] = 'inline'
         payload.append(mimeftr)
     msg.set_payload(payload)
diff --git a/Mailman/Message.py b/Mailman/Message.py
index cdeb8c383..f79990837 100644
--- a/Mailman/Message.py
+++ b/Mailman/Message.py
@@ -207,13 +207,14 @@ class UserNotification(Message):
         Message.__init__(self)
         charset = None
         if lang is not None:
-            charset = Charset(Utils.GetCharSet(lang))
+            csetstr = Utils.GetCharSet(lang)
+            charset = Charset(csetstr)
         if text is not None:
-            self.set_payload(text, charset)
+            self.set_payload(text.encode(csetstr), charset)
         if subject is None:
             subject = '(no subject)'
-        self['Subject'] = Header(subject, charset, header_name='Subject',
-                                 errors='replace')
+        self['Subject'] = Header(subject.encode(csetstr), charset,
+                                 header_name='Subject', errors='replace')
         self['From'] = sender
         if isinstance(recip, list):
             self['To'] = COMMASPACE.join(recip)
diff --git a/Mailman/SafeDict.py b/Mailman/SafeDict.py
index ff702bca3..876ab88e4 100644
--- a/Mailman/SafeDict.py
+++ b/Mailman/SafeDict.py
@@ -15,6 +15,8 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
 # USA.
 
+from Mailman.configuration import config
+
 """A `safe' dictionary for string interpolation."""
 
 COMMASPACE = ', '
@@ -28,6 +30,15 @@ class SafeDict(dict):
 
     This is used in maketext so that editing templates is a bit more robust.
     """
+    def __init__(self, d='', charset=None, lang=None):
+        super(SafeDict, self).__init__(d)
+        if charset:
+            self.cset = charset
+        elif lang:
+            self.cset = config.LC_DESCRIPTIONS[lang][1]
+        else:
+            self.cset = 'us-ascii'
+
     def __getitem__(self, key):
         try:
             return super(SafeDict, self).__getitem__(key)
@@ -38,6 +49,9 @@ class SafeDict(dict):
                 return '<Missing key: %s>' % `key`
 
     def interpolate(self, template):
+        for k, v in self.items():
+            if isinstance(v, str):
+                self.__setitem__(k, unicode(v, self.cset))
         return template % self
 
 
diff --git a/Mailman/Utils.py b/Mailman/Utils.py
index 94a84532f..5844d708c 100644
--- a/Mailman/Utils.py
+++ b/Mailman/Utils.py
@@ -503,16 +503,12 @@ def findtext(templatefile, dict=None, raw=False, lang=None, mlist=None):
             raise IOError(errno.ENOENT, 'No template file found', templatefile)
     template = fp.read()
     fp.close()
+    template = unicode(template, GetCharSet(lang), 'replace')
     text = template
     if dict is not None:
         try:
-            sdict = SafeDict(dict)
-            try:
-                text = sdict.interpolate(template)
-            except UnicodeError:
-                # Try again after coercing the template to unicode
-                utemplate = unicode(template, GetCharSet(lang), 'replace')
-                text = sdict.interpolate(utemplate)
+            sdict = SafeDict(dict, lang=lang)
+            text = sdict.interpolate(template)
         except (TypeError, ValueError), e:
             # The template is really screwed up
             log.error('broken template: %s\n%s', filename, e)
diff --git a/Mailman/htmlformat.py b/Mailman/htmlformat.py
index cd20c286c..22d6922b9 100644
--- a/Mailman/htmlformat.py
+++ b/Mailman/htmlformat.py
@@ -325,7 +325,7 @@ class Document(Container):
         if not self.suppress_head:
             output.append('%s</BODY>' % tab)
             output.append('%s</HTML>' % tab)
-        return NL.join(output)
+        return NL.join(output).encode(charset, 'replace')
 
     def addError(self, errmsg, tag=None):
         if tag is None:
diff --git a/Mailman/i18n.py b/Mailman/i18n.py
index d64560803..59d10db25 100644
--- a/Mailman/i18n.py
+++ b/Mailman/i18n.py
@@ -112,7 +112,9 @@ def _(s):
     # Are we using $-strings or %-strings?
     if use_templates:
         return Template(tns).safe_substitute(attrdict(d))
-    return tns % SafeDict(d)
+    if type(tns) == str:
+        tns = unicode(tns, charset)
+    return SafeDict(d, charset=charset).interpolate(tns)