14 files changed, 73 insertions, 43 deletions

diff --git a/src/mailman/app/bounces.py b/src/mailman/app/bounces.py
index 101d96f2a..b0a316ad6 100644
--- a/src/mailman/app/bounces.py
+++ b/src/mailman/app/bounces.py
@@ -200,10 +200,13 @@ def send_probe(member, msg):
                 optionsurl=member.options_url,
                 owneraddr=mlist.owner_address,
                 )
+    message_id = msg['message-id']
+    if isinstance(message_id, bytes):
+        message_id = message_id.decode('ascii')
     pendable = _ProbePendable(
         # We can only pend unicodes.
         member_id=member.member_id.hex,
-        message_id=msg['message-id'],
+        message_id=message_id,
         )
     token = getUtility(IPendings).add(pendable)
     mailbox, domain_parts = split_email(mlist.bounces_address)
diff --git a/src/mailman/app/moderator.py b/src/mailman/app/moderator.py
index 78332b84a..105e53617 100644
--- a/src/mailman/app/moderator.py
+++ b/src/mailman/app/moderator.py
@@ -86,9 +86,9 @@ def hold_message(mlist, msg, msgdata=None, reason=None):
     # Message-ID header.
     message_id = msg.get('message-id')
     if message_id is None:
-        msg['Message-ID'] = message_id = unicode(make_msgid())
-    assert isinstance(message_id, unicode), (
-        'Message-ID is not a unicode: %s' % message_id)
+        msg['Message-ID'] = message_id = make_msgid().decode('ascii')
+    elif isinstance(message_id, bytes):
+        message_id = message_id.decode('ascii')
     getUtility(IMessageStore).add(msg)
     # Prepare the message metadata with some extra information needed only by
     # the moderation interface.
diff --git a/src/mailman/archiving/mailarchive.py b/src/mailman/archiving/mailarchive.py
index 421d65cbd..c5fe5d0cb 100644
--- a/src/mailman/archiving/mailarchive.py
+++ b/src/mailman/archiving/mailarchive.py
@@ -68,6 +68,8 @@ class MailArchive:
         message_id_hash = msg.get('x-message-id-hash')
         if message_id_hash is None:
             return None
+        if isinstance(message_id_hash, bytes):
+            message_id_hash = message_id_hash.decode('ascii')
         return urljoin(self.base_url, message_id_hash)
 
     def archive_message(self, mlist, msg):
diff --git a/src/mailman/archiving/mhonarc.py b/src/mailman/archiving/mhonarc.py
index 4606925e0..f2d1f77fe 100644
--- a/src/mailman/archiving/mhonarc.py
+++ b/src/mailman/archiving/mhonarc.py
@@ -73,6 +73,8 @@ class MHonArc:
         message_id_hash = msg.get('x-message-id-hash')
         if message_id_hash is None:
             return None
+        if isinstance(message_id_hash, bytes):
+            message_id_hash = message_id_hash.decode('ascii')
         return urljoin(self.list_url(mlist), message_id_hash)
 
     def archive_message(self, mlist, msg):
diff --git a/src/mailman/archiving/prototype.py b/src/mailman/archiving/prototype.py
index 153c44b69..77b2294ed 100644
--- a/src/mailman/archiving/prototype.py
+++ b/src/mailman/archiving/prototype.py
@@ -68,6 +68,8 @@ class Prototype:
         message_id_hash = msg.get('x-message-id-hash')
         if message_id_hash is None:
             return None
+        if isinstance(message_id_hash, bytes):
+            message_id_hash = message_id_hash.decode('ascii')
         return urljoin(Prototype.list_url(mlist), message_id_hash)
 
     @staticmethod
diff --git a/src/mailman/commands/docs/unshunt.rst b/src/mailman/commands/docs/unshunt.rst
index 93d89f001..9532ae029 100644
--- a/src/mailman/commands/docs/unshunt.rst
+++ b/src/mailman/commands/docs/unshunt.rst
@@ -83,7 +83,7 @@ queue.
     2
 
     >>> sorted(item.msg['message-id'] for item in items)
-    [u'<badgers>', u'<crow>']
+    ['<badgers>', '<crow>']
 
 
 Return to the original queue
diff --git a/src/mailman/commands/eml_membership.py b/src/mailman/commands/eml_membership.py
index 617807783..c56b14041 100644
--- a/src/mailman/commands/eml_membership.py
+++ b/src/mailman/commands/eml_membership.py
@@ -72,6 +72,8 @@ used.
             print(_('$self.name: No valid address found to subscribe'),
                   file=results)
             return ContinueProcessing.no
+        if isinstance(address, bytes):
+            address = address.decode('ascii')
         # Have we already seen one join request from this user during the
         # processing of this email?
         joins = getattr(results, 'joins', set())
diff --git a/src/mailman/docs/NEWS.rst b/src/mailman/docs/NEWS.rst
index ca3a96214..9b37ca7e8 100644
--- a/src/mailman/docs/NEWS.rst
+++ b/src/mailman/docs/NEWS.rst
@@ -21,6 +21,8 @@ Bugs
    Given by Aurélien Bompard.  (LP: #1170347)
  * Fixed messages without a `text/plain` part crashing the `Approved` rule.
    Given by Aurélien Bompard.  (LP: #1158721)
+ * Fixed getting non-ASCII filenames from RFC 2231 i18n'd messages.  Given by
+   Aurélien Bompard.  (LP: #1060951)
 
 Commands
 --------
diff --git a/src/mailman/email/message.py b/src/mailman/email/message.py
index f3a44e63c..24c6ead9e 100644
--- a/src/mailman/email/message.py
+++ b/src/mailman/email/message.py
@@ -53,29 +53,6 @@ class Message(email.message.Message):
         self.__version__ = VERSION
         email.message.Message.__init__(self)
 
-    def __getitem__(self, key):
-        # Ensure that header values are unicodes.
-        value = email.message.Message.__getitem__(self, key)
-        if isinstance(value, str):
-            return unicode(value, 'ascii')
-        return value
-
-    def get(self, name, failobj=None):
-        # Ensure that header values are unicodes.
-        value = email.message.Message.get(self, name, failobj)
-        if isinstance(value, str):
-            return unicode(value, 'ascii')
-        return value
-
-    def get_all(self, name, failobj=None):
-        # Ensure all header values are unicodes.
-        missing = object()
-        all_values = email.message.Message.get_all(self, name, missing)
-        if all_values is missing:
-            return failobj
-        return [(unicode(value, 'ascii') if isinstance(value, str) else value)
-                for value in all_values]
-
     # BAW: For debugging w/ bin/dumpdb.  Apparently pprint uses repr.
     def __repr__(self):
         return self.__str__()
@@ -144,18 +121,15 @@ class Message(email.message.Message):
                 field_values = self.get_all(header, [])
                 senders.extend(address.lower() for (display_name, address)
                                in email.utils.getaddresses(field_values))
-        # Filter out None and the empty string.
-        return [sender for sender in senders if sender]
-
-    def get_filename(self, failobj=None):
-        """Some MUA have bugs in RFC2231 filename encoding and cause
-        Mailman to stop delivery in Scrubber.py (called from ToDigest.py).
-        """
-        try:
-            filename = email.message.Message.get_filename(self, failobj)
-            return filename
-        except (UnicodeError, LookupError, ValueError):
-            return failobj
+        # Filter out None and the empty string, and convert to unicode.
+        clean_senders = []
+        for sender in senders:
+            if not sender:
+                continue
+            if isinstance(sender, bytes):
+                sender = sender.decode('ascii')
+            clean_senders.append(sender)
+        return clean_senders
 
 
 
diff --git a/src/mailman/email/tests/test_message.py b/src/mailman/email/tests/test_message.py
index e281c0d06..1fdef5e86 100644
--- a/src/mailman/email/tests/test_message.py
+++ b/src/mailman/email/tests/test_message.py
@@ -22,13 +22,15 @@ from __future__ import absolute_import, print_function, unicode_literals
 __metaclass__ = type
 __all__ = [
     'TestMessage',
+    'TestMessageSubclass',
     ]
 
 
 import unittest
+from email.parser import FeedParser
 
 from mailman.app.lifecycle import create_list
-from mailman.email.message import UserNotification
+from mailman.email.message import Message, UserNotification
 from mailman.testing.helpers import get_queue_messages
 from mailman.testing.layers import ConfigLayer
 
@@ -56,5 +58,36 @@ class TestMessage(unittest.TestCase):
         self._msg.send(self._mlist)
         messages = get_queue_messages('virgin')
         self.assertEqual(len(messages), 1)
-        self.assertEqual(messages[0].msg.get_all('precedence'), 
+        self.assertEqual(messages[0].msg.get_all('precedence'),
                          ['omg wtf bbq'])
+
+
+
+class TestMessageSubclass(unittest.TestCase):
+    def test_i18n_filenames(self):
+        parser = FeedParser(_factory=Message)
+        parser.feed(b"""\
+Message-ID: <blah@example.com>
+Content-Type: multipart/mixed; boundary="------------050607040206050605060208"
+
+This is a multi-part message in MIME format.
+--------------050607040206050605060208
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: quoted-printable
+
+Test message containing an attachment with an accented filename
+
+--------------050607040206050605060208
+Content-Disposition: attachment;
+    filename*=UTF-8''d%C3%A9jeuner.txt
+
+Test content
+--------------050607040206050605060208--
+""")
+        msg = parser.close()
+        attachment = msg.get_payload(1)
+        try:
+            filename = attachment.get_filename()
+        except TypeError as e:
+            self.fail(e)
+        self.assertEqual(filename, u'd\xe9jeuner.txt')
diff --git a/src/mailman/model/bounce.py b/src/mailman/model/bounce.py
index cd658052d..26ebbe0c6 100644
--- a/src/mailman/model/bounce.py
+++ b/src/mailman/model/bounce.py
@@ -57,7 +57,10 @@ class BounceEvent(Model):
         self.list_id = list_id
         self.email = email
         self.timestamp = now()
-        self.message_id = msg['message-id']
+        msgid = msg['message-id']
+        if isinstance(msgid, bytes):
+            msgid = msgid.decode('ascii')
+        self.message_id = msgid
         self.context = (BounceContext.normal if context is None else context)
         self.processed = False
 
diff --git a/src/mailman/model/messagestore.py b/src/mailman/model/messagestore.py
index 19fa8133f..12b2aef46 100644
--- a/src/mailman/model/messagestore.py
+++ b/src/mailman/model/messagestore.py
@@ -24,6 +24,7 @@ __all__ = [
     'MessageStore',
     ]
 
+
 import os
 import errno
 import base64
@@ -58,6 +59,8 @@ class MessageStore:
             raise ValueError('Exactly one Message-ID header required')
         # Calculate and insert the X-Message-ID-Hash.
         message_id = message_ids[0]
+        if isinstance(message_id, bytes):
+            message_id = message_id.decode('ascii')
         # Complain if the Message-ID already exists in the storage.
         existing = store.query(Message).filter(
             Message.message_id == message_id).first()
diff --git a/src/mailman/rules/implicit_dest.py b/src/mailman/rules/implicit_dest.py
index 8bfb0d2e0..0bc229b15 100644
--- a/src/mailman/rules/implicit_dest.py
+++ b/src/mailman/rules/implicit_dest.py
@@ -73,6 +73,8 @@ class ImplicitDestination:
         recipients = set()
         for header in ('to', 'cc', 'resent-to', 'resent-cc'):
             for fullname, address in getaddresses(msg.get_all(header, [])):
+                if isinstance(address, bytes):
+                    address = address.decode('ascii')
                 address = address.lower()
                 if address in aliases:
                     return False
diff --git a/src/mailman/utilities/email.py b/src/mailman/utilities/email.py
index 7025ddb89..ea44ad0a4 100644
--- a/src/mailman/utilities/email.py
+++ b/src/mailman/utilities/email.py
@@ -62,6 +62,8 @@ def add_message_hash(msg):
     message_id = msg.get('message-id')
     if message_id is None:
         return
+    if isinstance(message_id, bytes):
+        message_id = message_id.decode('ascii')
     # The angle brackets are not part of the Message-ID.  See RFC 2822
     # and http://wiki.list.org/display/DEV/Stable+URLs
     if message_id.startswith('<') and message_id.endswith('>'):