8 files changed, 495 insertions, 19 deletions

diff --git a/src/mailman/docs/NEWS.rst b/src/mailman/docs/NEWS.rst
index 0dcba4450..3a2398b74 100644
--- a/src/mailman/docs/NEWS.rst
+++ b/src/mailman/docs/NEWS.rst
@@ -111,6 +111,8 @@ Bugs
    (Closes: #255)
  * Update documentation links for ``config.cfg`` settings.  (Closes: #306)
  * Disallow problematic characters in listnames.  (Closes: #311)
+ * Forward port several content filtering fixes from the 2.1 branch.
+   (Closes: #330, #331, #332 and #334)
 
 Configuration
 -------------
diff --git a/src/mailman/handlers/docs/filtering.rst b/src/mailman/handlers/docs/filtering.rst
index 427db4273..2d2480555 100644
--- a/src/mailman/handlers/docs/filtering.rst
+++ b/src/mailman/handlers/docs/filtering.rst
@@ -76,7 +76,8 @@ Simple multipart filtering
 ==========================
 
 If one of the subparts in a ``multipart`` message matches the filter type,
-then just that subpart will be stripped.
+then just that subpart will be stripped.  If that leaves just a single subpart,
+the ``multipart`` will be replaced by the subpart.
 ::
 
     >>> msg = message_from_string("""\
@@ -101,17 +102,11 @@ then just that subpart will be stripped.
     >>> process(mlist, msg, {})
     >>> print(msg.as_string())
     From: aperson@example.com
-    Content-Type: multipart/mixed; boundary=BOUNDARY
     MIME-Version: 1.0
-    X-Content-Filtered-By: Mailman/MimeDel ...
-    <BLANKLINE>
-    --BOUNDARY
     Content-Type: image/gif
-    MIME-Version: 1.0
+    X-Content-Filtered-By: Mailman/MimeDel ...
     <BLANKLINE>
     yyy
-    --BOUNDARY--
-    <BLANKLINE>
 
 
 Collapsing multipart/alternative messages
@@ -128,7 +123,8 @@ Content filtering will remove the jpeg part, leaving the
 ``multipart/alternative`` with only a single gif subpart.  Because there's
 only one subpart left, the MIME structure of the message will be reorganized,
 removing the inner ``multipart/alternative`` so that the outer
-``multipart/mixed`` has just a single gif subpart.
+``multipart/mixed`` has just a single gif subpart, and then the multipart is
+recast as just the subpart.
 
     >>> mlist.collapse_alternatives = True
     >>> msg = message_from_string("""\
@@ -158,17 +154,11 @@ removing the inner ``multipart/alternative`` so that the outer
     >>> process(mlist, msg, {})
     >>> print(msg.as_string())
     From: aperson@example.com
-    Content-Type: multipart/mixed; boundary=BOUNDARY
     MIME-Version: 1.0
-    X-Content-Filtered-By: Mailman/MimeDel ...
-    <BLANKLINE>
-    --BOUNDARY
     Content-Type: image/gif
-    MIME-Version: 1.0
+    X-Content-Filtered-By: Mailman/MimeDel ...
     <BLANKLINE>
     yyy
-    --BOUNDARY--
-    <BLANKLINE>
 
 When the outer part is a ``multipart/alternative`` and filtering leaves this
 outer part with just one subpart, the entire message is converted to the left
diff --git a/src/mailman/handlers/mime_delete.py b/src/mailman/handlers/mime_delete.py
index de1e92d1c..14aee1a0e 100644
--- a/src/mailman/handlers/mime_delete.py
+++ b/src/mailman/handlers/mime_delete.py
@@ -139,6 +139,11 @@ def process(mlist, msg, msgdata):
         if ctype == 'multipart/alternative':
             firstalt = msg.get_payload(0)
             reset_payload(msg, firstalt)
+    # Now that we've collapsed the MPA parts, go through the message
+    # and recast any multipart parts with only one sub-part as just
+    # the sub-part.
+    if msg.is_multipart():
+        recast_multipart(msg)
     # If we removed some parts, make note of this
     changedp = 0
     if numparts != len([subpart for subpart in msg.walk()]):
@@ -222,12 +227,43 @@ def collapse_multipart_alternatives(msg):
         if subpart.get_content_type() == 'multipart/alternative':
             with suppress(IndexError):
                 firstalt = subpart.get_payload(0)
-                newpayload.append(firstalt)
+                if msg.get_content_type() == 'message/rfc822':
+                    # This is a multipart/alternative message in a
+                    # message/rfc822 part. We treat it specially so as not to
+                    # lose the headers.
+                    reset_payload(subpart, firstalt)
+                    newpayload.append(subpart)
+                else:
+                    newpayload.append(firstalt)
+        elif subpart.is_multipart():
+            collapse_multipart_alternatives(subpart)
+            newpayload.append(subpart)
         else:
             newpayload.append(subpart)
     msg.set_payload(newpayload)
 
 
+def recast_multipart(msg):
+    # If we're left with a multipart message with only one sub-part, recast
+    # the message to just the sub-part, but not if the part is message/rfc822
+    # because we don't want to lose the headers.
+    # Also, if this is a multipart/signed part, stop now as the original part
+    # may have had a multipart sub-part with only one sub-sub-part, the sig
+    # may still be valid and going further may break it.  (LP: #1551075)
+    if msg.get_content_type() == 'multipart/signed':
+        return
+    if msg.is_multipart():
+        if (len(msg.get_payload()) == 1 and
+                msg.get_content_type() != 'message/rfc822'):
+            reset_payload(msg, msg.get_payload(0))
+            # now that we've recast this part, check the subordinate parts
+            recast_multipart(msg)
+        else:
+            # This part's OK but check deeper.
+            for part in msg.get_payload():
+                recast_multipart(part)
+
+
 def to_plaintext(msg):
     changedp = 0
     counter = count()
@@ -263,12 +299,12 @@ def get_file_ext(m):
     fext = ''
     filename = m.get_filename('') or m.get_param('name', '')
     if filename:
-        fext = os.path.splitext(oneline(filename, 'utf-8'))[1]
+        fext = os.path.splitext(oneline(filename, 'utf-8', in_unicode=True))[1]
         if len(fext) > 1:
             fext = fext[1:]
         else:
             fext = ''
-    return fext
+    return fext.lower()
 
 
 @public
diff --git a/src/mailman/handlers/tests/data/__init__.py b/src/mailman/handlers/tests/data/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/src/mailman/handlers/tests/data/__init__.py
diff --git a/src/mailman/handlers/tests/data/collapse_alternatives.eml b/src/mailman/handlers/tests/data/collapse_alternatives.eml
new file mode 100644
index 000000000..38ee3524e
--- /dev/null
+++ b/src/mailman/handlers/tests/data/collapse_alternatives.eml
@@ -0,0 +1,52 @@
+To: mailman-users@mailman3.org
+Message-ID: <5847c753-d3c5-23ac-edbd-c102999636a2@example.com>
+Date: Fri, 19 May 2017 12:58:25 +0200
+MIME-Version: 1.0
+Content-Type: multipart/signed; micalg=pgp-sha256;
+ protocol="application/pgp-signature";
+ boundary="vFaMjPs797GQgsvxAoMXW6m0wITFbm1Hh"
+Subject: Test multipart alternative
+From: user@example.com
+
+This is an OpenPGP/MIME signed message (RFC 4880 and 3156)
+--vFaMjPs797GQgsvxAoMXW6m0wITFbm1Hh
+Content-Type: multipart/mixed; boundary="hFtv7Gl3eCgubGHcAccJQEFmFcP6jKVWP";
+ protected-headers="v1"
+
+--hFtv7Gl3eCgubGHcAccJQEFmFcP6jKVWP
+Content-Type: multipart/alternative;
+ boundary="------------826A33926CBE213C2D92E2EE"
+
+This is a multi-part message in MIME format.
+--------------826A33926CBE213C2D92E2EE
+Content-Type: text/plain; charset=utf-8
+
+This is the plain text
+
+--------------826A33926CBE213C2D92E2EE
+Content-Type: text/html; charset="utf-8"
+
+<html><head></head><body>
+This is the html.
+</body></html>
+
+--------------826A33926CBE213C2D92E2EE--
+
+--hFtv7Gl3eCgubGHcAccJQEFmFcP6jKVWP
+Content-Type: text/plain; charset="utf-8"
+
+This is just another part
+
+--hFtv7Gl3eCgubGHcAccJQEFmFcP6jKVWP--
+
+--vFaMjPs797GQgsvxAoMXW6m0wITFbm1Hh
+Content-Type: application/pgp-signature; name="signature.asc"
+Content-Description: OpenPGP digital signature
+Content-Disposition: attachment; filename="signature.asc"
+
+-----BEGIN PGP SIGNATURE-----
+
+some data
+-----END PGP SIGNATURE-----
+
+--vFaMjPs797GQgsvxAoMXW6m0wITFbm1Hh--
diff --git a/src/mailman/handlers/tests/data/msg_rfc822.eml b/src/mailman/handlers/tests/data/msg_rfc822.eml
new file mode 100644
index 000000000..dda65095c
--- /dev/null
+++ b/src/mailman/handlers/tests/data/msg_rfc822.eml
@@ -0,0 +1,183 @@
+Message-ID: <4D9E6AEA.1060802@example.net>
+Date: Thu, 07 Apr 2011 18:54:50 -0700
+From: User <user@example.com>
+MIME-Version: 1.0
+To: Someone <someone@example.net>
+Subject: Message Subject
+Content-Type: multipart/mixed;
+ boundary="------------050603050603060608020908"
+
+This is a multi-part message in MIME format.
+--------------050603050603060608020908
+Content-Type: text/plain; charset=ISO-8859-1
+Content-Transfer-Encoding: 7bit
+
+Plain body.
+
+--------------050603050603060608020908
+Content-Type: message/rfc822
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment
+
+Message-ID: <4D9E647F.4050308@example.net>
+Date: Thu, 07 Apr 2011 18:27:27 -0700
+From: User1 <user1@example.com>
+MIME-Version: 1.0
+To: Someone1 <someone1@example.net>
+Content-Type: multipart/mixed; boundary="------------060107040402070208020705"
+Subject: Attached Message 1 Subject
+
+This is a multi-part message in MIME format.
+--------------060107040402070208020705
+Content-Type: text/plain; charset=ISO-8859-1
+Content-Transfer-Encoding: 7bit
+
+Attached Message 1 body.
+
+--------------060107040402070208020705
+Content-Type: message/rfc822
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment
+
+From: User2 <user2@example.com>
+To: Someone2 <someone2@example.net>
+Subject: Attached Message 2 Subject
+Date: Thu, 7 Apr 2011 19:09:35 -0500
+Message-ID: <DAE689E1FD1D493BACD15180145B4151@example.net>
+MIME-Version: 1.0
+Content-Type: multipart/mixed;
+	boundary="----=_NextPart_000_0066_01CBF557.56C6F370"
+
+This is a multi-part message in MIME format.
+
+------=_NextPart_000_0066_01CBF557.56C6F370
+Content-Type: text/plain;
+	charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+
+Attached Message 2 body.
+
+------=_NextPart_000_0066_01CBF557.56C6F370
+Content-Type: message/rfc822
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment
+
+From: User3 <user3@example.com>
+To: Someone3 <someone3@example.net>
+Subject: Attached Message 3 Subject
+Date: Thu, 7 Apr 2011 17:22:04 -0500
+Message-ID: <BANLkTi=SzfNJo-V7cvrg3nE3uOi9uxXv3g@example.net>
+MIME-Version: 1.0
+Content-Type: multipart/alternative;
+	boundary="----=_NextPart_000_0058_01CBF557.56C48270"
+
+This is a multi-part message in MIME format.
+
+------=_NextPart_000_0058_01CBF557.56C48270
+Content-Type: text/plain;
+	charset="iso-8859-1"
+Content-Transfer-Encoding: 7bit
+
+Attached Message 3 plain body.
+
+------=_NextPart_000_0058_01CBF557.56C48270
+Content-Type: text/html;
+	charset="iso-8859-1"
+Content-Transfer-Encoding: quoted-printable
+
+
+Attached Message 3 html body.
+
+------=_NextPart_000_0058_01CBF557.56C48270--
+
+------=_NextPart_000_0066_01CBF557.56C6F370
+Content-Type: message/rfc822
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment
+
+From: User4 <user4@example.com>
+To: Someone4 <someone4@example.net>
+Subject: Attached Message 4 Subject
+Date: Thu, 7 Apr 2011 17:24:26 -0500
+Message-ID: <19CC3BDF28CF49AD988FF43B2DBC5F1D@example>
+MIME-Version: 1.0
+Content-Type: multipart/mixed;
+	boundary="----=_NextPart_000_0060_01CBF557.56C6F370"
+
+This is a multi-part message in MIME format.
+
+------=_NextPart_000_0060_01CBF557.56C6F370
+Content-Type: multipart/alternative;
+	boundary="----=_NextPart_001_0061_01CBF557.56C6F370"
+
+------=_NextPart_001_0061_01CBF557.56C6F370
+Content-Type: text/plain;
+	charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+
+Attached Message 4 plain body.
+
+------=_NextPart_001_0061_01CBF557.56C6F370
+Content-Type: text/html;
+	charset="us-ascii"
+Content-Transfer-Encoding: quoted-printable
+
+Attached Message 4 html body.
+
+------=_NextPart_001_0061_01CBF557.56C6F370--
+
+------=_NextPart_000_0060_01CBF557.56C6F370
+Content-Type: message/rfc822
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment
+
+From: User5 <user5@example.com>
+To: Someone5 <someone5@example.net>
+Subject: Attached Message 5 Subject
+Date: Thu, 7 Apr 2011 16:24:26 -0500
+Message-ID: <some_id@example>
+Content-Type: multipart/alternative;
+	boundary="----=_NextPart_000_005C_01CBF557.56C6F370"
+
+This is a multi-part message in MIME format.
+
+------=_NextPart_000_005C_01CBF557.56C6F370
+Content-Type: text/plain;
+	charset="iso-8859-1"
+Content-Transfer-Encoding: 7bit
+
+Attached Message 5 plain body.
+
+------=_NextPart_000_005C_01CBF557.56C6F370
+Content-Type: text/html;
+	charset="iso-8859-1"
+Content-Transfer-Encoding: quoted-printable
+
+Attached Message 5 html body.
+
+------=_NextPart_000_005C_01CBF557.56C6F370--
+
+------=_NextPart_000_0060_01CBF557.56C6F370
+Content-Type: text/plain;
+	name="ATT00055.txt"
+Content-Transfer-Encoding: quoted-printable
+Content-Disposition: attachment;
+	filename="ATT00055.txt"
+
+Another plain part.
+
+------=_NextPart_000_0060_01CBF557.56C6F370--
+
+------=_NextPart_000_0066_01CBF557.56C6F370--
+
+--------------060107040402070208020705
+Content-Type: text/plain; charset="us-ascii"
+MIME-Version: 1.0
+Content-Transfer-Encoding: 7bit
+Content-Disposition: inline
+
+Final plain part.
+
+--------------060107040402070208020705--
+
+--------------050603050603060608020908--
diff --git a/src/mailman/handlers/tests/data/msg_rfc822_out.eml b/src/mailman/handlers/tests/data/msg_rfc822_out.eml
new file mode 100644
index 000000000..a0f7c632d
--- /dev/null
+++ b/src/mailman/handlers/tests/data/msg_rfc822_out.eml
@@ -0,0 +1,140 @@
+Message-ID: <4D9E6AEA.1060802@example.net>
+Date: Thu, 07 Apr 2011 18:54:50 -0700
+From: User <user@example.com>
+MIME-Version: 1.0
+To: Someone <someone@example.net>
+Subject: Message Subject
+Content-Type: multipart/mixed;
+ boundary="------------050603050603060608020908"
+X-Content-Filtered-By: Mailman/MimeDel 3.1.0b5
+
+This is a multi-part message in MIME format.
+--------------050603050603060608020908
+Content-Type: text/plain; charset=ISO-8859-1
+Content-Transfer-Encoding: 7bit
+
+Plain body.
+
+--------------050603050603060608020908
+Content-Type: message/rfc822
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment
+
+Message-ID: <4D9E647F.4050308@example.net>
+Date: Thu, 07 Apr 2011 18:27:27 -0700
+From: User1 <user1@example.com>
+MIME-Version: 1.0
+To: Someone1 <someone1@example.net>
+Content-Type: multipart/mixed; boundary="------------060107040402070208020705"
+Subject: Attached Message 1 Subject
+
+This is a multi-part message in MIME format.
+--------------060107040402070208020705
+Content-Type: text/plain; charset=ISO-8859-1
+Content-Transfer-Encoding: 7bit
+
+Attached Message 1 body.
+
+--------------060107040402070208020705
+Content-Type: message/rfc822
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment
+
+From: User2 <user2@example.com>
+To: Someone2 <someone2@example.net>
+Subject: Attached Message 2 Subject
+Date: Thu, 7 Apr 2011 19:09:35 -0500
+Message-ID: <DAE689E1FD1D493BACD15180145B4151@example.net>
+MIME-Version: 1.0
+Content-Type: multipart/mixed;
+	boundary="----=_NextPart_000_0066_01CBF557.56C6F370"
+
+This is a multi-part message in MIME format.
+
+------=_NextPart_000_0066_01CBF557.56C6F370
+Content-Type: text/plain;
+	charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+
+Attached Message 2 body.
+
+------=_NextPart_000_0066_01CBF557.56C6F370
+Content-Type: message/rfc822
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment
+
+From: User3 <user3@example.com>
+To: Someone3 <someone3@example.net>
+Subject: Attached Message 3 Subject
+Date: Thu, 7 Apr 2011 17:22:04 -0500
+Message-ID: <BANLkTi=SzfNJo-V7cvrg3nE3uOi9uxXv3g@example.net>
+MIME-Version: 1.0
+Content-Type: text/plain;
+	charset="iso-8859-1"
+Content-Transfer-Encoding: 7bit
+
+Attached Message 3 plain body.
+
+------=_NextPart_000_0066_01CBF557.56C6F370
+Content-Type: message/rfc822
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment
+
+From: User4 <user4@example.com>
+To: Someone4 <someone4@example.net>
+Subject: Attached Message 4 Subject
+Date: Thu, 7 Apr 2011 17:24:26 -0500
+Message-ID: <19CC3BDF28CF49AD988FF43B2DBC5F1D@example>
+MIME-Version: 1.0
+Content-Type: multipart/mixed;
+	boundary="----=_NextPart_000_0060_01CBF557.56C6F370"
+
+This is a multi-part message in MIME format.
+
+------=_NextPart_000_0060_01CBF557.56C6F370
+Content-Type: text/plain;
+	charset="us-ascii"
+Content-Transfer-Encoding: 7bit
+
+Attached Message 4 plain body.
+
+------=_NextPart_000_0060_01CBF557.56C6F370
+Content-Type: message/rfc822
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment
+
+From: User5 <user5@example.com>
+To: Someone5 <someone5@example.net>
+Subject: Attached Message 5 Subject
+Date: Thu, 7 Apr 2011 16:24:26 -0500
+Message-ID: <some_id@example>
+Content-Type: text/plain;
+	charset="iso-8859-1"
+Content-Transfer-Encoding: 7bit
+
+Attached Message 5 plain body.
+
+------=_NextPart_000_0060_01CBF557.56C6F370
+Content-Type: text/plain;
+	name="ATT00055.txt"
+Content-Transfer-Encoding: quoted-printable
+Content-Disposition: attachment;
+	filename="ATT00055.txt"
+
+Another plain part.
+
+------=_NextPart_000_0060_01CBF557.56C6F370--
+
+------=_NextPart_000_0066_01CBF557.56C6F370--
+
+--------------060107040402070208020705
+Content-Type: text/plain; charset="us-ascii"
+MIME-Version: 1.0
+Content-Transfer-Encoding: 7bit
+Content-Disposition: inline
+
+Final plain part.
+
+--------------060107040402070208020705--
+
+--------------050603050603060608020908--
diff --git a/src/mailman/handlers/tests/test_mimedel.py b/src/mailman/handlers/tests/test_mimedel.py
index 5e536134f..01cac0f20 100644
--- a/src/mailman/handlers/tests/test_mimedel.py
+++ b/src/mailman/handlers/tests/test_mimedel.py
@@ -19,11 +19,13 @@
 
 import os
 import sys
+import email
 import shutil
 import tempfile
 import unittest
 
 from contextlib import ExitStack, contextmanager
+from io import StringIO
 from mailman.app.lifecycle import create_list
 from mailman.config import config
 from mailman.handlers import mime_delete
@@ -35,6 +37,7 @@ from mailman.testing.helpers import (
     LogFileMark, configuration, get_queue_messages,
     specialized_message_from_string as mfs)
 from mailman.testing.layers import ConfigLayer
+from pkg_resources import resource_filename
 from zope.component import getUtility
 
 
@@ -215,3 +218,73 @@ MIME-Version: 1.0
             msg['x-content-filtered-by'].startswith('Mailman/MimeDel'))
         payload_lines = msg.get_payload().splitlines()
         self.assertEqual(payload_lines[0], 'Converted text/html to text/plain')
+
+
+class TestMiscellaneous(unittest.TestCase):
+    """Test various miscellaneous filtering actions."""
+
+    layer = ConfigLayer
+
+    def setUp(self):
+        self._mlist = create_list('test@example.com')
+        self._mlist.collapse_alternatives = True
+        self._mlist.filter_content = True
+        self._mlist.filter_extensions = ['xlsx']
+
+    def test_collapse_alternatives(self):
+        email_file = resource_filename(
+            'mailman.handlers.tests.data', 'collapse_alternatives.eml')
+        with open(email_file) as fp:
+            msg = email.message_from_file(fp)
+        process = config.handlers['mime-delete'].process
+        process(self._mlist, msg, {})
+        structure = StringIO()
+        email.iterators._structure(msg, fp=structure)
+        self.assertEqual(structure.getvalue(), """\
+multipart/signed
+    multipart/mixed
+        text/plain
+        text/plain
+    application/pgp-signature
+""")
+
+    def test_msg_rfc822(self):
+        email_file = resource_filename(
+            'mailman.handlers.tests.data', 'msg_rfc822.eml')
+        email_file2 = resource_filename(
+            'mailman.handlers.tests.data', 'msg_rfc822_out.eml')
+        with open(email_file) as fp:
+            msg = email.message_from_file(fp)
+        process = config.handlers['mime-delete'].process
+        process(self._mlist, msg, {})
+        with open(email_file2) as fp:
+            self.assertEqual(msg.as_string(), fp.read())
+
+    def test_mixed_case_ext_and_recast(self):
+        msg = mfs("""\
+From: anne@example.com
+To: test@example.com
+Subject: Testing mixed extension
+Message-ID: <ant>
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="AAAA"
+
+--AAAA
+Content-Type: text/plain; charset="utf-8"
+
+Plain text
+
+--AAAA
+Content-Type: application/octet-stream; name="test.xlsX"
+Content-Disposition: attachment; filename="test.xlsX"
+
+spreadsheet
+
+--AAAA--
+""")
+        process = config.handlers['mime-delete'].process
+        process(self._mlist, msg, {})
+        self.assertEqual(msg['content-type'], 'text/plain; charset="utf-8"')
+        self.assertEqual(msg.get_payload(), """\
+Plain text
+""")