8 files changed, 113 insertions, 7 deletions

diff --git a/Mailman/Cgi/options.py b/Mailman/Cgi/options.py
index 9fb008e6a..bfc63920e 100644
--- a/Mailman/Cgi/options.py
+++ b/Mailman/Cgi/options.py
@@ -33,6 +33,7 @@ from Mailman import Utils
 
 from Mailman.htmlformat import *
 
+OR = '|'
 SLASH = '/'
 SETLANGUAGE = -1
 
@@ -1021,7 +1022,8 @@ def topic_details(mlist, doc, user, cpuser, userlang, varhelp):
     table.AddRow([Bold(Label(_('Name:'))),
                   Utils.websafe(name)])
     table.AddRow([Bold(Label(_('Pattern (as regexp):'))),
-                  '<pre>' + Utils.websafe(pattern) + '</pre>'])
+                  '<pre>' + Utils.websafe(OR.join(pattern.splitlines()))
+                   + '</pre>'])
     table.AddRow([Bold(Label(_('Description:'))),
                   Utils.websafe(description)])
     # Make colors look nice
diff --git a/Mailman/Gui/Topics.py b/Mailman/Gui/Topics.py
index 282930b7c..147b2de04 100644
--- a/Mailman/Gui/Topics.py
+++ b/Mailman/Gui/Topics.py
@@ -22,6 +22,8 @@ from Mailman import Utils
 from Mailman.i18n import _
 from Mailman.Gui.GUIBase import GUIBase
 
+OR = '|'
+
 
 
 class Topics(GUIBase):
@@ -119,9 +121,10 @@ class Topics(GUIBase):
             # Make sure the pattern was a legal regular expression
             name = Utils.websafe(name)
             try:
-                re.compile(pattern)
+                orpattern = OR.join(pattern.splitlines())
+                re.compile(orpattern)
             except (re.error, TypeError):
-                safepattern = Utils.websafe(pattern)
+                safepattern = Utils.websafe(orpattern)
                 doc.addError(_("""The topic pattern '%(safepattern)s' is not a
                 legal regular expression.  It will be discarded."""))
                 continue
diff --git a/Mailman/Handlers/Tagger.py b/Mailman/Handlers/Tagger.py
index f384356d1..58e27fde2 100644
--- a/Mailman/Handlers/Tagger.py
+++ b/Mailman/Handlers/Tagger.py
@@ -23,6 +23,7 @@ import email.Errors
 import email.Iterators
 import email.Parser
 
+OR = '|'
 CRNL = '\r\n'
 EMPTYSTRING = ''
 NLTAB = '\n\t'
@@ -51,7 +52,8 @@ def process(mlist, msg, msgdata):
     # added to the specific topics bucket.
     hits = {}
     for name, pattern, desc, emptyflag in mlist.topics:
-        cre = re.compile(pattern, re.IGNORECASE | re.VERBOSE)
+        pattern = OR.join(pattern.splitlines())
+        cre = re.compile(pattern, re.IGNORECASE)
         for line in matchlines:
             if cre.search(line):
                 hits[name] = 1
diff --git a/Mailman/MailList.py b/Mailman/MailList.py
index 7acd3b7fa..04fb0f22c 100644
--- a/Mailman/MailList.py
+++ b/Mailman/MailList.py
@@ -74,6 +74,7 @@ from Mailman.OldStyleMemberships import OldStyleMemberships
 _ = i18n._
 
 EMPTYSTRING = ''
+OR = '|'
 
 clog    = logging.getLogger('mailman.config')
 elog    = logging.getLogger('mailman.error')
@@ -742,10 +743,11 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
         goodtopics = []
         for name, pattern, desc, emptyflag in self.topics:
             try:
-                re.compile(pattern)
+                orpattern = OR.join(pattern.splitlines())
+                re.compile(orpattern)
             except (re.error, TypeError):
                 elog.error('Bad topic pattern "%s" for list: %s',
-                           pattern, self.internal_name())
+                           orpattern, self.internal_name())
             else:
                 goodtopics.append((name, pattern, desc, emptyflag))
         self.topics = goodtopics
diff --git a/Mailman/Utils.py b/Mailman/Utils.py
index 5e319cad3..42bacc16a 100644
--- a/Mailman/Utils.py
+++ b/Mailman/Utils.py
@@ -47,6 +47,7 @@ from Mailman.SafeDict import SafeDict
 
 EMPTYSTRING = ''
 UEMPTYSTRING = u''
+CR = '\r'
 NL = '\n'
 DOT = '.'
 IDENTCHARS = ascii_letters + digits + '_'
@@ -206,9 +207,16 @@ def ValidateEmail(s):
 
 
 
+# Patterns which may be used to form malicious path to inject a new
+# line in the mailman error log. (TK: advisory by Moritz Naumann)
+CRNLpat = re.compile(r'[^\x21-\x7e]')
+
 def GetPathPieces(envar='PATH_INFO'):
     path = os.environ.get(envar)
     if path:
+        if CRNLpat.search(path):
+            path = CRNLpat.split(path)[0]
+            log.error('Warning: Possible malformed path attack.')
         return [p for p in path.split('/') if p]
     return None
 
@@ -849,3 +857,58 @@ def oneline(s, cset):
     except (LookupError, UnicodeError, ValueError, HeaderParseError):
         # possibly charset problem. return with undecoded string in one line.
         return EMPTYSTRING.join(s.splitlines())
+
+
+def strip_verbose_pattern(pattern):
+    # Remove white space and comments from a verbose pattern and return a
+    # non-verbose, equivalent pattern.  Replace CR and NL in the result
+    # with '\\r' and '\\n' respectively to avoid multi-line results.
+    if not isinstance(pattern, str):
+        return pattern
+    newpattern = ''
+    i = 0
+    inclass = False
+    skiptoeol = False
+    copynext = False
+    while i < len(pattern):
+        c = pattern[i]
+        if copynext:
+            if c == NL:
+                newpattern += '\\n'
+            elif c == CR:
+                newpattern += '\\r'
+            else:
+                newpattern += c
+            copynext = False
+        elif skiptoeol:
+            if c == NL:
+                skiptoeol = False
+        elif c == '#' and not inclass:
+            skiptoeol = True
+        elif c == '[' and not inclass:
+            inclass = True
+            newpattern += c
+            copynext = True
+        elif c == ']' and inclass:
+            inclass = False
+            newpattern += c
+        elif re.search('\s', c):
+            if inclass:
+                if c == NL:
+                    newpattern += '\\n'
+                elif c == CR:
+                    newpattern += '\\r'
+                else:
+                    newpattern += c
+        elif c == '\\' and not inclass:
+            newpattern += c
+            copynext = True
+        else:
+            if c == NL:
+                newpattern += '\\n'
+            elif c == CR:
+                newpattern += '\\r'
+            else:
+                newpattern += c
+        i += 1
+    return newpattern
diff --git a/Mailman/Version.py b/Mailman/Version.py
index 35e6c91c4..6c9aad05e 100644
--- a/Mailman/Version.py
+++ b/Mailman/Version.py
@@ -36,7 +36,7 @@ HEX_VERSION = ((MAJOR_REV << 24) | (MINOR_REV << 16) | (MICRO_REV << 8) |
                (REL_LEVEL << 4)  | (REL_SERIAL << 0))
 
 # config.pck schema version number
-DATA_FILE_VERSION = 97
+DATA_FILE_VERSION = 98
 
 # qfile/*.db schema version number
 QFILE_SCHEMA_VERSION = 3
diff --git a/Mailman/versions.py b/Mailman/versions.py
index 531bb5cff..56dc840a2 100644
--- a/Mailman/versions.py
+++ b/Mailman/versions.py
@@ -307,6 +307,15 @@ def UpdateOldVars(l, stored_state):
             pass
         else:
             l.digest_members[k] = 0
+    #
+    # Convert pre 2.2 topics regexps which were compiled in verbose mode
+    # to a non-verbose equivalent.
+    #
+    if stored_state['data_version'] <= 97 and stored_state.has_key('topics'):
+        l.topics = []
+        for name, pattern, description, emptyflag in stored_state['topics']:
+            pattern = Utils.strip_verbose_pattern(pattern)
+            l.topics.append((name, pattern, description, emptyflag))
 
 
 
diff --git a/NEWS b/NEWS
index 21a6f5184..58542d554 100644
--- a/NEWS
+++ b/NEWS
@@ -23,6 +23,31 @@ Here is a history of user visible changes to Mailman.
       of email addresses and regular expressions matching email addresses
       whose subscriptions are exempt from admin approval. RFE 403066.
 
+  Bug fixes and other patches
+
+    - The processing of Topics regular expressions has changed. Previously the
+      Topics regexp was compiled in verbose mode but not documented as such
+      which caused some confusion.  Also, the documentation indicated that topic
+      keywords could be entered one per line, but these entries were not
+      properly.  Topics regexps are now compiled in non-verbose mode and multi-
+      line entries are 'ored'.  Existing Topics regexps will be converted when
+      the list is updated so they will continue to work.
+
+2.1.9 (xx-xxx-xxxx)
+
+  Security
+
+    - A malicious user could visit a specially crafted URI and inject an
+      apparent log message into Mailman's error log which might induce an
+      unsuspecting administrator to visit a phishing site.  This has been
+      blocked.  Thanks to Moritz Naumann for its discovery.
+
+  Bug fixes and other patches
+
+    - Fixed Decorate.py so that characters in message header/footer which
+      are not in the character set of the list's language are ignored rather
+      than causing shunted messages (1507248).
+
 2.1.8 (15-Apr-2006)
 
   Security