- Utils.py Fixed a security hole which allowed a crafted URI to inject

  bogus apparent messages into the error log, possibly inducing an admin to
  visit a phishing site.

- options.py
  Topics.py
  Tagger.py
  MailList.py
  Utils.py
  Version.py
  versions.py
  The processing of Topics regular expressions has changed. Previously the
  Topics regexp was compiled in verbose mode but not documented as such
  which caused some confusion.  Also, the documentation indicated that topic
  keywords could be entered one per line, but these entries were not
  properly.  Topics regexps are now compiled in non-verbose mode and multi-
  line entries are 'ored'.  Existing Topics regexps will be converted when
  the list is updated so they will continue to work.

7 files changed, 88 insertions, 7 deletions

diff --git a/Mailman/Cgi/options.py b/Mailman/Cgi/options.py
index 9fb008e6a..bfc63920e 100644
--- a/Mailman/Cgi/options.py
+++ b/Mailman/Cgi/options.py
@@ -33,6 +33,7 @@ from Mailman import Utils
 
 from Mailman.htmlformat import *
 
+OR = '|'
 SLASH = '/'
 SETLANGUAGE = -1
 
@@ -1021,7 +1022,8 @@ def topic_details(mlist, doc, user, cpuser, userlang, varhelp):
     table.AddRow([Bold(Label(_('Name:'))),
                   Utils.websafe(name)])
     table.AddRow([Bold(Label(_('Pattern (as regexp):'))),
-                  '<pre>' + Utils.websafe(pattern) + '</pre>'])
+                  '<pre>' + Utils.websafe(OR.join(pattern.splitlines()))
+                   + '</pre>'])
     table.AddRow([Bold(Label(_('Description:'))),
                   Utils.websafe(description)])
     # Make colors look nice
diff --git a/Mailman/Gui/Topics.py b/Mailman/Gui/Topics.py
index 282930b7c..147b2de04 100644
--- a/Mailman/Gui/Topics.py
+++ b/Mailman/Gui/Topics.py
@@ -22,6 +22,8 @@ from Mailman import Utils
 from Mailman.i18n import _
 from Mailman.Gui.GUIBase import GUIBase
 
+OR = '|'
+
 
 
 class Topics(GUIBase):
@@ -119,9 +121,10 @@ class Topics(GUIBase):
             # Make sure the pattern was a legal regular expression
             name = Utils.websafe(name)
             try:
-                re.compile(pattern)
+                orpattern = OR.join(pattern.splitlines())
+                re.compile(orpattern)
             except (re.error, TypeError):
-                safepattern = Utils.websafe(pattern)
+                safepattern = Utils.websafe(orpattern)
                 doc.addError(_("""The topic pattern '%(safepattern)s' is not a
                 legal regular expression.  It will be discarded."""))
                 continue
diff --git a/Mailman/Handlers/Tagger.py b/Mailman/Handlers/Tagger.py
index f384356d1..58e27fde2 100644
--- a/Mailman/Handlers/Tagger.py
+++ b/Mailman/Handlers/Tagger.py
@@ -23,6 +23,7 @@ import email.Errors
 import email.Iterators
 import email.Parser
 
+OR = '|'
 CRNL = '\r\n'
 EMPTYSTRING = ''
 NLTAB = '\n\t'
@@ -51,7 +52,8 @@ def process(mlist, msg, msgdata):
     # added to the specific topics bucket.
     hits = {}
     for name, pattern, desc, emptyflag in mlist.topics:
-        cre = re.compile(pattern, re.IGNORECASE | re.VERBOSE)
+        pattern = OR.join(pattern.splitlines())
+        cre = re.compile(pattern, re.IGNORECASE)
         for line in matchlines:
             if cre.search(line):
                 hits[name] = 1
diff --git a/Mailman/MailList.py b/Mailman/MailList.py
index 7acd3b7fa..04fb0f22c 100644
--- a/Mailman/MailList.py
+++ b/Mailman/MailList.py
@@ -74,6 +74,7 @@ from Mailman.OldStyleMemberships import OldStyleMemberships
 _ = i18n._
 
 EMPTYSTRING = ''
+OR = '|'
 
 clog    = logging.getLogger('mailman.config')
 elog    = logging.getLogger('mailman.error')
@@ -742,10 +743,11 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
         goodtopics = []
         for name, pattern, desc, emptyflag in self.topics:
             try:
-                re.compile(pattern)
+                orpattern = OR.join(pattern.splitlines())
+                re.compile(orpattern)
             except (re.error, TypeError):
                 elog.error('Bad topic pattern "%s" for list: %s',
-                           pattern, self.internal_name())
+                           orpattern, self.internal_name())
             else:
                 goodtopics.append((name, pattern, desc, emptyflag))
         self.topics = goodtopics
diff --git a/Mailman/Utils.py b/Mailman/Utils.py
index 5e319cad3..42bacc16a 100644
--- a/Mailman/Utils.py
+++ b/Mailman/Utils.py
@@ -47,6 +47,7 @@ from Mailman.SafeDict import SafeDict
 
 EMPTYSTRING = ''
 UEMPTYSTRING = u''
+CR = '\r'
 NL = '\n'
 DOT = '.'
 IDENTCHARS = ascii_letters + digits + '_'
@@ -206,9 +207,16 @@ def ValidateEmail(s):
 
 
 
+# Patterns which may be used to form malicious path to inject a new
+# line in the mailman error log. (TK: advisory by Moritz Naumann)
+CRNLpat = re.compile(r'[^\x21-\x7e]')
+
 def GetPathPieces(envar='PATH_INFO'):
     path = os.environ.get(envar)
     if path:
+        if CRNLpat.search(path):
+            path = CRNLpat.split(path)[0]
+            log.error('Warning: Possible malformed path attack.')
         return [p for p in path.split('/') if p]
     return None
 
@@ -849,3 +857,58 @@ def oneline(s, cset):
     except (LookupError, UnicodeError, ValueError, HeaderParseError):
         # possibly charset problem. return with undecoded string in one line.
         return EMPTYSTRING.join(s.splitlines())
+
+
+def strip_verbose_pattern(pattern):
+    # Remove white space and comments from a verbose pattern and return a
+    # non-verbose, equivalent pattern.  Replace CR and NL in the result
+    # with '\\r' and '\\n' respectively to avoid multi-line results.
+    if not isinstance(pattern, str):
+        return pattern
+    newpattern = ''
+    i = 0
+    inclass = False
+    skiptoeol = False
+    copynext = False
+    while i < len(pattern):
+        c = pattern[i]
+        if copynext:
+            if c == NL:
+                newpattern += '\\n'
+            elif c == CR:
+                newpattern += '\\r'
+            else:
+                newpattern += c
+            copynext = False
+        elif skiptoeol:
+            if c == NL:
+                skiptoeol = False
+        elif c == '#' and not inclass:
+            skiptoeol = True
+        elif c == '[' and not inclass:
+            inclass = True
+            newpattern += c
+            copynext = True
+        elif c == ']' and inclass:
+            inclass = False
+            newpattern += c
+        elif re.search('\s', c):
+            if inclass:
+                if c == NL:
+                    newpattern += '\\n'
+                elif c == CR:
+                    newpattern += '\\r'
+                else:
+                    newpattern += c
+        elif c == '\\' and not inclass:
+            newpattern += c
+            copynext = True
+        else:
+            if c == NL:
+                newpattern += '\\n'
+            elif c == CR:
+                newpattern += '\\r'
+            else:
+                newpattern += c
+        i += 1
+    return newpattern
diff --git a/Mailman/Version.py b/Mailman/Version.py
index 35e6c91c4..6c9aad05e 100644
--- a/Mailman/Version.py
+++ b/Mailman/Version.py
@@ -36,7 +36,7 @@ HEX_VERSION = ((MAJOR_REV << 24) | (MINOR_REV << 16) | (MICRO_REV << 8) |
                (REL_LEVEL << 4)  | (REL_SERIAL << 0))
 
 # config.pck schema version number
-DATA_FILE_VERSION = 97
+DATA_FILE_VERSION = 98
 
 # qfile/*.db schema version number
 QFILE_SCHEMA_VERSION = 3
diff --git a/Mailman/versions.py b/Mailman/versions.py
index 531bb5cff..56dc840a2 100644
--- a/Mailman/versions.py
+++ b/Mailman/versions.py
@@ -307,6 +307,15 @@ def UpdateOldVars(l, stored_state):
             pass
         else:
             l.digest_members[k] = 0
+    #
+    # Convert pre 2.2 topics regexps which were compiled in verbose mode
+    # to a non-verbose equivalent.
+    #
+    if stored_state['data_version'] <= 97 and stored_state.has_key('topics'):
+        l.topics = []
+        for name, pattern, description, emptyflag in stored_state['topics']:
+            pattern = Utils.strip_verbose_pattern(pattern)
+            l.topics.append((name, pattern, description, emptyflag))