diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/mailman/utilities/importer.py | 59 | ||||
| -rw-r--r-- | src/mailman/utilities/tests/test_import.py | 109 |
2 files changed, 168 insertions, 0 deletions
diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py index 293e9c39c..df0557c08 100644 --- a/src/mailman/utilities/importer.py +++ b/src/mailman/utilities/importer.py @@ -24,9 +24,11 @@ __all__ = [ import os +import re import sys import codecs import datetime +import logging from mailman.config import config from mailman.core.errors import MailmanError @@ -38,6 +40,7 @@ from mailman.interfaces.autorespond import ResponseAction from mailman.interfaces.bans import IBanManager from mailman.interfaces.bounce import UnrecognizedBounceDisposition from mailman.interfaces.digests import DigestFrequency +from mailman.interfaces.chain import LinkAction from mailman.interfaces.languages import ILanguageManager from mailman.interfaces.mailinglist import IAcceptableAliasSet from mailman.interfaces.mailinglist import Personalization, ReplyToMunging @@ -51,6 +54,8 @@ from sqlalchemy import Boolean from urllib.error import URLError from zope.component import getUtility +log = logging.getLogger('mailman.error') + class Import21Error(MailmanError): @@ -124,6 +129,24 @@ def nonmember_action_mapping(value): 3: Action.discard, }[value] + +def action_to_chain(value): + # Converts an action number in Mailman 2.1 to the name of the corresponding + # chain in 3.x. The actions "approve", "subscribe" and "unsubscribe" are + # ignored. The defer action is converted to None, because it is not a jump + # to a terminal chain. + return { + 0: None, + #1: "approve", + 2: "reject", + 3: "discard", + #4: "subscribe", + #5: "unsubscribe", + 6: "accept", + 7: "hold", + }[value] + + def check_language_code(code): if code is None: @@ -310,6 +333,42 @@ def import_config_pck(mlist, config_dict): # When .add() rejects this, the line probably contains a regular # expression. Make that explicit for MM3. alias_set.add('^' + address) + # Handle header_filter_rules conversion to header_matches + header_matches = [] + for line_patterns, action, _unused in \ + config_dict.get('header_filter_rules', []): + chain = action_to_chain(action) + # now split the pattern in a header and a pattern + for line_pattern in line_patterns.splitlines(): + if not line_pattern.strip(): + continue + for sep in (': ', ':.', ':'): + header, sep, pattern = line_pattern.partition(sep) + if sep: + break # found it. + else: + # matches any header. Those are not supported. XXX + log.warning('Unsupported header_filter_rules pattern: %r', + line_pattern) + continue + header = header.strip().lstrip("^").lower() + header = header.replace('\\', '') + if not header: + log.warning('Can\'t parse the header in header_filter_rule: %r', + line_pattern) + continue + if not pattern: + # The line matched only the header, therefore the header can + # be anything. + pattern = '.*' + try: + re.compile(pattern) + except re.error: + log.warning('Skipping header_filter rule because of an ' + 'invalid regular expression: %r', line_pattern) + continue + header_matches.append((header, pattern, chain)) + mlist.header_matches = header_matches # Handle conversion to URIs. In MM2.1, the decorations are strings # containing placeholders, and there's no provision for language-specific # templates. In MM3, template locations are specified by URLs with the diff --git a/src/mailman/utilities/tests/test_import.py b/src/mailman/utilities/tests/test_import.py index dd3940cdd..52d3469c0 100644 --- a/src/mailman/utilities/tests/test_import.py +++ b/src/mailman/utilities/tests/test_import.py @@ -50,6 +50,7 @@ from mailman.interfaces.nntp import NewsgroupModeration from mailman.interfaces.templates import ITemplateLoader from mailman.interfaces.usermanager import IUserManager from mailman.testing.layers import ConfigLayer +from mailman.testing.helpers import LogFileMark from mailman.utilities.filesystem import makedirs from mailman.utilities.importer import import_config_pck, Import21Error from mailman.utilities.string import expand @@ -330,6 +331,114 @@ class TestBasicImport(unittest.TestCase): self.assertEqual(self._mlist.subscription_policy, SubscriptionPolicy.confirm_then_moderate) + def test_header_matches(self): + # This test contail real cases of header_filter_rules + self._pckdict['header_filter_rules'] = [ + ('^X-Spam-Status: Yes', 3, False), + ('X-Spam-Status: Yes', 3, False), + ('X\\-Spam\\-Status\\: Yes.*', 3, False), + ('X-Spam-Status: Yes\r\n\r\n', 2, False), + ('^X-Spam-Level: \\*\\*\\*.*$', 3, False), + ('^X-Spam-Level:.\\*\\*\r\n^X-Spam:.\\Yes', 3, False), + ('Subject: \\[SPAM\\].*', 3, False), + ('^Subject: .*loan.*', 3, False), + ('Original-Received: from *linkedin.com*\r\n', 3, False), + ('X-Git-Module: rhq.*git', 6, False), + ('Approved: verysecretpassword', 6, False), + ('^Subject: dev-\r\n^Subject: staging-', 3, False), + ('from: .*info@aolanchem.com\r\nfrom: .*@jw-express.com', 2, False), + ('^Received: from smtp-.*\\.fedoraproject\\.org\r\n' + '^Received: from mx.*\\.redhat.com\r\n' + '^Resent-date:\r\n' + '^Resent-from:\r\n' + '^Resent-Message-ID:\r\n' + '^Resent-to:\r\n' + '^Subject: [^mtv]\r\n', + 7, False), + ('^Received: from fedorahosted\\.org.*by fedorahosted\\.org\r\n' + '^Received: from hosted.*\\.fedoraproject.org.*by ' + 'hosted.*\\.fedoraproject\\.org\r\n' + '^Received: from hosted.*\\.fedoraproject.org.*by fedoraproject\\.org\r\n' + '^Received: from hosted.*\\.fedoraproject.org.*by fedorahosted\\.org', + 6, False), + ] + error_log = LogFileMark('mailman.error') + self._import() + self.assertListEqual(self._mlist.header_matches, [ + ('x-spam-status', 'Yes', 'discard'), + ('x-spam-status', 'Yes', 'discard'), + ('x-spam-status', 'Yes.*', 'discard'), + ('x-spam-status', 'Yes', 'reject'), + ('x-spam-level', '\\*\\*\\*.*$', 'discard'), + ('x-spam-level', '\\*\\*', 'discard'), + ('x-spam', '\\Yes', 'discard'), + ('subject', '\\[SPAM\\].*', 'discard'), + ('subject', '.*loan.*', 'discard'), + ('original-received', 'from *linkedin.com*', 'discard'), + ('x-git-module', 'rhq.*git', 'accept'), + ('approved', 'verysecretpassword', 'accept'), + ('subject', 'dev-', 'discard'), + ('subject', 'staging-', 'discard'), + ('from', '.*info@aolanchem.com', 'reject'), + ('from', '.*@jw-express.com', 'reject'), + ('received', 'from smtp-.*\\.fedoraproject\\.org', 'hold'), + ('received', 'from mx.*\\.redhat.com', 'hold'), + ('resent-date', '.*', 'hold'), + ('resent-from', '.*', 'hold'), + ('resent-message-id', '.*', 'hold'), + ('resent-to', '.*', 'hold'), + ('subject', '[^mtv]', 'hold'), + ('received', 'from fedorahosted\\.org.*by fedorahosted\\.org', 'accept'), + ('received', 'from hosted.*\\.fedoraproject.org.*by hosted.*\\.fedoraproject\\.org', 'accept'), + ('received', 'from hosted.*\\.fedoraproject.org.*by fedoraproject\\.org', 'accept'), + ('received', 'from hosted.*\\.fedoraproject.org.*by fedorahosted\\.org', 'accept'), + ]) + loglines = error_log.read().strip() + self.assertEqual(len(loglines), 0) + + def test_header_matches_header_only(self): + # Check that an empty pattern is skipped + self._pckdict['header_filter_rules'] = [ + ('SomeHeaderName', 3, False), + ] + error_log = LogFileMark('mailman.error') + self._import() + self.assertListEqual(self._mlist.header_matches, []) + self.assertIn('Unsupported header_filter_rules pattern', + error_log.readline()) + + def test_header_matches_anything(self): + # Check that an empty pattern is skipped + self._pckdict['header_filter_rules'] = [ + ('.*', 7, False), + ] + error_log = LogFileMark('mailman.error') + self._import() + self.assertListEqual(self._mlist.header_matches, []) + self.assertIn('Unsupported header_filter_rules pattern', + error_log.readline()) + + def test_header_matches_invalid_re(self): + # Check that an empty pattern is skipped + self._pckdict['header_filter_rules'] = [ + ('SomeHeaderName: *invalid-re', 3, False), + ] + error_log = LogFileMark('mailman.error') + self._import() + self.assertListEqual(self._mlist.header_matches, []) + self.assertIn('Skipping header_filter rule because of an invalid ' + 'regular expression', error_log.readline()) + + def test_header_matches_defer(self): + # Check that a defer action is properly converted. + self._pckdict['header_filter_rules'] = [ + ('^X-Spam-Status: Yes', 0, False), + ] + self._import() + self.assertListEqual(self._mlist.header_matches, [ + ('x-spam-status', 'Yes', None), + ]) + class TestArchiveImport(unittest.TestCase): |