diff options
| author | Barry Warsaw | 2011-04-10 18:03:37 -0400 |
|---|---|---|
| committer | Barry Warsaw | 2011-04-10 18:03:37 -0400 |
| commit | 37038a683cd909438a6dee43beb9b258ef4e4313 (patch) | |
| tree | df50eabfcc890f01203d90c453dc1b1dbde65d2c /src | |
| parent | cce9729cac32b6c5fe2acc77b2bfb6b7c545711f (diff) | |
| parent | ef3a4a87e2c0f4b640e31afc4828d2edbd005846 (diff) | |
| download | mailman-37038a683cd909438a6dee43beb9b258ef4e4313.tar.gz mailman-37038a683cd909438a6dee43beb9b258ef4e4313.tar.zst mailman-37038a683cd909438a6dee43beb9b258ef4e4313.zip | |
Trunk merge
Diffstat (limited to 'src')
53 files changed, 1586 insertions, 984 deletions
diff --git a/src/mailman/Archiver/Archiver.py b/src/mailman/Archiver/Archiver.py index f8d1baa46..fe1dec252 100644 --- a/src/mailman/Archiver/Archiver.py +++ b/src/mailman/Archiver/Archiver.py @@ -32,9 +32,9 @@ from cStringIO import StringIO from string import Template from zope.component import getUtility -from mailman import Utils from mailman.config import config from mailman.interfaces.domain import IDomainManager +from mailman.utilities.i18n import make log = logging.getLogger('mailman.error') @@ -111,11 +111,11 @@ class Archiver: fp = open(indexfile, 'w') finally: os.umask(omask) - fp.write(Utils.maketext( - 'emptyarchive.html', - {'listname': self.real_name, - 'listinfo': self.GetScriptURL('listinfo'), - }, mlist=self)) + fp.write(make('emptyarchive.html', + mailing_list=self, + listname=self.real_name, + listinfo=self.GetScriptURL('listinfo'), + )) if fp: fp.close() finally: diff --git a/src/mailman/Archiver/HyperArch.py b/src/mailman/Archiver/HyperArch.py index 7281cdb0f..646608590 100644 --- a/src/mailman/Archiver/HyperArch.py +++ b/src/mailman/Archiver/HyperArch.py @@ -44,12 +44,13 @@ from lazr.config import as_boolean from string import Template from zope.component import getUtility -from mailman import Utils from mailman.Archiver import HyperDatabase from mailman.Archiver import pipermail from mailman.config import config from mailman.core.i18n import _, ctime from mailman.interfaces.listmanager import IListManager +from mailman.utilities.i18n import find +from mailman.utilities.string import uncanonstr, websafe log = logging.getLogger('mailman.error') @@ -85,7 +86,7 @@ def html_quote(s, langcode=None): ('"', '"')) for thing, repl in repls: s = s.replace(thing, repl) - return Utils.uncanonstr(s, langcode) + return uncanonstr(s, langcode) def url_quote(s): @@ -119,10 +120,10 @@ html_charset = '<META http-equiv="Content-Type" ' \ def CGIescape(arg, lang=None): if isinstance(arg, unicode): - s = Utils.websafe(arg) + s = websafe(arg) else: - s = Utils.websafe(str(arg)) - return Utils.uncanonstr(s.replace('"', '"'), lang.code) + s = websafe(str(arg)) + return uncanonstr(s.replace('"', '"'), lang.code) # Parenthesized human name paren_name_pat = re.compile(r'([(].*[)])') @@ -182,8 +183,8 @@ def quick_maketext(templatefile, dict=None, lang=None, mlist=None): template = _templatecache.get(filepath) if filepath is None or template is None: # Use the basic maketext, with defaults to get the raw template - template, filepath = Utils.findtext(templatefile, lang=lang.code, - raw=True, mlist=mlist) + template, filepath = find(templatefile, mailing_list=mlist, + language=lang.code) _templatefilepathcache[cachekey] = filepath _templatecache[filepath] = template # Copied from Utils.maketext() @@ -201,7 +202,7 @@ def quick_maketext(templatefile, dict=None, lang=None, mlist=None): pass # Make sure the text is in the given character set, or html-ify any bogus # characters. - return Utils.uncanonstr(text, lang.code) + return uncanonstr(text, lang.code) diff --git a/src/mailman/Archiver/pipermail.py b/src/mailman/Archiver/pipermail.py index f47600eb1..e11cb7173 100644 --- a/src/mailman/Archiver/pipermail.py +++ b/src/mailman/Archiver/pipermail.py @@ -10,7 +10,7 @@ import mailbox import cPickle as pickle from cStringIO import StringIO -from email.Utils import parseaddr, parsedate_tz, mktime_tz, formatdate +from email.utils import parseaddr, parsedate_tz, mktime_tz, formatdate from string import lowercase __version__ = '0.11 (Mailman edition)' diff --git a/src/mailman/Utils.py b/src/mailman/Utils.py deleted file mode 100644 index c07888fc5..000000000 --- a/src/mailman/Utils.py +++ /dev/null @@ -1,553 +0,0 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Miscellaneous essential routines. - -This includes actual message transmission routines, address checking and -message and address munging, a handy-dandy routine to map a function on all -the mailing lists, and whatever else doesn't belong elsewhere. -""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - ] - - -import os -import re -import cgi -import errno -import base64 -import random -import logging - -# pylint: disable-msg=E0611,W0403 -from email.errors import HeaderParseError -from email.header import decode_header, make_header -from lazr.config import as_boolean -from string import ascii_letters, digits, whitespace -from zope.component import getUtility - -import mailman.templates - -from mailman import passwords -from mailman.config import config -from mailman.core.i18n import _ -from mailman.interfaces.languages import ILanguageManager -from mailman.utilities.string import expand - - -AT = '@' -CR = '\r' -DOT = '.' -EMPTYSTRING = '' -IDENTCHARS = ascii_letters + digits + '_' -NL = '\n' -UEMPTYSTRING = u'' -TEMPLATE_DIR = os.path.dirname(mailman.templates.__file__) - -# Search for $(identifier)s strings, except that the trailing s is optional, -# since that's a common mistake -cre = re.compile(r'%\(([_a-z]\w*?)\)s?', re.IGNORECASE) -# Search for $$, $identifier, or ${identifier} -dre = re.compile(r'(\${2})|\$([_a-z]\w*)|\${([_a-z]\w*)}', re.IGNORECASE) - -log = logging.getLogger('mailman.error') - - - -# A much more naive implementation than say, Emacs's fill-paragraph! -# pylint: disable-msg=R0912 -def wrap(text, column=70, honor_leading_ws=True): - """Wrap and fill the text to the specified column. - - Wrapping is always in effect, although if it is not possible to wrap a - line (because some word is longer than `column' characters) the line is - broken at the next available whitespace boundary. Paragraphs are also - always filled, unless honor_leading_ws is true and the line begins with - whitespace. This is the algorithm that the Python FAQ wizard uses, and - seems like a good compromise. - - """ - wrapped = '' - # first split the text into paragraphs, defined as a blank line - paras = re.split('\n\n', text) - for para in paras: - # fill - lines = [] - fillprev = False - for line in para.split(NL): - if not line: - lines.append(line) - continue - if honor_leading_ws and line[0] in whitespace: - fillthis = False - else: - fillthis = True - if fillprev and fillthis: - # if the previous line should be filled, then just append a - # single space, and the rest of the current line - lines[-1] = lines[-1].rstrip() + ' ' + line - else: - # no fill, i.e. retain newline - lines.append(line) - fillprev = fillthis - # wrap each line - for text in lines: - while text: - if len(text) <= column: - line = text - text = '' - else: - bol = column - # find the last whitespace character - while bol > 0 and text[bol] not in whitespace: - bol -= 1 - # now find the last non-whitespace character - eol = bol - while eol > 0 and text[eol] in whitespace: - eol -= 1 - # watch out for text that's longer than the column width - if eol == 0: - # break on whitespace after column - eol = column - while eol < len(text) and text[eol] not in whitespace: - eol += 1 - bol = eol - while bol < len(text) and text[bol] in whitespace: - bol += 1 - bol -= 1 - line = text[:eol+1] + '\n' - # find the next non-whitespace character - bol += 1 - while bol < len(text) and text[bol] in whitespace: - bol += 1 - text = text[bol:] - wrapped += line - wrapped += '\n' - # end while text - wrapped += '\n' - # end for text in lines - # the last two newlines are bogus - return wrapped[:-2] - - - -_vowels = ('a', 'e', 'i', 'o', 'u') -_consonants = ('b', 'c', 'd', 'f', 'g', 'h', 'k', 'm', 'n', - 'p', 'r', 's', 't', 'v', 'w', 'x', 'z') -_syllables = [] - -for v in _vowels: - for c in _consonants: - _syllables.append(c+v) - _syllables.append(v+c) -del c, v - -def UserFriendly_MakeRandomPassword(length): - syls = [] - while len(syls) * 2 < length: - syls.append(random.choice(_syllables)) - return EMPTYSTRING.join(syls)[:length] - - -def Secure_MakeRandomPassword(length): - bytesread = 0 - bytes = [] - fd = None - try: - while bytesread < length: - try: - # Python 2.4 has this on available systems. - newbytes = os.urandom(length - bytesread) - except (AttributeError, NotImplementedError): - if fd is None: - try: - fd = os.open('/dev/urandom', os.O_RDONLY) - except OSError, e: - if e.errno != errno.ENOENT: - raise - # We have no available source of cryptographically - # secure random characters. Log an error and fallback - # to the user friendly passwords. - log.error( - 'urandom not available, passwords not secure') - return UserFriendly_MakeRandomPassword(length) - newbytes = os.read(fd, length - bytesread) - bytes.append(newbytes) - bytesread += len(newbytes) - s = base64.encodestring(EMPTYSTRING.join(bytes)) - # base64 will expand the string by 4/3rds - return s.replace('\n', '')[:length] - finally: - if fd is not None: - os.close(fd) - - -def MakeRandomPassword(length=None): - if length is None: - length = int(config.passwords.member_password_length) - if as_boolean(config.passwords.user_friendly_passwords): - password = UserFriendly_MakeRandomPassword(length) - else: - password = Secure_MakeRandomPassword(length) - return password.decode('ascii') - - -def GetRandomSeed(): - chr1 = int(random.random() * 52) - chr2 = int(random.random() * 52) - def mkletter(c): - if 0 <= c < 26: - c += 65 - if 26 <= c < 52: - #c = c - 26 + 97 - c += 71 - return c - return "%c%c" % tuple(map(mkletter, (chr1, chr2))) - - - -def set_global_password(pw, siteadmin=True, scheme=None): - if scheme is None: - scheme = passwords.Schemes.ssha - if siteadmin: - filename = config.SITE_PW_FILE - else: - filename = config.LISTCREATOR_PW_FILE - try: - fp = open(filename, 'w') - print >> fp, passwords.make_secret(pw, scheme) - finally: - fp.close() - - -def get_global_password(siteadmin=True): - if siteadmin: - filename = config.SITE_PW_FILE - else: - filename = config.LISTCREATOR_PW_FILE - try: - fp = open(filename) - challenge = fp.read()[:-1] # strip off trailing nl - fp.close() - except IOError, e: - if e.errno != errno.ENOENT: - raise - # It's okay not to have a site admin password - return None - return challenge - - -def check_global_password(response, siteadmin=True): - challenge = get_global_password(siteadmin) - if challenge is None: - return False - return passwords.check_response(challenge, response) - - - -def websafe(s): - return cgi.escape(s, quote=True) - - -def nntpsplit(s): - parts = s.split(':', 1) - if len(parts) == 2: - try: - return parts[0], int(parts[1]) - except ValueError: - pass - # Use the defaults - return s, 119 - - - -# Just changing these two functions should be enough to control the way -# that email address obscuring is handled. -def ObscureEmail(addr, for_text=False): - """Make email address unrecognizable to web spiders, but invertable. - - When for_text option is set (not default), make a sentence fragment - instead of a token.""" - if for_text: - return addr.replace('@', ' at ') - else: - return addr.replace('@', '--at--') - -def UnobscureEmail(addr): - """Invert ObscureEmail() conversion.""" - # Contrived to act as an identity operation on already-unobscured - # emails, so routines expecting obscured ones will accept both. - return addr.replace('--at--', '@') - - - -class OuterExit(Exception): - pass - -def findtext(templatefile, raw_dict=None, raw=False, lang=None, mlist=None): - # Make some text from a template file. The order of searches depends on - # whether mlist and lang are provided. Once the templatefile is found, - # string substitution is performed by interpolation in `dict'. If `raw' - # is false, the resulting text is wrapped/filled by calling wrap(). - # - # When looking for a template in a specific language, there are 4 places - # that are searched, in this order: - # - # 1. the list-specific language directory - # lists/<listname>/<language> - # - # 2. the domain-specific language directory - # templates/<list.host_name>/<language> - # - # 3. the site-wide language directory - # templates/site/<language> - # - # 4. the global default language directory - # templates/<language> - # - # The first match found stops the search. In this way, you can specialize - # templates at the desired level, or, if you use only the default - # templates, you don't need to change anything. You should never modify - # files in the templates/<language> subdirectory, since Mailman will - # overwrite these when you upgrade. That's what the templates/site - # language directories are for. - # - # A further complication is that the language to search for is determined - # by both the `lang' and `mlist' arguments. The search order there is - # that if lang is given, then the 4 locations above are searched, - # substituting lang for <language>. If no match is found, and mlist is - # given, then the 4 locations are searched using the list's preferred - # language. After that, the server default language is used for - # <language>. If that still doesn't yield a template, then the standard - # distribution's English language template is used as an ultimate - # fallback, and when lang is not 'en', the resulting template is passed - # through the translation service. If this template is missing you've got - # big problems. ;) - # - # A word on backwards compatibility: Mailman versions prior to 2.1 stored - # templates in templates/*.{html,txt} and lists/<listname>/*.{html,txt}. - # Those directories are no longer searched so if you've got customizations - # in those files, you should move them to the appropriate directory based - # on the above description. Mailman's upgrade script cannot do this for - # you. - # - # The function has been revised and renamed as it now returns both the - # template text and the path from which it retrieved the template. The - # original function is now a wrapper which just returns the template text - # as before, by calling this renamed function and discarding the second - # item returned. - # - # Calculate the languages to scan - languages = set() - if lang is not None: - languages.add(lang) - if mlist is not None: - languages.add(mlist.preferred_language.code) - languages.add(config.mailman.default_language) - assert None not in languages, 'None in languages' - # Calculate the locations to scan - searchdirs = [] - if mlist is not None: - searchdirs.append(mlist.data_path) - searchdirs.append(os.path.join(TEMPLATE_DIR, mlist.host_name)) - searchdirs.append(os.path.join(TEMPLATE_DIR, 'site')) - searchdirs.append(TEMPLATE_DIR) - # Start scanning - fp = None - try: - for lang in languages: - for dir in searchdirs: - filename = os.path.join(dir, lang, templatefile) - try: - fp = open(filename) - raise OuterExit - except IOError, e: - if e.errno != errno.ENOENT: - raise - # Okay, it doesn't exist, keep looping - fp = None - except OuterExit: - pass - if fp is None: - # Try one last time with the distro English template, which, unless - # you've got a really broken installation, must be there. - try: - filename = os.path.join(TEMPLATE_DIR, 'en', templatefile) - fp = open(filename) - except IOError, e: - if e.errno != errno.ENOENT: - raise - # We never found the template. BAD! - raise IOError(errno.ENOENT, 'No template file found', templatefile) - else: - # XXX BROKEN HACK - data = fp.read()[:-1] - template = _(data) - fp.close() - else: - template = fp.read() - fp.close() - charset = getUtility(ILanguageManager)[lang].charset - template = unicode(template, charset, 'replace') - text = template - if raw_dict is not None: - text = expand(template, raw_dict) - if raw: - return text, filename - return wrap(text), filename - - -def maketext(templatefile, dict=None, raw=False, lang=None, mlist=None): - return findtext(templatefile, dict, raw, lang, mlist)[0] - - - -# The opposite of canonstr() -- sorta. I.e. it attempts to encode s in the -# charset of the given language, which is the character set that the page will -# be rendered in, and failing that, replaces non-ASCII characters with their -# html references. It always returns a byte string. -def uncanonstr(s, lang=None): - if s is None: - s = u'' - if lang is None: - charset = 'us-ascii' - else: - charset = getUtility(ILanguageManager)[lang].charset - # See if the string contains characters only in the desired character - # set. If so, return it unchanged, except for coercing it to a byte - # string. - try: - if isinstance(s, unicode): - return s.encode(charset) - else: - unicode(s, charset) - return s - except UnicodeError: - # Nope, it contains funny characters, so html-ref it - return uquote(s) - - -def uquote(s): - a = [] - for c in s: - o = ord(c) - if o > 127: - a.append('&#%3d;' % o) - else: - a.append(c) - # Join characters together and coerce to byte string - return str(EMPTYSTRING.join(a)) - - -def oneline(s, cset='us-ascii', in_unicode=False): - # Decode header string in one line and convert into specified charset - try: - h = make_header(decode_header(s)) - ustr = h.__unicode__() - line = UEMPTYSTRING.join(ustr.splitlines()) - if in_unicode: - return line - else: - return line.encode(cset, 'replace') - except (LookupError, UnicodeError, ValueError, HeaderParseError): - # possibly charset problem. return with undecoded string in one line. - return EMPTYSTRING.join(s.splitlines()) - - -def strip_verbose_pattern(pattern): - # Remove white space and comments from a verbose pattern and return a - # non-verbose, equivalent pattern. Replace CR and NL in the result - # with '\\r' and '\\n' respectively to avoid multi-line results. - if not isinstance(pattern, str): - return pattern - newpattern = '' - i = 0 - inclass = False - skiptoeol = False - copynext = False - while i < len(pattern): - c = pattern[i] - if copynext: - if c == NL: - newpattern += '\\n' - elif c == CR: - newpattern += '\\r' - else: - newpattern += c - copynext = False - elif skiptoeol: - if c == NL: - skiptoeol = False - elif c == '#' and not inclass: - skiptoeol = True - elif c == '[' and not inclass: - inclass = True - newpattern += c - copynext = True - elif c == ']' and inclass: - inclass = False - newpattern += c - elif re.search('\s', c): - if inclass: - if c == NL: - newpattern += '\\n' - elif c == CR: - newpattern += '\\r' - else: - newpattern += c - elif c == '\\' and not inclass: - newpattern += c - copynext = True - else: - if c == NL: - newpattern += '\\n' - elif c == CR: - newpattern += '\\r' - else: - newpattern += c - i += 1 - return newpattern - - - -def get_pattern(email, pattern_list): - """Returns matched entry in pattern_list if email matches. - Otherwise returns None. - """ - if not pattern_list: - return None - matched = None - for pattern in pattern_list: - if pattern.startswith('^'): - # This is a regular expression match - try: - if re.search(pattern, email, re.IGNORECASE): - matched = pattern - break - except re.error: - # BAW: we should probably remove this pattern - pass - else: - # Do the comparison case insensitively - if pattern.lower() == email.lower(): - matched = pattern - break - return matched diff --git a/src/mailman/app/bounces.py b/src/mailman/app/bounces.py index 7504c441b..7f6507a14 100644 --- a/src/mailman/app/bounces.py +++ b/src/mailman/app/bounces.py @@ -30,11 +30,11 @@ import logging from email.mime.message import MIMEMessage from email.mime.text import MIMEText -from mailman.Utils import oneline from mailman.app.finder import find_components from mailman.core.i18n import _ from mailman.email.message import UserNotification from mailman.interfaces.bounce import IBounceDetector +from mailman.utilities.string import oneline log = logging.getLogger('mailman.config') diff --git a/src/mailman/app/docs/bans.rst b/src/mailman/app/docs/bans.rst new file mode 100644 index 000000000..bb6d5902e --- /dev/null +++ b/src/mailman/app/docs/bans.rst @@ -0,0 +1,167 @@ +======================= +Banning email addresses +======================= + +Email addresses can be banned from ever subscribing, either to a specific +mailing list or globally within the Mailman system. Both explicit email +addresses and email address patterns can be banned. + +Bans are managed through the `Ban Manager`. + + >>> from zope.component import getUtility + >>> from mailman.interfaces.bans import IBanManager + >>> ban_manager = getUtility(IBanManager) + +At first, no email addresses are banned, either globally... + + >>> ban_manager.is_banned('anne@example.com') + False + +...or for a specific mailing list. + + >>> ban_manager.is_banned('bart@example.com', 'test@example.com') + False + + +Specific bans +============= + +An email address can be banned from a specific mailing list by adding a ban to +the ban manager. + + >>> ban_manager.ban('cris@example.com', 'test@example.com') + >>> ban_manager.is_banned('cris@example.com', 'test@example.com') + True + >>> ban_manager.is_banned('bart@example.com', 'test@example.com') + False + +However, this is not a global ban. + + >>> ban_manager.is_banned('cris@example.com') + False + + +Global bans +=========== + +An email address can be banned globally, so that it cannot be subscribed to +any mailing list. + + >>> ban_manager.ban('dave@example.com') + +Dave is banned from the test mailing list... + + >>> ban_manager.is_banned('dave@example.com', 'test@example.com') + True + +...and the sample mailing list. + + >>> ban_manager.is_banned('dave@example.com', 'sample@example.com') + True + +Dave is also banned globally. + + >>> ban_manager.is_banned('dave@example.com') + True + +Cris however is not banned globally. + + >>> ban_manager.is_banned('cris@example.com') + False + +Even though Cris is not banned globally, we can add a global ban for her. + + >>> ban_manager.ban('cris@example.com') + >>> ban_manager.is_banned('cris@example.com') + True + +Cris is obviously still banned from specific mailing lists. + + >>> ban_manager.is_banned('cris@example.com', 'test@example.com') + True + >>> ban_manager.is_banned('cris@example.com', 'sample@example.com') + True + +We can remove the global ban to once again just ban her address from the test +list. + + >>> ban_manager.unban('cris@example.com') + >>> ban_manager.is_banned('cris@example.com', 'test@example.com') + True + >>> ban_manager.is_banned('cris@example.com', 'sample@example.com') + False + + +Regular expression bans +======================= + +Entire email address patterns can be banned, both for a specific mailing list +and globally, just as specific addresses can be banned. Use this for example, +when an entire domain is a spam faucet. When using a pattern, the email +address must start with a caret (^). + + >>> ban_manager.ban('^.*@example.org', 'test@example.com') + +Now, no one from example.org can subscribe to the test list. + + >>> ban_manager.is_banned('elle@example.org', 'test@example.com') + True + >>> ban_manager.is_banned('eperson@example.org', 'test@example.com') + True + >>> ban_manager.is_banned('elle@example.com', 'test@example.com') + False + +They are not, however banned globally. + + >>> ban_manager.is_banned('elle@example.org', 'sample@example.com') + False + >>> ban_manager.is_banned('elle@example.org') + False + +Of course, we can ban everyone from example.org globally too. + + >>> ban_manager.ban('^.*@example.org') + >>> ban_manager.is_banned('elle@example.org', 'sample@example.com') + True + >>> ban_manager.is_banned('elle@example.org') + True + +We can remove the mailing list ban on the pattern, though the global ban will +still be in place. + + >>> ban_manager.unban('^.*@example.org', 'test@example.com') + >>> ban_manager.is_banned('elle@example.org', 'test@example.com') + True + >>> ban_manager.is_banned('elle@example.org', 'sample@example.com') + True + >>> ban_manager.is_banned('elle@example.org') + True + +But once the global ban is removed, everyone from example.org can subscribe to +the mailing lists. + + >>> ban_manager.unban('^.*@example.org') + >>> ban_manager.is_banned('elle@example.org', 'test@example.com') + False + >>> ban_manager.is_banned('elle@example.org', 'sample@example.com') + False + >>> ban_manager.is_banned('elle@example.org') + False + + +Adding and removing bans +======================== + +It is not an error to add a ban more than once. These are just ignored. + + >>> ban_manager.ban('fred@example.com', 'test@example.com') + >>> ban_manager.ban('fred@example.com', 'test@example.com') + >>> ban_manager.is_banned('fred@example.com', 'test@example.com') + True + +Nor is it an error to remove a ban more than once. + + >>> ban_manager.unban('fred@example.com', 'test@example.com') + >>> ban_manager.unban('fred@example.com', 'test@example.com') + >>> ban_manager.is_banned('fred@example.com', 'test@example.com') + False diff --git a/src/mailman/app/lifecycle.py b/src/mailman/app/lifecycle.py index 39ec09aa7..b30266f3b 100644 --- a/src/mailman/app/lifecycle.py +++ b/src/mailman/app/lifecycle.py @@ -33,7 +33,7 @@ import logging from zope.component import getUtility from mailman.config import config -from mailman.email.validate import validate +from mailman.interfaces.address import IEmailValidator from mailman.interfaces.domain import ( BadDomainSpecificationError, IDomainManager) from mailman.interfaces.listmanager import IListManager @@ -58,13 +58,15 @@ def create_list(fqdn_listname, owners=None): :type owners: list of string email addresses :return: The new mailing list. :rtype: `IMailingList` - :raises `BadDomainSpecificationError`: when the hostname part of + :raises BadDomainSpecificationError: when the hostname part of `fqdn_listname` does not exist. - :raises `ListAlreadyExistsError`: when the mailing list already exists. + :raises ListAlreadyExistsError: when the mailing list already exists. + :raises InvalidEmailAddressError: when the fqdn email address is invalid. """ if owners is None: owners = [] - validate(fqdn_listname) + # This raises I + getUtility(IEmailValidator).validate(fqdn_listname) # pylint: disable-msg=W0612 listname, domain = fqdn_listname.split('@', 1) if domain not in getUtility(IDomainManager): diff --git a/src/mailman/app/membership.py b/src/mailman/app/membership.py index 8ea8769a6..fcbedc2f5 100644 --- a/src/mailman/app/membership.py +++ b/src/mailman/app/membership.py @@ -29,15 +29,16 @@ __all__ = [ from email.utils import formataddr from zope.component import getUtility -from mailman import Utils from mailman.app.notifications import send_goodbye_message from mailman.core.i18n import _ from mailman.email.message import OwnerNotification -from mailman.email.validate import validate +from mailman.interfaces.address import IEmailValidator +from mailman.interfaces.bans import IBanManager from mailman.interfaces.member import ( AlreadySubscribedError, MemberRole, MembershipIsBannedError, NotAMemberError) from mailman.interfaces.usermanager import IUserManager +from mailman.utilities.i18n import make @@ -67,14 +68,12 @@ def add_member(mlist, email, realname, password, delivery_mode, language): :raises MembershipIsBannedError: if the membership is not allowed. """ # Let's be extra cautious. - validate(email) + getUtility(IEmailValidator).validate(email) if mlist.members.get_member(email) is not None: raise AlreadySubscribedError( mlist.fqdn_listname, email, MemberRole.member) - # Check for banned email addresses here too for administrative mass - # subscribes and confirmations. - pattern = Utils.get_pattern(email, mlist.ban_list) - if pattern: + # Check to see if the email address is banned. + if getUtility(IBanManager).is_banned(email, mlist.fqdn_listname): raise MembershipIsBannedError(mlist, email) # See if there's already a user linked with the given address. user_manager = getUtility(IUserManager) @@ -104,7 +103,7 @@ def add_member(mlist, email, realname, password, delivery_mode, language): else: # The user exists and is linked to the address. for address in user.addresses: - if address.email == address: + if address.email == email: break else: raise AssertionError( @@ -150,10 +149,10 @@ def delete_member(mlist, address, admin_notif=None, userack=None): user = getUtility(IUserManager).get_user(address) realname = user.real_name subject = _('$mlist.real_name unsubscription notification') - text = Utils.maketext( - 'adminunsubscribeack.txt', - {'listname': mlist.real_name, - 'member' : formataddr((realname, address)), - }, mlist=mlist) + text = make('adminunsubscribeack.txt', + mailing_list=mlist, + listname=mlist.real_name, + member=formataddr((realname, address)), + ) msg = OwnerNotification(mlist, subject, text) msg.send(mlist) diff --git a/src/mailman/app/moderator.py b/src/mailman/app/moderator.py index cdfedd44b..a2f838934 100644 --- a/src/mailman/app/moderator.py +++ b/src/mailman/app/moderator.py @@ -35,7 +35,6 @@ from datetime import datetime from email.utils import formataddr, formatdate, getaddresses, make_msgid from zope.component import getUtility -from mailman import Utils from mailman.app.membership import add_member, delete_member from mailman.app.notifications import ( send_admin_subscription_notice, send_welcome_message) @@ -48,6 +47,7 @@ from mailman.interfaces.member import ( AlreadySubscribedError, DeliveryMode, NotAMemberError) from mailman.interfaces.messages import IMessageStore from mailman.interfaces.requests import IRequests, RequestType +from mailman.utilities.i18n import make NL = '\n' @@ -209,12 +209,12 @@ def hold_subscription(mlist, address, realname, password, mode, language): if mlist.admin_immed_notify: subject = _( 'New subscription request to list $mlist.real_name from $address') - text = Utils.maketext( - 'subauth.txt', - {'username' : address, - 'listname' : mlist.fqdn_listname, - 'admindb_url': mlist.script_url('admindb'), - }, mlist=mlist) + text = make('subauth.txt', + mailing_list=mlist, + username=address, + listname=mlist.fqdn_listname, + admindb_url=mlist.script_url('admindb'), + ) # This message should appear to come from the <list>-owner so as # to avoid any useless bounce processing. msg = UserNotification( @@ -281,12 +281,12 @@ def hold_unsubscription(mlist, address): if mlist.admin_immed_notify: subject = _( 'New unsubscription request from $mlist.real_name by $address') - text = Utils.maketext( - 'unsubauth.txt', - {'address' : address, - 'listname' : mlist.fqdn_listname, - 'admindb_url': mlist.script_url('admindb'), - }, mlist=mlist) + text = make('unsubauth.txt', + mailing_list=mlist, + address=address, + listname=mlist.fqdn_listname, + admindb_url=mlist.script_url('admindb'), + ) # This message should appear to come from the <list>-owner so as # to avoid any useless bounce processing. msg = UserNotification( @@ -336,13 +336,14 @@ def _refuse(mlist, request, recip, comment, origmsg=None, lang=None): lang = (mlist.preferred_language if member is None else member.preferred_language) - text = Utils.maketext( - 'refuse.txt', - {'listname' : mlist.fqdn_listname, - 'request' : request, - 'reason' : comment, - 'adminaddr': mlist.owner_address, - }, lang=lang.code, mlist=mlist) + text = make('refuse.txt', + mailing_list=mlist, + language=lang.code, + listname=mlist.fqdn_listname, + request=request, + reason=comment, + adminaddr=mlist.owner_address, + ) with _.using(lang.code): # add in original message, but not wrap/filled if origmsg: diff --git a/src/mailman/app/notifications.py b/src/mailman/app/notifications.py index 985f4eece..8bfbd0934 100644 --- a/src/mailman/app/notifications.py +++ b/src/mailman/app/notifications.py @@ -30,11 +30,12 @@ __all__ = [ from email.utils import formataddr from lazr.config import as_boolean -from mailman import Utils from mailman.config import config from mailman.core.i18n import _ from mailman.email.message import OwnerNotification, UserNotification from mailman.interfaces.member import DeliveryMode +from mailman.utilities.i18n import make +from mailman.utilities.string import wrap @@ -54,7 +55,7 @@ def send_welcome_message(mlist, address, language, delivery_mode, text=''): :type delivery_mode: DeliveryMode """ if mlist.welcome_msg: - welcome = Utils.wrap(mlist.welcome_msg) + '\n' + welcome = wrap(mlist.welcome_msg) + '\n' else: welcome = '' # Find the IMember object which is subscribed to the mailing list, because @@ -62,15 +63,16 @@ def send_welcome_message(mlist, address, language, delivery_mode, text=''): member = mlist.members.get_member(address) options_url = member.options_url # Get the text from the template. - text += Utils.maketext( - 'subscribeack.txt', { - 'real_name' : mlist.real_name, - 'posting_address' : mlist.fqdn_listname, - 'listinfo_url' : mlist.script_url('listinfo'), - 'optionsurl' : options_url, - 'request_address' : mlist.request_address, - 'welcome' : welcome, - }, lang=language.code, mlist=mlist) + text += make('subscribeack.txt', + mailing_list=mlist, + language=language.code, + real_name=mlist.real_name, + posting_address=mlist.fqdn_listname, + listinfo_url=mlist.script_url('listinfo'), + optionsurl=options_url, + request_address=mlist.request_address, + welcome=welcome, + ) if delivery_mode is not DeliveryMode.regular: digmode = _(' (Digest mode)') else: @@ -98,7 +100,7 @@ def send_goodbye_message(mlist, address, language): :type language: string """ if mlist.goodbye_msg: - goodbye = Utils.wrap(mlist.goodbye_msg) + '\n' + goodbye = wrap(mlist.goodbye_msg) + '\n' else: goodbye = '' msg = UserNotification( @@ -124,10 +126,10 @@ def send_admin_subscription_notice(mlist, address, full_name, language): with _.using(mlist.preferred_language.code): subject = _('$mlist.real_name subscription notification') full_name = full_name.encode(language.charset, 'replace') - text = Utils.maketext( - 'adminsubscribeack.txt', - {'listname' : mlist.real_name, - 'member' : formataddr((full_name, address)), - }, mlist=mlist) + text = make('adminsubscribeack.txt', + mailing_list=mlist, + listname=mlist.real_name, + member=formataddr((full_name, address)), + ) msg = OwnerNotification(mlist, subject, text) msg.send(mlist) diff --git a/src/mailman/app/registrar.py b/src/mailman/app/registrar.py index 181d48126..ec899237a 100644 --- a/src/mailman/app/registrar.py +++ b/src/mailman/app/registrar.py @@ -33,7 +33,7 @@ from zope.interface import implements from mailman.core.i18n import _ from mailman.email.message import UserNotification -from mailman.email.validate import validate +from mailman.interfaces.address import IEmailValidator from mailman.interfaces.listmanager import IListManager from mailman.interfaces.member import MemberRole from mailman.interfaces.pending import IPendable, IPendings @@ -57,7 +57,7 @@ class Registrar: """See `IUserRegistrar`.""" # First, do validation on the email address. If the address is # invalid, it will raise an exception, otherwise it just returns. - validate(email) + getUtility(IEmailValidator).validate(email) # Create a pendable for the registration. pendable = PendableRegistration( type=PendableRegistration.PEND_KEY, diff --git a/src/mailman/app/tests/__init__.py b/src/mailman/app/tests/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/mailman/app/tests/__init__.py diff --git a/src/mailman/app/tests/test_membership.py b/src/mailman/app/tests/test_membership.py new file mode 100644 index 000000000..b0e1bae5d --- /dev/null +++ b/src/mailman/app/tests/test_membership.py @@ -0,0 +1,131 @@ +# Copyright (C) 2011 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Tests of application level membership functions.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'test_suite', + ] + + +import unittest + +from zope.component import getUtility + +from mailman.app.lifecycle import create_list +from mailman.app.membership import add_member +from mailman.core.constants import system_preferences +from mailman.interfaces.bans import IBanManager +from mailman.interfaces.member import DeliveryMode, MembershipIsBannedError +from mailman.interfaces.usermanager import IUserManager +from mailman.testing.helpers import reset_the_world +from mailman.testing.layers import ConfigLayer + + + +class AddMemberTest(unittest.TestCase): + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + + def tearDown(self): + reset_the_world() + + def test_add_member_new_user(self): + # Test subscribing a user to a mailing list when the email address has + # not yet been associated with a user. + member = add_member(self._mlist, 'aperson@example.com', + 'Anne Person', '123', DeliveryMode.regular, + system_preferences.preferred_language) + self.assertEqual(member.address.email, 'aperson@example.com') + self.assertEqual(member.mailing_list, 'test@example.com') + + def test_add_member_existing_user(self): + # Test subscribing a user to a mailing list when the email address has + # already been associated with a user. + user_manager = getUtility(IUserManager) + user_manager.create_user('aperson@example.com', 'Anne Person') + member = add_member(self._mlist, 'aperson@example.com', + 'Anne Person', '123', DeliveryMode.regular, + system_preferences.preferred_language) + self.assertEqual(member.address.email, 'aperson@example.com') + self.assertEqual(member.mailing_list, 'test@example.com') + + def test_add_member_banned(self): + # Test that members who are banned by specific address cannot + # subscribe to the mailing list. + getUtility(IBanManager).ban('anne@example.com', 'test@example.com') + self.assertRaises( + MembershipIsBannedError, + add_member, self._mlist, 'anne@example.com', 'Anne Person', + '123', DeliveryMode.regular, system_preferences.preferred_language) + + def test_add_member_globally_banned(self): + # Test that members who are banned by specific address cannot + # subscribe to the mailing list. + getUtility(IBanManager).ban('anne@example.com') + self.assertRaises( + MembershipIsBannedError, + add_member, self._mlist, 'anne@example.com', 'Anne Person', + '123', DeliveryMode.regular, system_preferences.preferred_language) + + def test_add_member_banned_from_different_list(self): + # Test that members who are banned by specific address cannot + # subscribe to the mailing list. + getUtility(IBanManager).ban('anne@example.com', 'sample@example.com') + member = add_member(self._mlist, 'anne@example.com', + 'Anne Person', '123', DeliveryMode.regular, + system_preferences.preferred_language) + self.assertEqual(member.address.email, 'anne@example.com') + + def test_add_member_banned_by_pattern(self): + # Test that members who are banned by specific address cannot + # subscribe to the mailing list. + getUtility(IBanManager).ban('^.*@example.com', 'test@example.com') + self.assertRaises( + MembershipIsBannedError, + add_member, self._mlist, 'anne@example.com', 'Anne Person', + '123', DeliveryMode.regular, system_preferences.preferred_language) + + def test_add_member_globally_banned_by_pattern(self): + # Test that members who are banned by specific address cannot + # subscribe to the mailing list. + getUtility(IBanManager).ban('^.*@example.com') + self.assertRaises( + MembershipIsBannedError, + add_member, self._mlist, 'anne@example.com', 'Anne Person', + '123', DeliveryMode.regular, system_preferences.preferred_language) + + def test_add_member_banned_from_different_list_by_pattern(self): + # Test that members who are banned by specific address cannot + # subscribe to the mailing list. + getUtility(IBanManager).ban('^.*@example.com', 'sample@example.com') + member = add_member(self._mlist, 'anne@example.com', + 'Anne Person', '123', DeliveryMode.regular, + system_preferences.preferred_language) + self.assertEqual(member.address.email, 'anne@example.com') + + + +def test_suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(AddMemberTest)) + return suite diff --git a/src/mailman/bin/config_list.py b/src/mailman/bin/config_list.py index 845a1371d..a40f4ee52 100644 --- a/src/mailman/bin/config_list.py +++ b/src/mailman/bin/config_list.py @@ -22,10 +22,10 @@ import optparse from mailman import MailList from mailman import errors -from mailman.Utils import wrap from mailman.configuration import config from mailman.core.i18n import _ from mailman.initialize import initialize +from mailman.utilities.string import wrap from mailman.version import MAILMAN_VERSION @@ -140,7 +140,7 @@ def do_list_categories(mlist, k, subcat, outfp): # triple-quoted string nonsense in the source code. desc = NL.join([s.lstrip() for s in info[0].splitlines()]) # Print out the category description - desc = Utils.wrap(desc) + desc = wrap(desc) for line in desc.splitlines(): print >> outfp, '#', line print >> outfp @@ -162,7 +162,7 @@ def do_list_categories(mlist, k, subcat, outfp): desc = re.sub('<', '<', desc) desc = re.sub('>', '>', desc) # Print out the variable description. - desc = Utils.wrap(desc) + desc = wrap(desc) for line in desc.split('\n'): print >> outfp, '#', line # munge the value based on its type @@ -246,8 +246,8 @@ def do_input(listname, infile, checkonly, verbose, parser): # Open the specified list locked, unless checkonly is set try: mlist = MailList.MailList(listname, lock=not checkonly) - except errors.MMListError, e: - parser.error(_('No such list "$listname"\n$e')) + except errors.MMListError as error: + parser.error(_('No such list "$listname"\n$error')) savelist = False guibyprop = getPropertyMap(mlist) try: diff --git a/src/mailman/bin/gate_news.py b/src/mailman/bin/gate_news.py index 1873def37..4ad29affc 100644 --- a/src/mailman/bin/gate_news.py +++ b/src/mailman/bin/gate_news.py @@ -26,10 +26,10 @@ import email.Errors from email.Parser import Parser from flufl.lock import Lock, TimeOutError +from lazr.config import as_host_port from mailman import MailList from mailman import Message -from mailman import Utils from mailman import loginit from mailman.configuration import config from mailman.core.i18n import _ @@ -69,8 +69,8 @@ Poll the NNTP servers for messages to be gatewayed to mailing lists.""")) _hostcache = {} def open_newsgroup(mlist): - # Split host:port if given - nntp_host, nntp_port = Utils.nntpsplit(mlist.nntp_host) + # Split host:port if given. + nntp_host, nntp_port = as_host_port(mlist.nntp_host, default_port=119) # Open up a "mode reader" connection to nntp server. This will be shared # for all the gated lists having the same nntp_host. conn = _hostcache.get(mlist.nntp_host) diff --git a/src/mailman/bin/import.py b/src/mailman/bin/import.py index d4173aef2..7ed2d830e 100644 --- a/src/mailman/bin/import.py +++ b/src/mailman/bin/import.py @@ -18,7 +18,6 @@ """Import the XML representation of a mailing list.""" import sys -import codecs import optparse import traceback @@ -27,7 +26,6 @@ from xml.parsers.expat import ExpatError from mailman import Defaults from mailman import MemberAdaptor -from mailman import Utils from mailman import passwords from mailman.MailList import MailList from mailman.core.i18n import _ diff --git a/src/mailman/bin/set_members.py b/src/mailman/bin/set_members.py index 15ebeb29a..ef65d1b45 100644 --- a/src/mailman/bin/set_members.py +++ b/src/mailman/bin/set_members.py @@ -20,7 +20,6 @@ import optparse from zope.component import getUtility -from mailman import Message from mailman import Utils from mailman import passwords from mailman.app.membership import add_member diff --git a/src/mailman/chains/hold.py b/src/mailman/chains/hold.py index 00dd9cf4b..7b78f118e 100644 --- a/src/mailman/chains/hold.py +++ b/src/mailman/chains/hold.py @@ -35,7 +35,6 @@ from zope.component import getUtility from zope.event import notify from zope.interface import implements -from mailman.Utils import maketext, oneline, wrap from mailman.app.moderator import hold_message from mailman.app.replybot import can_acknowledge from mailman.chains.base import ChainNotification, TerminalChainBase @@ -46,6 +45,8 @@ from mailman.interfaces.autorespond import IAutoResponseSet, Response from mailman.interfaces.languages import ILanguageManager from mailman.interfaces.pending import IPendable, IPendings from mailman.interfaces.usermanager import IUserManager +from mailman.utilities.i18n import make +from mailman.utilities.string import oneline, wrap log = logging.getLogger('mailman.vette') @@ -102,14 +103,13 @@ def autorespond_to_sender(mlist, sender, lang=None): log.info('hold autoresponse limit hit: %s', sender) response_set.response_sent(address, Response.hold) # Send this notification message instead. - text = maketext( - 'nomoretoday.txt', - {'sender' : sender, - 'listname': mlist.fqdn_listname, - 'num' : todays_count, - 'owneremail': mlist.owner_address, - }, - lang=lang) + text = make('nomoretoday.txt', + language=lang, + sender=sender, + listname=mlist.fqdn_listname, + num=todays_count, + owneremail=mlist.owner_address, + ) with _.using(lang.code): msg = UserNotification( sender, mlist.owner_address, @@ -193,8 +193,10 @@ class HoldChain(TerminalChainBase): subject = _( 'Your message to $mlist.fqdn_listname awaits moderator approval') send_language_code = msgdata.get('lang', language.code) - text = maketext('postheld.txt', substitutions, - lang=send_language_code, mlist=mlist) + text = make('postheld.txt', + mailing_list=mlist, + language=send_language_code, + **substitutions) adminaddr = mlist.bounces_address nmsg = UserNotification( msg.sender, adminaddr, subject, text, @@ -221,10 +223,11 @@ class HoldChain(TerminalChainBase): mlist.owner_address, subject, lang=language) nmsg.set_type('multipart/mixed') - text = MIMEText( - maketext('postauth.txt', substitutions, - raw=True, mlist=mlist), - _charset=charset) + text = MIMEText(make('postauth.txt', + mailing_list=mlist, + wrap=False, + **substitutions), + _charset=charset) dmsg = MIMEText(wrap(_("""\ If you reply to this message, keeping the Subject: header intact, Mailman will discard the held message. Do this if the message is spam. If you reply to diff --git a/src/mailman/commands/cli_lists.py b/src/mailman/commands/cli_lists.py index a127dd816..021ce5991 100644 --- a/src/mailman/commands/cli_lists.py +++ b/src/mailman/commands/cli_lists.py @@ -30,7 +30,6 @@ __all__ = [ from zope.component import getUtility from zope.interface import implements -from mailman.Utils import maketext from mailman.app.lifecycle import create_list, remove_list from mailman.config import config from mailman.core.constants import system_preferences @@ -42,6 +41,7 @@ from mailman.interfaces.domain import ( BadDomainSpecificationError, IDomainManager) from mailman.interfaces.languages import ILanguageManager from mailman.interfaces.listmanager import IListManager, ListAlreadyExistsError +from mailman.utilities.i18n import make @@ -213,7 +213,7 @@ class Create: requestaddr = mlist.request_address, siteowner = mlist.no_reply_address, ) - text = maketext('newlist.txt', d, mlist=mlist) + text = make('newlist.txt', mailing_list=mlist, **d) # Set the I18N language to the list's preferred language so the # header will match the template language. Stashing and restoring # the old translation context is just (healthy? :) paranoia. diff --git a/src/mailman/commands/docs/info.txt b/src/mailman/commands/docs/info.txt index bccb78fda..12fce3223 100644 --- a/src/mailman/commands/docs/info.txt +++ b/src/mailman/commands/docs/info.txt @@ -74,6 +74,7 @@ The File System Hierarchy layout is the same every by definition. PUBLIC_ARCHIVE_FILE_DIR = /var/lib/mailman/archives/public QUEUE_DIR = /var/spool/mailman SITE_PW_FILE = /var/lib/mailman/data/adm.pw + TEMPLATE_DIR = .../mailman/templates VAR_DIR = /var/lib/mailman diff --git a/src/mailman/config/config.py b/src/mailman/config/config.py index 4cfb6b5a5..636b9ef9e 100644 --- a/src/mailman/config/config.py +++ b/src/mailman/config/config.py @@ -34,6 +34,8 @@ from string import Template from zope.component import getUtility from zope.interface import Interface, implements +import mailman.templates + from mailman import version from mailman.interfaces.languages import ILanguageManager from mailman.styles.manager import StyleManager @@ -176,6 +178,10 @@ class Configuration: pipermail_public_dir = category.pipermail_public_dir, queue_dir = category.queue_dir, var_dir = var_dir, + template_dir = ( + os.path.dirname(mailman.templates.__file__) + if category.template_dir == ':source:' + else category.template_dir), # Files. creator_pw_file = category.creator_pw_file, lock_file = category.lock_file, diff --git a/src/mailman/config/configure.zcml b/src/mailman/config/configure.zcml index a7d12e7a2..3b4497ab8 100644 --- a/src/mailman/config/configure.zcml +++ b/src/mailman/config/configure.zcml @@ -6,14 +6,19 @@ <adapter for="mailman.interfaces.mailinglist.IMailingList" - provides="mailman.interfaces.autorespond.IAutoResponseSet" factory="mailman.model.autorespond.AutoResponseSet" + provides="mailman.interfaces.autorespond.IAutoResponseSet" /> <adapter for="mailman.interfaces.mailinglist.IMailingList" - provides="mailman.interfaces.mailinglist.IAcceptableAliasSet" factory="mailman.model.mailinglist.AcceptableAliasSet" + provides="mailman.interfaces.mailinglist.IAcceptableAliasSet" + /> + + <utility + factory="mailman.model.bans.BanManager" + provides="mailman.interfaces.bans.IBanManager" /> <utility @@ -52,8 +57,8 @@ /> <utility - provides="mailman.interfaces.registrar.IRegistrar" factory="mailman.app.registrar.Registrar" + provides="mailman.interfaces.registrar.IRegistrar" /> <utility @@ -61,4 +66,9 @@ provides="mailman.interfaces.membership.ISubscriptionService" /> + <utility + factory="mailman.email.validate.Validator" + provides="mailman.interfaces.address.IEmailValidator" + /> + </configure> diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg index f789d28f9..09f575459 100644 --- a/src/mailman/config/schema.cfg +++ b/src/mailman/config/schema.cfg @@ -103,6 +103,12 @@ pipermail_public_dir: $var_dir/archives/public # Directory for private Pipermail archiver artifacts. pipermail_private_dir: $var_dir/archives/private # +# Where Mailman looks for its templates. This can either be a file system +# path or the special symbol ':source:' to locate them within the source tree +# (specifically, inside the mailman.templates package directory). +# +template_dir: :source: +# # There are also a number of paths to specific file locations that can be # defined. For these, the directory containing the file must already exist, # or be one of the directories created by Mailman as per above. diff --git a/src/mailman/database/mailman.sql b/src/mailman/database/mailman.sql index bb7f9dc57..9d2b015b7 100644 --- a/src/mailman/database/mailman.sql +++ b/src/mailman/database/mailman.sql @@ -116,8 +116,7 @@ CREATE TABLE mailinglist ( autorespond_requests INTEGER, autoresponse_request_text TEXT, autoresponse_grace_period TEXT, - -- Bounce and ban. - ban_list BLOB, + -- Bounces. bounce_info_stale_after TEXT, bounce_matching_headers TEXT, bounce_notify_owner_on_disable BOOLEAN, @@ -270,3 +269,10 @@ CREATE INDEX ix_member_address_id ON member (address_id); CREATE INDEX ix_member_preferences_id ON member (preferences_id); CREATE INDEX ix_pendedkeyvalue_pended_id ON pendedkeyvalue (pended_id); CREATE INDEX ix_user_preferences_id ON user (preferences_id); + +CREATE TABLE ban ( + id INTEGER NOT NULL, + email TEXT, + mailing_list TEXT, + PRIMARY KEY (id) +); diff --git a/src/mailman/docs/NEWS.txt b/src/mailman/docs/NEWS.txt index ed91fb7aa..f86e805f4 100644 --- a/src/mailman/docs/NEWS.txt +++ b/src/mailman/docs/NEWS.txt @@ -60,6 +60,7 @@ Build Bugs fixed ---------- * Typo in scan_message(). (LP: #645897) + * Typo in add_member(). (LP: #710182) (Florian Fuchs) * Clean up many pyflakes problems. diff --git a/src/mailman/email/validate.py b/src/mailman/email/validate.py index 4eb13eee8..0c4c6a5e0 100644 --- a/src/mailman/email/validate.py +++ b/src/mailman/email/validate.py @@ -21,15 +21,18 @@ from __future__ import absolute_import, unicode_literals __metaclass__ = type __all__ = [ - 'is_valid', - 'validate', + 'Validator', ] import re +from zope.interface import implements + + from mailman.email.utils import split_email -from mailman.interfaces.address import InvalidEmailAddressError +from mailman.interfaces.address import ( + IEmailValidator, InvalidEmailAddressError) # What other characters should be disallowed? @@ -37,34 +40,31 @@ _badchars = re.compile(r'[][()<>|;^,\000-\037\177-\377]') -def validate(address): - """Validate an email address. +class Validator: + """An email address validator.""" - :param address: An email address. - :type address: string - :raise InvalidEmailAddressError: when the address is deemed invalid. - """ - if not is_valid(address): - raise InvalidEmailAddressError(repr(address)) + implements(IEmailValidator) + def is_valid(self, email): + """See `IEmailValidator`.""" + if not email or ' ' in email: + return False + if _badchars.search(email) or email[0] == '-': + return False + user, domain_parts = split_email(email) + # Local, unqualified addresses are not allowed. + if not domain_parts: + return False + if len(domain_parts) < 2: + return False + return True - -def is_valid(address): - """Check if an email address if valid. + def validate(self, email): + """Validate an email address. - :param address: An email address. - :type address: string - :return: A flag indicating whether the email address is okay or not. - :rtype: bool - """ - if not address or ' ' in address: - return False - if _badchars.search(address) or address[0] == '-': - return False - user, domain_parts = split_email(address) - # Local, unqualified addresses are not allowed. - if not domain_parts: - return False - if len(domain_parts) < 2: - return False - return True + :param address: An email address. + :type address: string + :raise InvalidEmailAddressError: when the address is deemed invalid. + """ + if not self.is_valid(email): + raise InvalidEmailAddressError(repr(email)) diff --git a/src/mailman/interfaces/address.py b/src/mailman/interfaces/address.py index 446bae3f3..391eae849 100644 --- a/src/mailman/interfaces/address.py +++ b/src/mailman/interfaces/address.py @@ -26,6 +26,7 @@ __all__ = [ 'AddressNotLinkedError', 'ExistingAddressError', 'IAddress', + 'IEmailValidator', 'InvalidEmailAddressError', ] @@ -104,3 +105,25 @@ class IAddress(Interface): preferences = Attribute( """This address's preferences.""") + + + +class IEmailValidator(Interface): + """An email validator.""" + + def is_valid(email): + """Check if an email address if valid. + + :param email: A text email address. + :type email: str + :return: A flag indicating whether the email address is okay or not. + :rtype: bool + """ + + def validate(email): + """Validate an email address. + + :param email: A text email address. + :type email: str + :raise InvalidEmailAddressError: when `email` is deemed invalid. + """ diff --git a/src/mailman/interfaces/bans.py b/src/mailman/interfaces/bans.py new file mode 100644 index 000000000..d14109cc1 --- /dev/null +++ b/src/mailman/interfaces/bans.py @@ -0,0 +1,110 @@ +# Copyright (C) 2011 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Manager of email address bans.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'IBan', + 'IBanManager', + ] + + +from zope.interface import Attribute, Interface + + + +class IBan(Interface): + """A specific ban. + + In general, this interface isn't publicly useful. + """ + + email = Attribute('The banned email address, or pattern.') + + mailing_list = Attribute( + """The fqdn name of the mailing list the ban applies to. + + Use None if this is a global ban. + """) + + + +class IBanManager(Interface): + """The global manager of email address bans.""" + + def ban(email, mailing_list=None): + """Ban an email address from subscribing to a mailing list. + + When an email address is banned, it will not be allowed to subscribe + to a the named mailing list. This does not affect any email address + already subscribed to the mailing list. With the default arguments, + an email address can be banned globally from subscribing to any + mailing list on the system. + + It is also possible to add a 'ban pattern' whereby all email addresses + matching a Python regular expression can be banned. This is + accomplished by using a `^` as the first character in `email`. + + When an email address is already banned for the given mailing list (or + globally), then this method does nothing. However, it is possible to + extend a ban for a specific mailing list into a global ban; both bans + would be in place and they can be removed individually. + + :param email: The text email address being banned or, if the string + starts with a caret (^), the email address pattern to ban. + :type email: str + :param mailing_list: The fqdn name of the mailing list to which the + ban applies. If None, then the ban is global. + :type mailing_list: string + """ + + def unban(email, mailing_list=None): + """Remove an email address ban. + + This removes a specific or global email address ban, which would have + been added with the `ban()` method. If a ban is lifted which did not + exist, this method does nothing. + + :param email: The text email address being unbanned or, if the string + starts with a caret (^), the email address pattern to unban. + :type email: str + :param mailing_list: The fqdn name of the mailing list to which the + unban applies. If None, then the unban is global. + :type mailing_list: string + """ + + def is_banned(email, mailing_list=None): + """Check whether a specific email address is banned. + + `email` must be a text email address; it cannot be a pattern. The + given email address is checked against all registered bans, both + specific and regular expression, both for the named mailing list (if + given), and globally. + + :param email: The text email address being checked. + :type email: str + :param mailing_list: The fqdn name of the mailing list being checked. + Note that if not None, both specific and global bans will be + checked. If None, then only global bans will be checked. + :type mailing_list: string + :return: A flag indicating whether the given email address is banned + or not. + :rtype: bool + """ diff --git a/src/mailman/model/bans.py b/src/mailman/model/bans.py new file mode 100644 index 000000000..95711cfa9 --- /dev/null +++ b/src/mailman/model/bans.py @@ -0,0 +1,110 @@ +# Copyright (C) 2011 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Ban manager.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'BanManager', + ] + + +import re + +from storm.locals import Int, Unicode +from zope.interface import implements + +from mailman.config import config +from mailman.database.model import Model +from mailman.interfaces.bans import IBan, IBanManager + + + +class Ban(Model): + implements(IBan) + + id = Int(primary=True) + email = Unicode() + mailing_list = Unicode() + + def __init__(self, email, mailing_list): + super(Ban, self).__init__() + self.email = email + self.mailing_list = mailing_list + + + +class BanManager: + implements(IBanManager) + + def ban(self, email, mailing_list=None): + """See `IBanManager`.""" + bans = config.db.store.find( + Ban, email=email, mailing_list=mailing_list) + if bans.count() == 0: + ban = Ban(email, mailing_list) + config.db.store.add(ban) + + def unban(self, email, mailing_list=None): + """See `IBanManager`.""" + ban = config.db.store.find( + Ban, email=email, mailing_list=mailing_list).one() + if ban is not None: + config.db.store.remove(ban) + + def is_banned(self, email, mailing_list=None): + """See `IBanManager`.""" + # A specific mailing list ban is being checked, however the email + # address could be banned specifically, or globally. + if mailing_list is not None: + # Try specific bans first. + bans = config.db.store.find( + Ban, email=email, mailing_list=mailing_list) + if bans.count() > 0: + return True + # Try global bans next. + bans = config.db.store.find(Ban, email=email, mailing_list=None) + if bans.count() > 0: + return True + # Now try specific mailing list bans, but with a pattern. + bans = config.db.store.find(Ban, mailing_list=mailing_list) + for ban in bans: + if (ban.email.startswith('^') and + re.match(ban.email, email, re.IGNORECASE) is not None): + return True + # And now try global pattern bans. + bans = config.db.store.find(Ban, mailing_list=None) + for ban in bans: + if (ban.email.startswith('^') and + re.match(ban.email, email, re.IGNORECASE) is not None): + return True + else: + # The client is asking for global bans. Look up bans on the + # specific email address first. + bans = config.db.store.find( + Ban, email=email, mailing_list=None) + if bans.count() > 0: + return True + # And now look for global pattern bans. + bans = config.db.store.find(Ban, mailing_list=None) + for ban in bans: + if (ban.email.startswith('^') and + re.match(ban.email, email, re.IGNORECASE) is not None): + return True + return False diff --git a/src/mailman/model/mailinglist.py b/src/mailman/model/mailinglist.py index cdebd5ca6..488b6da3d 100644 --- a/src/mailman/model/mailinglist.py +++ b/src/mailman/model/mailinglist.py @@ -108,8 +108,7 @@ class MailingList(Model): filter_content = Bool() collapse_alternatives = Bool() convert_html_to_plaintext = Bool() - # Bounces and bans. - ban_list = Pickle() # XXX + # Bounces. bounce_info_stale_after = TimeDelta() # XXX bounce_matching_headers = Unicode() # XXX bounce_notify_owner_on_disable = Bool() # XXX diff --git a/src/mailman/pipeline/acknowledge.py b/src/mailman/pipeline/acknowledge.py index 744238f44..e5b49ffa0 100644 --- a/src/mailman/pipeline/acknowledge.py +++ b/src/mailman/pipeline/acknowledge.py @@ -31,11 +31,12 @@ __all__ = [ from zope.component import getUtility from zope.interface import implements -from mailman import Utils from mailman.core.i18n import _ from mailman.email.message import UserNotification from mailman.interfaces.handler import IHandler from mailman.interfaces.languages import ILanguageManager +from mailman.utilities.i18n import make +from mailman.utilities.string import oneline @@ -70,13 +71,15 @@ class Acknowledge: charset = language_manager[language.code].charset # Now get the acknowledgement template. realname = mlist.real_name - text = Utils.maketext( - 'postack.txt', - {'subject' : Utils.oneline(original_subject, charset), - 'listname' : realname, - 'listinfo_url': mlist.script_url('listinfo'), - 'optionsurl' : member.options_url, - }, lang=language.code, mlist=mlist, raw=True) + text = make('postack.txt', + mailing_list=mlist, + language=language.code, + wrap=False, + subject=oneline(original_subject, charset), + listname=realname, + listinfo_url=mlist.script_url('listinfo'), + optionsurl=member.options_url, + ) # Craft the outgoing message, with all headers and attributes # necessary for general delivery. Then enqueue it to the outgoing # queue. diff --git a/src/mailman/pipeline/calculate_recipients.py b/src/mailman/pipeline/calculate_recipients.py index 36f391fd0..d2527e608 100644 --- a/src/mailman/pipeline/calculate_recipients.py +++ b/src/mailman/pipeline/calculate_recipients.py @@ -32,12 +32,12 @@ __all__ = [ from zope.interface import implements -from mailman import Utils from mailman.config import config from mailman.core import errors from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler from mailman.interfaces.member import DeliveryStatus +from mailman.utilities.string import wrap @@ -89,7 +89,7 @@ class CalculateRecipients: Your urgent message to the $realname mailing list was not authorized for delivery. The original message as received by Mailman is attached. """) - raise errors.RejectMessage(Utils.wrap(text)) + raise errors.RejectMessage(wrap(text)) # Calculate the regular recipients of the message recipients = set(member.address.email for member in mlist.regular_members.members diff --git a/src/mailman/pipeline/mime_delete.py b/src/mailman/pipeline/mime_delete.py index 3066f4003..43197a878 100644 --- a/src/mailman/pipeline/mime_delete.py +++ b/src/mailman/pipeline/mime_delete.py @@ -41,12 +41,12 @@ from email.Iterators import typed_subpart_iterator from os.path import splitext from zope.interface import implements -from mailman.Utils import oneline from mailman.config import config from mailman.core import errors from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler from mailman.queue import Switchboard +from mailman.utilities.string import oneline from mailman.version import VERSION diff --git a/src/mailman/pipeline/replybot.py b/src/mailman/pipeline/replybot.py index 5f17160c1..dffffbf47 100644 --- a/src/mailman/pipeline/replybot.py +++ b/src/mailman/pipeline/replybot.py @@ -30,7 +30,6 @@ import logging from zope.component import getUtility from zope.interface import implements -from mailman import Utils from mailman.core.i18n import _ from mailman.email.message import UserNotification from mailman.interfaces.autorespond import ( @@ -38,7 +37,7 @@ from mailman.interfaces.autorespond import ( from mailman.interfaces.handler import IHandler from mailman.interfaces.usermanager import IUserManager from mailman.utilities.datetime import today -from mailman.utilities.string import expand +from mailman.utilities.string import expand, wrap log = logging.getLogger('mailman.error') @@ -113,7 +112,7 @@ class Replybot: owneremail = mlist.owner_address, ) # Interpolation and Wrap the response text. - text = Utils.wrap(expand(response_text, d)) + text = wrap(expand(response_text, d)) outmsg = UserNotification(msg.sender, mlist.bounces_address, subject, text, mlist.preferred_language) outmsg['X-Mailer'] = _('The Mailman Replybot') diff --git a/src/mailman/pipeline/scrubber.py b/src/mailman/pipeline/scrubber.py index a102d9f6b..f25c2978c 100644 --- a/src/mailman/pipeline/scrubber.py +++ b/src/mailman/pipeline/scrubber.py @@ -40,13 +40,13 @@ from mimetypes import guess_all_extensions from string import Template from zope.interface import implements -from mailman.Utils import oneline, websafe from mailman.config import config from mailman.core.errors import DiscardMessage from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler from mailman.utilities.filesystem import makedirs from mailman.utilities.modules import find_name +from mailman.utilities.string import oneline, websafe # Path characters for common platforms diff --git a/src/mailman/queue/__init__.py b/src/mailman/queue/__init__.py index 901e99cfc..8abc5e9a6 100644 --- a/src/mailman/queue/__init__.py +++ b/src/mailman/queue/__init__.py @@ -462,6 +462,7 @@ class Runner: def _clean_up(self): """See `IRunner`.""" + pass def _dispose(self, mlist, msg, msgdata): """See `IRunner`.""" diff --git a/src/mailman/queue/archive.py b/src/mailman/queue/archive.py index 24dab34f5..99682f310 100644 --- a/src/mailman/queue/archive.py +++ b/src/mailman/queue/archive.py @@ -27,7 +27,7 @@ import os import logging from datetime import datetime -from email.Utils import parsedate_tz, mktime_tz, formatdate +from email.utils import parsedate_tz, mktime_tz, formatdate from flufl.lock import Lock from lazr.config import as_timedelta diff --git a/src/mailman/queue/bounce.py b/src/mailman/queue/bounce.py index a53b2d072..968659352 100644 --- a/src/mailman/queue/bounce.py +++ b/src/mailman/queue/bounce.py @@ -23,7 +23,7 @@ import cPickle import logging import datetime -from email.Utils import parseaddr +from email.utils import parseaddr from lazr.config import as_timedelta from mailman.config import config diff --git a/src/mailman/queue/digest.py b/src/mailman/queue/digest.py index aaade6c4d..075335158 100644 --- a/src/mailman/queue/digest.py +++ b/src/mailman/queue/digest.py @@ -38,7 +38,6 @@ from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.utils import formatdate, getaddresses, make_msgid -from mailman.Utils import maketext, oneline, wrap from mailman.config import config from mailman.core.errors import DiscardMessage from mailman.core.i18n import _ @@ -46,7 +45,9 @@ from mailman.interfaces.member import DeliveryMode, DeliveryStatus from mailman.pipeline.decorate import decorate from mailman.pipeline.scrubber import process as scrubber from mailman.queue import Runner +from mailman.utilities.i18n import make from mailman.utilities.mailbox import Mailbox +from mailman.utilities.string import oneline, wrap @@ -75,15 +76,14 @@ class Digester: # digest header are separate MIME subobjects. In either case, it's # the first thing in the digest, and we can calculate it now, so go # ahead and add it now. - self._masthead = maketext( - 'masthead.txt', dict( - real_name=mlist.real_name, - got_list_email=mlist.posting_address, - got_listinfo_url=mlist.script_url('listinfo'), - got_request_email=mlist.request_address, - got_owner_email=mlist.owner_address, - ), - mlist=mlist) + self._masthead = make('masthead.txt', + mailing_list=mlist, + real_name=mlist.real_name, + got_list_email=mlist.posting_address, + got_listinfo_url=mlist.script_url('listinfo'), + got_request_email=mlist.request_address, + got_owner_email=mlist.owner_address, + ) # Set things up for the table of contents. self._header = decorate(mlist, mlist.digest_header) self._toc = StringIO() diff --git a/src/mailman/queue/maildir.py b/src/mailman/queue/maildir.py index 3d0b9497c..fe4dc9da1 100644 --- a/src/mailman/queue/maildir.py +++ b/src/mailman/queue/maildir.py @@ -53,8 +53,8 @@ import os import errno import logging -from email.Parser import Parser -from email.Utils import parseaddr +from email.parser import Parser +from email.utils import parseaddr from mailman.config import config from mailman.message import Message diff --git a/src/mailman/queue/news.py b/src/mailman/queue/news.py index dcb0deba6..a3d915244 100644 --- a/src/mailman/queue/news.py +++ b/src/mailman/queue/news.py @@ -24,8 +24,8 @@ import logging import nntplib from cStringIO import StringIO +from lazr.config import as_host_port -from mailman import Utils from mailman.config import config from mailman.interfaces.nntp import NewsModeration from mailman.queue import Runner @@ -61,7 +61,8 @@ class NewsRunner(Runner): conn = None try: try: - nntp_host, nntp_port = Utils.nntpsplit(mlist.nntp_host) + nntp_host, nntp_port = as_host_port( + mlist.nntp_host, default_port=119) conn = nntplib.NNTP(nntp_host, nntp_port, readermode=True, user=config.nntp.username, diff --git a/src/mailman/styles/default.py b/src/mailman/styles/default.py index f35b2519a..4f2b3d7d5 100644 --- a/src/mailman/styles/default.py +++ b/src/mailman/styles/default.py @@ -131,8 +131,6 @@ ${listinfo_page} mlist.forward_auto_discards = True mlist.generic_nonmember_action = 1 mlist.nonmember_rejection_notice = '' - # Ban lists - mlist.ban_list = [] # Max autoresponses per day. A mapping between addresses and a # 2-tuple of the date of the last autoresponse and the number of # autoresponses sent on that date. diff --git a/src/mailman/testing/helpers.py b/src/mailman/testing/helpers.py index fd2b9ffb3..2ba778813 100644 --- a/src/mailman/testing/helpers.py +++ b/src/mailman/testing/helpers.py @@ -26,6 +26,7 @@ __all__ = [ 'get_lmtp_client', 'get_queue_messages', 'make_testable_runner', + 'reset_the_world', 'subscribe', 'wait_for_webservice', ] @@ -47,6 +48,7 @@ from zope.component import getUtility from mailman.bin.master import Loop as Master from mailman.config import config from mailman.interfaces.member import MemberRole +from mailman.interfaces.messages import IMessageStore from mailman.interfaces.usermanager import IUserManager from mailman.utilities.mailbox import Mailbox @@ -277,3 +279,31 @@ def subscribe(mlist, first_name, role=MemberRole.member): preferred_address = list(person.addresses)[0] preferred_address.subscribe(mlist, role) config.db.commit() + + + +def reset_the_world(): + """Reset everything: + + * Clear out the database + * Remove all residual queue files + * Clear the message store + * Reset the global style manager + + This should be as thorough a reset of the system as necessary to keep + tests isolated. + """ + # Reset the database between tests. + config.db._reset() + # Remove all residual queue files. + for dirpath, dirnames, filenames in os.walk(config.QUEUE_DIR): + for filename in filenames: + os.remove(os.path.join(dirpath, filename)) + # Clear out messages in the message store. + message_store = getUtility(IMessageStore) + for message in message_store.messages: + message_store.delete_message(message['message-id']) + config.db.commit() + # Reset the global style manager. + config.style_manager.populate() + diff --git a/src/mailman/testing/layers.py b/src/mailman/testing/layers.py index 319248ebb..353dd9edd 100644 --- a/src/mailman/testing/layers.py +++ b/src/mailman/testing/layers.py @@ -46,8 +46,7 @@ from mailman.core import initialize from mailman.core.initialize import INHIBIT_CONFIG_FILE from mailman.core.logging import get_handler from mailman.interfaces.domain import IDomainManager -from mailman.interfaces.messages import IMessageStore -from mailman.testing.helpers import TestableMaster +from mailman.testing.helpers import TestableMaster, reset_the_world from mailman.testing.mta import ConnectionCountingController from mailman.utilities.datetime import factory from mailman.utilities.string import expand @@ -179,19 +178,7 @@ class ConfigLayer(MockAndMonkeyLayer): @classmethod def testTearDown(cls): - # Reset the database between tests. - config.db._reset() - # Remove all residual queue files. - for dirpath, dirnames, filenames in os.walk(config.QUEUE_DIR): - for filename in filenames: - os.remove(os.path.join(dirpath, filename)) - # Clear out messages in the message store. - message_store = getUtility(IMessageStore) - for message in message_store.messages: - message_store.delete_message(message['message-id']) - config.db.commit() - # Reset the global style manager. - config.style_manager.populate() + reset_the_world() # Flag to indicate that loggers should propagate to the console. stderr = False diff --git a/src/mailman/tests/test_documentation.py b/src/mailman/tests/test_documentation.py index e65885d79..f723ecf3e 100644 --- a/src/mailman/tests/test_documentation.py +++ b/src/mailman/tests/test_documentation.py @@ -254,7 +254,7 @@ def test_suite(): layer = getattr(sys.modules[package_path], 'layer', SMTPLayer) for filename in os.listdir(docsdir): base, extension = os.path.splitext(filename) - if os.path.splitext(filename)[1] == '.txt': + if os.path.splitext(filename)[1] in ('.txt', '.rst'): module_path = package_path + '.' + base doctest_files[module_path] = ( os.path.join(docsdir, filename), layer) diff --git a/src/mailman/tests/test_membership.py b/src/mailman/tests/test_membership.py index c3e40cfcc..8425cf65f 100644 --- a/src/mailman/tests/test_membership.py +++ b/src/mailman/tests/test_membership.py @@ -28,11 +28,11 @@ __all__ = [ import time import unittest -from mailman import passwords from mailman.app.lifecycle import create_list, remove_list from mailman.config import config from mailman.interfaces.member import NotAMemberError from mailman.testing.layers import ConfigLayer +from mailman.utilities import passwords diff --git a/src/mailman/tests/test_security_mgr.py b/src/mailman/tests/test_security_mgr.py deleted file mode 100644 index a4f9c1cf4..000000000 --- a/src/mailman/tests/test_security_mgr.py +++ /dev/null @@ -1,241 +0,0 @@ -# Copyright (C) 2001-2011 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Unit tests for the SecurityManager module.""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'test_suite', - ] - - -import os -import errno -import unittest - -# Don't use cStringIO because we're going to inherit -from StringIO import StringIO - -from mailman import Utils -from mailman import passwords -from mailman.config import config - - - -def password(cleartext): - return passwords.make_secret(cleartext, passwords.Schemes.ssha) - - - -class TestSecurityManager(unittest.TestCase): - def test_init_vars(self): - eq = self.assertEqual - eq(self._mlist.mod_password, None) - eq(self._mlist.passwords, {}) - - def test_auth_context_info_authuser(self): - mlist = self._mlist - self.assertRaises(TypeError, mlist.AuthContextInfo, config.AuthUser) - # Add a member - mlist.addNewMember('aperson@dom.ain', password='xxXXxx') - self.assertEqual( - mlist.AuthContextInfo(config.AuthUser, 'aperson@dom.ain'), - ('_xtest%40example.com+user+aperson--at--dom.ain', 'xxXXxx')) - - def test_auth_context_moderator(self): - mlist = self._mlist - mlist.mod_password = 'yyYYyy' - self.assertEqual( - mlist.AuthContextInfo(config.AuthListModerator), - ('_xtest%40example.com+moderator', 'yyYYyy')) - - def test_auth_context_admin(self): - mlist = self._mlist - mlist.password = 'zzZZzz' - self.assertEqual( - mlist.AuthContextInfo(config.AuthListAdmin), - ('_xtest%40example.com+admin', 'zzZZzz')) - - def test_auth_context_site(self): - mlist = self._mlist - mlist.password = 'aaAAaa' - self.assertEqual( - mlist.AuthContextInfo(config.AuthSiteAdmin), - ('_xtest%40example.com+admin', 'aaAAaa')) - - def test_auth_context_huh(self): - self.assertEqual( - self._mlist.AuthContextInfo('foo'), - (None, None)) - - - -class TestAuthenticate(unittest.TestCase): - def setUp(self): - Utils.set_global_password('bbBBbb', siteadmin=True) - Utils.set_global_password('ccCCcc', siteadmin=False) - - def tearDown(self): - try: - os.unlink(config.SITE_PW_FILE) - except OSError, e: - if e.errno <> errno.ENOENT: - raise - try: - os.unlink(config.LISTCREATOR_PW_FILE) - except OSError, e: - if e.errno <> errno.ENOENT: - raise - - def test_auth_creator(self): - self.assertEqual(self._mlist.Authenticate( - [config.AuthCreator], 'ccCCcc'), config.AuthCreator) - - def test_auth_creator_unauth(self): - self.assertEqual(self._mlist.Authenticate( - [config.AuthCreator], 'xxxxxx'), config.UnAuthorized) - - def test_auth_site_admin(self): - self.assertEqual(self._mlist.Authenticate( - [config.AuthSiteAdmin], 'bbBBbb'), config.AuthSiteAdmin) - - def test_auth_site_admin_unauth(self): - self.assertEqual(self._mlist.Authenticate( - [config.AuthSiteAdmin], 'xxxxxx'), config.UnAuthorized) - - def test_list_admin(self): - self._mlist.password = password('ttTTtt') - self.assertEqual(self._mlist.Authenticate( - [config.AuthListAdmin], 'ttTTtt'), config.AuthListAdmin) - - def test_list_admin_unauth(self): - self._mlist.password = password('ttTTtt') - self.assertEqual(self._mlist.Authenticate( - [config.AuthListAdmin], 'xxxxxx'), config.UnAuthorized) - - def test_list_moderator(self): - self._mlist.mod_password = password('mmMMmm') - self.assertEqual(self._mlist.Authenticate( - [config.AuthListModerator], 'mmMMmm'), config.AuthListModerator) - - def test_user(self): - mlist = self._mlist - mlist.addNewMember('aperson@dom.ain', password=password('nosrepa')) - self.assertEqual(mlist.Authenticate( - [config.AuthUser], 'nosrepa', 'aperson@dom.ain'), config.AuthUser) - - def test_wrong_user(self): - mlist = self._mlist - mlist.addNewMember('aperson@dom.ain', password='nosrepa') - self.assertEqual( - mlist.Authenticate([config.AuthUser], 'nosrepa', 'bperson@dom.ain'), - config.UnAuthorized) - - def test_no_user(self): - mlist = self._mlist - mlist.addNewMember('aperson@dom.ain', password='nosrepa') - self.assertEqual(mlist.Authenticate([config.AuthUser], 'norespa'), - config.UnAuthorized) - - def test_user_unauth(self): - mlist = self._mlist - mlist.addNewMember('aperson@dom.ain', password='nosrepa') - self.assertEqual(mlist.Authenticate( - [config.AuthUser], 'xxxxxx', 'aperson@dom.ain'), - config.UnAuthorized) - - def test_value_error(self): - self.assertRaises(ValueError, self._mlist.Authenticate, - ['spooge'], 'xxxxxx', 'zperson@dom.ain') - - - -class StripperIO(StringIO): - HEAD = 'Set-Cookie: ' - def write(self, s): - if s.startswith(self.HEAD): - s = s[len(self.HEAD):] - StringIO.write(self, s) - - -class TestWebAuthenticate(unittest.TestCase): - def setUp(self): - Utils.set_global_password('bbBBbb', siteadmin=True) - Utils.set_global_password('ccCCcc', siteadmin=False) - mlist = self._mlist - mlist.mod_password = password('abcdefg') - mlist.addNewMember('aperson@dom.ain', password='qqQQqq') - # Set up the cookie data - sfp = StripperIO() - print >> sfp, mlist.MakeCookie(config.AuthSiteAdmin) - # AuthCreator isn't handled in AuthContextInfo() - print >> sfp, mlist.MakeCookie(config.AuthListAdmin) - print >> sfp, mlist.MakeCookie(config.AuthListModerator) - print >> sfp, mlist.MakeCookie(config.AuthUser, 'aperson@dom.ain') - # Strip off the "Set-Cookie: " prefix - cookie = sfp.getvalue() - os.environ['HTTP_COOKIE'] = cookie - - def tearDown(self): - try: - os.unlink(config.SITE_PW_FILE) - except OSError, e: - if e.errno <> errno.ENOENT: - raise - try: - os.unlink(config.LISTCREATOR_PW_FILE) - except OSError, e: - if e.errno <> errno.ENOENT: - raise - del os.environ['HTTP_COOKIE'] - - def test_auth_site_admin(self): - self.failUnless(self._mlist.WebAuthenticate( - [config.AuthSiteAdmin], 'does not matter')) - - def test_list_admin(self): - self.failUnless(self._mlist.WebAuthenticate( - [config.AuthListAdmin], 'does not matter')) - - def test_list_moderator(self): - self.failUnless(self._mlist.WebAuthenticate( - [config.AuthListModerator], 'does not matter')) - - def test_user(self): - self.failUnless(self._mlist.WebAuthenticate( - [config.AuthUser], 'does not matter')) - - def test_not_a_user(self): - self._mlist.removeMember('aperson@dom.ain') - self.failIf(self._mlist.WebAuthenticate( - [config.AuthUser], 'does not matter', 'aperson@dom.ain')) - - - -# TBD: Tests for MakeCookie(), ZapCookie(), CheckCookie() -- although the -# latter is implicitly tested by testing WebAuthenticate() above. - - - -def test_suite(): - suite = unittest.TestSuite() -## suite.addTest(unittest.makeSuite(TestSecurityManager)) -## suite.addTest(unittest.makeSuite(TestAuthenticate)) -## suite.addTest(unittest.makeSuite(TestWebAuthenticate)) - return suite diff --git a/src/mailman/utilities/i18n.py b/src/mailman/utilities/i18n.py new file mode 100644 index 000000000..8e769329c --- /dev/null +++ b/src/mailman/utilities/i18n.py @@ -0,0 +1,196 @@ +# Copyright (C) 2011 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""i18n template search and interpolation.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TemplateNotFoundError', + 'find', + 'make', + ] + + +import os +import errno + +from itertools import product + +from mailman.config import config +from mailman.core.constants import system_preferences +from mailman.core.i18n import _ +from mailman.utilities.string import expand, wrap as wrap_text + + + +class TemplateNotFoundError(Exception): + """The named template was not found.""" + + def __init__(self, template_file): + self.template_file = template_file + + + +def _search(template_file, mailing_list=None, language=None): + """Generator that provides file system search order.""" + + languages = ['en', system_preferences.preferred_language.code] + if mailing_list is not None: + languages.append(mailing_list.preferred_language.code) + if language is not None: + languages.append(language) + languages.reverse() + # File system locations to search. + paths = [config.TEMPLATE_DIR, + os.path.join(config.TEMPLATE_DIR, 'site')] + if mailing_list is not None: + paths.append(os.path.join(config.TEMPLATE_DIR, + mailing_list.host_name)) + paths.append(os.path.join(config.LIST_DATA_DIR, + mailing_list.fqdn_listname)) + paths.reverse() + for language, path in product(languages, paths): + yield os.path.join(path, language, template_file) + + + +def find(template_file, mailing_list=None, language=None): + """Locate an i18n template file. + + When something in Mailman needs a template file, it always asks for the + file through this interface. The results of the search is path to the + 'matching' template, with the search order depending on whether + `mailing_list` and `language` are provided. + + When looking for a template in a specific language, there are 4 locations + that are searched, in this order: + + * The list-specific language directory + <var_dir>/lists/<fqdn_listname>/<language> + + * The domain-specific language directory + <template_dir>/<list-host-name>/<language> + + * The site-wide language directory + <template_dir>/site/<language> + + * The global default language directory + <template_dir>/<language> + + The first match stops the search. In this way, you can specialize + templates at the desired level, or if you only use the default templates, + you don't need to change anything. NEVER modify files in + <template_dir>/<language> since Mailman will overwrite these when you + upgrade. Instead you can use <template_dir>/site. + + The <language> path component is calculated as follows, in this order: + + * The `language` parameter if given + * `mailing_list.preferred_language` if given + * The server's default language + * English ('en') + + Languages are iterated after each of the four locations are searched. So + for example, when searching for the 'foo.txt' template, where the server's + default language is 'fr', the mailing list's (test@example.com) language + is 'de' and the `language` parameter is 'it', these locations are searched + in order: + + * <var_dir>/lists/test@example.com/it/foo.txt + * <template_dir>/example.com/it/foo.txt + * <template_dir>/site/it/foo.txt + * <template_dir>/it/foo.txt + + * <var_dir>/lists/test@example.com/de/foo.txt + * <template_dir>/example.com/de/foo.txt + * <template_dir>/site/de/foo.txt + * <template_dir>/de/foo.txt + + * <var_dir>/lists/test@example.com/fr/foo.txt + * <template_dir>/example.com/fr/foo.txt + * <template_dir>/site/fr/foo.txt + * <template_dir>/fr/foo.txt + + * <var_dir>/lists/test@example.com/en/foo.txt + * <template_dir>/example.com/en/foo.txt + * <template_dir>/site/en/foo.txt + * <template_dir>/en/foo.txt + + :param template_file: The name of the template file to search for. + :type template_file: string + :param mailing_list: Optional mailing list used as the context for + searching for the template file. The list's preferred language will + influence the search, as will the list's data directory. + :type mailing_list: `IMailingList` + :param language: Optional language code, which influences the search. + :type language: string + :return: A tuple of the file system path to the first matching template, + and an open file object allowing reading of the file. + :rtype: (string, file) + :raises TemplateNotFoundError: when the template could not be found. + """ + raw_search_order = _search(template_file, mailing_list, language) + for path in raw_search_order: + try: + fp = open(path) + except IOError as error: + if error.errno != errno.ENOENT: + raise + else: + return path, fp + raise TemplateNotFoundError(template_file) + + +def make(template_file, mailing_list=None, language=None, wrap=True, **kw): + """Locate and 'make' a template file. + + The template file is located as with `find()`, and the resulting text is + optionally wrapped and interpolated with the keyword argument dictionary. + + :param template_file: The name of the template file to search for. + :type template_file: string + :param mailing_list: Optional mailing list used as the context for + searching for the template file. The list's preferred language will + influence the search, as will the list's data directory. + :type mailing_list: `IMailingList` + :param language: Optional language code, which influences the search. + :type language: string + :param wrap: When True, wrap the text. + :type wrap: bool + :param **kw: Keyword arguments for template interpolation. + :return: A tuple of the file system path to the first matching template, + and an open file object allowing reading of the file. + :rtype: (string, file) + :raises TemplateNotFoundError: when the template could not be found. + """ + path, fp = find(template_file, mailing_list, language) + try: + # XXX Removing the trailing newline is a hack carried over from + # Mailman 2. The (stripped) template text is then passed through the + # translation catalog. This ensures that the translated text is + # unicode, and also allows for volunteers to translate the templates + # into the language catalogs. + template = _(fp.read()[:-1]) + finally: + fp.close() + assert isinstance(template, unicode), 'Translated template is not unicode' + text = expand(template, kw) + if wrap: + return wrap_text(text) + return text diff --git a/src/mailman/passwords.py b/src/mailman/utilities/passwords.py index c14584748..c14584748 100644 --- a/src/mailman/passwords.py +++ b/src/mailman/utilities/passwords.py diff --git a/src/mailman/utilities/string.py b/src/mailman/utilities/string.py index 44b99876e..9054ed076 100644 --- a/src/mailman/utilities/string.py +++ b/src/mailman/utilities/string.py @@ -21,12 +21,28 @@ from __future__ import absolute_import, unicode_literals __metaclass__ = type __all__ = [ - 'expand' + 'expand', + 'oneline', + 'uncanonstr', + 'websafe', + 'wrap', ] +import cgi import logging -from string import Template + +from email.errors import HeaderParseError +from email.header import decode_header, make_header +from string import Template, whitespace +from textwrap import TextWrapper, dedent +from zope.component import getUtility + +from mailman.interfaces.languages import ILanguageManager + + +EMPTYSTRING = '' +NL = '\n' log = logging.getLogger('mailman.error') @@ -57,3 +73,155 @@ def expand(template, substitutions, template_class=Template): except (TypeError, ValueError): # The template is really screwed up. log.exception('broken template: %s', template) + + + +def oneline(s, cset='us-ascii', in_unicode=False): + """Decode a header string in one line and convert into specified charset. + + :param s: The header string + :type s: string + :param cset: The character set (encoding) to use. + :type cset: string + :param in_unicode: Flag specifying whether to return the converted string + as a unicode (True) or an 8-bit string (False, the default). + :type in_unicode: bool + :return: The decoded header string. If an error occurs while converting + the input string, return the string undecoded, as an 8-bit string. + :rtype: string + """ + try: + h = make_header(decode_header(s)) + ustr = h.__unicode__() + line = EMPTYSTRING.join(ustr.splitlines()) + if in_unicode: + return line + else: + return line.encode(cset, 'replace') + except (LookupError, UnicodeError, ValueError, HeaderParseError): + # possibly charset problem. return with undecoded string in one line. + return EMPTYSTRING.join(s.splitlines()) + + + +def websafe(s): + return cgi.escape(s, quote=True) + + + +# The opposite of canonstr() -- sorta. I.e. it attempts to encode s in the +# charset of the given language, which is the character set that the page will +# be rendered in, and failing that, replaces non-ASCII characters with their +# html references. It always returns a byte string. +def uncanonstr(s, lang=None): + if s is None: + s = u'' + if lang is None: + charset = 'us-ascii' + else: + charset = getUtility(ILanguageManager)[lang].charset + # See if the string contains characters only in the desired character + # set. If so, return it unchanged, except for coercing it to a byte + # string. + try: + if isinstance(s, unicode): + return s.encode(charset) + else: + unicode(s, charset) + return s + except UnicodeError: + # Nope, it contains funny characters, so html-ref it + a = [] + for c in s: + o = ord(c) + if o > 127: + a.append('&#%3d;' % o) + else: + a.append(c) + # Join characters together and coerce to byte string + return str(EMPTYSTRING.join(a)) + + + +def wrap(text, column=70, honor_leading_ws=True): + """Wrap and fill the text to the specified column. + + The input text is wrapped and filled as done by the standard library + textwrap module. The differences here being that this function is capable + of filling multiple paragraphs (as defined by text separated by blank + lines). Also, when `honor_leading_ws` is True (the default), paragraphs + that being with whitespace are not wrapped. This is the algorithm that + the Python FAQ wizard used. + """ + # First, split the original text into paragraph, keeping all blank lines + # between them. + paragraphs = [] + paragraph = [] + last_indented = False + for line in text.splitlines(True): + is_indented = (len(line) > 0 and line[0] in whitespace) + if line == NL: + if len(paragraph) > 0: + paragraphs.append(EMPTYSTRING.join(paragraph)) + paragraphs.append(line) + last_indented = False + paragraph = [] + elif last_indented != is_indented: + # The indentation level changed. We treat this as a paragraph + # break but no blank line will be issued between paragraphs. + if len(paragraph) > 0: + paragraphs.append(EMPTYSTRING.join(paragraph)) + # The next paragraph starts with this line. + paragraph = [line] + last_indented = is_indented + else: + # This line does not constitute a paragraph break. + paragraph.append(line) + # We've consumed all the lines in the original text. Transfer the last + # paragraph we were collecting to the full set of paragraphs. + paragraphs.append(EMPTYSTRING.join(paragraph)) + # Now iterate through all paragraphs, wrapping as necessary. + wrapped_paragraphs = [] + # The dedented wrapper. + wrapper = TextWrapper(width=column, + fix_sentence_endings=True) + # The indented wrapper. For this one, we'll clobber initial_indent and + # subsequent_indent as needed per indented chunk of text. + iwrapper = TextWrapper(width=column, + fix_sentence_endings=True, + ) + add_paragraph_break = False + for paragraph in paragraphs: + if add_paragraph_break: + wrapped_paragraphs.append(NL) + add_paragraph_break = False + paragraph_text = EMPTYSTRING.join(paragraph) + # Just copy the blank lines to the final set of paragraphs. + if paragraph == NL: + wrapped_paragraphs.append(NL) + # Choose the wrapper based on whether the paragraph is indented or + # not. Also, do not wrap indented paragraphs if honor_leading_ws is + # set. + elif paragraph[0] in whitespace: + if honor_leading_ws: + # Leave the indented paragraph verbatim. + wrapped_paragraphs.append(paragraph_text) + else: + # The paragraph should be wrapped, but it must first be + # dedented. The leading whitespace on the first line of the + # original text will be used as the indentation for all lines + # in the wrapped text. + for i, ch in enumerate(paragraph_text): + if ch not in whitespace: + break + leading_ws = paragraph[:i] + iwrapper.initial_indent=leading_ws + iwrapper.subsequent_indent=leading_ws + paragraph_text = dedent(paragraph_text) + wrapped_paragraphs.append(iwrapper.fill(paragraph_text)) + add_paragraph_break = True + else: + # Fill this paragraph. fill() consumes the trailing newline. + wrapped_paragraphs.append(wrapper.fill(paragraph_text)) + add_paragraph_break = True + return EMPTYSTRING.join(wrapped_paragraphs) diff --git a/src/mailman/tests/test_passwords.py b/src/mailman/utilities/tests/test_passwords.py index 49f55e82f..60c201a5a 100644 --- a/src/mailman/tests/test_passwords.py +++ b/src/mailman/utilities/tests/test_passwords.py @@ -27,8 +27,8 @@ __all__ = [ import unittest -from mailman import passwords from mailman.core import errors +from mailman.utilities import passwords diff --git a/src/mailman/utilities/tests/test_templates.py b/src/mailman/utilities/tests/test_templates.py new file mode 100644 index 000000000..e21b44544 --- /dev/null +++ b/src/mailman/utilities/tests/test_templates.py @@ -0,0 +1,287 @@ +# Copyright (C) 2011 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Testing i18n template search and interpolation.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'test_suite', + ] + + +import os +import shutil +import tempfile +import unittest + +from zope.component import getUtility + +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.interfaces.languages import ILanguageManager +from mailman.testing.layers import ConfigLayer +from mailman.utilities.i18n import TemplateNotFoundError, _search, find, make + + + +class TestSearchOrder(unittest.TestCase): + """Test internal search order for language templates.""" + + layer = ConfigLayer + + def setUp(self): + self.template_dir = tempfile.mkdtemp() + config.push('no template dir', """\ + [mailman] + default_language: fr + [paths.testing] + template_dir: {0}/t + var_dir: {0}/v + """.format(self.template_dir)) + language_manager = getUtility(ILanguageManager) + language_manager.add('de', 'utf-8', 'German') + language_manager.add('it', 'utf-8', 'Italian') + self.mlist = create_list('l@example.com') + self.mlist.preferred_language = 'de' + + def tearDown(self): + config.pop('no template dir') + shutil.rmtree(self.template_dir) + + def _stripped_search_order(self, template_file, + mailing_list=None, language=None): + raw_search_order = _search(template_file, mailing_list, language) + for path in raw_search_order: + yield path[len(self.template_dir):] + + def test_fully_specified_search_order(self): + search_order = self._stripped_search_order('foo.txt', self.mlist, 'it') + # language argument + self.assertEqual(next(search_order), + '/v/lists/l@example.com/it/foo.txt') + self.assertEqual(next(search_order), '/t/example.com/it/foo.txt') + self.assertEqual(next(search_order), '/t/site/it/foo.txt') + self.assertEqual(next(search_order), '/t/it/foo.txt') + # mlist.preferred_language + self.assertEqual(next(search_order), + '/v/lists/l@example.com/de/foo.txt') + self.assertEqual(next(search_order), '/t/example.com/de/foo.txt') + self.assertEqual(next(search_order), '/t/site/de/foo.txt') + self.assertEqual(next(search_order), '/t/de/foo.txt') + # site's default language + self.assertEqual(next(search_order), + '/v/lists/l@example.com/fr/foo.txt') + self.assertEqual(next(search_order), '/t/example.com/fr/foo.txt') + self.assertEqual(next(search_order), '/t/site/fr/foo.txt') + self.assertEqual(next(search_order), '/t/fr/foo.txt') + # English + self.assertEqual(next(search_order), + '/v/lists/l@example.com/en/foo.txt') + self.assertEqual(next(search_order), '/t/example.com/en/foo.txt') + self.assertEqual(next(search_order), '/t/site/en/foo.txt') + self.assertEqual(next(search_order), '/t/en/foo.txt') + + def test_no_language_argument_search_order(self): + search_order = self._stripped_search_order('foo.txt', self.mlist) + # mlist.preferred_language + self.assertEqual(next(search_order), + '/v/lists/l@example.com/de/foo.txt') + self.assertEqual(next(search_order), '/t/example.com/de/foo.txt') + self.assertEqual(next(search_order), '/t/site/de/foo.txt') + self.assertEqual(next(search_order), '/t/de/foo.txt') + # site's default language + self.assertEqual(next(search_order), + '/v/lists/l@example.com/fr/foo.txt') + self.assertEqual(next(search_order), '/t/example.com/fr/foo.txt') + self.assertEqual(next(search_order), '/t/site/fr/foo.txt') + self.assertEqual(next(search_order), '/t/fr/foo.txt') + # English + self.assertEqual(next(search_order), + '/v/lists/l@example.com/en/foo.txt') + self.assertEqual(next(search_order), '/t/example.com/en/foo.txt') + self.assertEqual(next(search_order), '/t/site/en/foo.txt') + self.assertEqual(next(search_order), '/t/en/foo.txt') + + def test_no_mailing_list_argument_search_order(self): + search_order = self._stripped_search_order('foo.txt', language='it') + # language argument + self.assertEqual(next(search_order), '/t/site/it/foo.txt') + self.assertEqual(next(search_order), '/t/it/foo.txt') + # site's default language + self.assertEqual(next(search_order), '/t/site/fr/foo.txt') + self.assertEqual(next(search_order), '/t/fr/foo.txt') + # English + self.assertEqual(next(search_order), '/t/site/en/foo.txt') + self.assertEqual(next(search_order), '/t/en/foo.txt') + + def test_no_optional_arguments_search_order(self): + search_order = self._stripped_search_order('foo.txt') + # site's default language + self.assertEqual(next(search_order), '/t/site/fr/foo.txt') + self.assertEqual(next(search_order), '/t/fr/foo.txt') + # English + self.assertEqual(next(search_order), '/t/site/en/foo.txt') + self.assertEqual(next(search_order), '/t/en/foo.txt') + + + +class TestFind(unittest.TestCase): + """Test template search.""" + + layer = ConfigLayer + + def setUp(self): + self.template_dir = tempfile.mkdtemp() + config.push('template config', """\ + [paths.testing] + template_dir: {0} + """.format(self.template_dir)) + # The following MUST happen AFTER the push() above since pushing a new + # config also clears out the language manager. + getUtility(ILanguageManager).add('xx', 'utf-8', 'Xlandia') + self.mlist = create_list('test@example.com') + self.mlist.preferred_language = 'xx' + self.fp = None + # Populate global tempdir with a few fake templates. + self.xxdir = os.path.join(self.template_dir, 'xx') + os.mkdir(self.xxdir) + with open(os.path.join(self.xxdir, 'global.txt'), 'w') as fp: + fp.write('Global template') + self.sitedir = os.path.join(self.template_dir, 'site', 'xx') + os.makedirs(self.sitedir) + with open(os.path.join(self.sitedir, 'site.txt'), 'w') as fp: + fp.write('Site template') + self.domaindir = os.path.join(self.template_dir, 'example.com', 'xx') + os.makedirs(self.domaindir) + with open(os.path.join(self.domaindir, 'domain.txt'), 'w') as fp: + fp.write('Domain template') + self.listdir = os.path.join(self.mlist.data_path, 'xx') + os.makedirs(self.listdir) + with open(os.path.join(self.listdir, 'list.txt'), 'w') as fp: + fp.write('List template') + + def tearDown(self): + if self.fp is not None: + self.fp.close() + config.pop('template config') + shutil.rmtree(self.template_dir) + shutil.rmtree(self.listdir) + + def test_find_global_template(self): + filename, self.fp = find('global.txt', language='xx') + self.assertEqual(filename, os.path.join(self.xxdir, 'global.txt')) + self.assertEqual(self.fp.read(), 'Global template') + + def test_find_site_template(self): + filename, self.fp = find('site.txt', language='xx') + self.assertEqual(filename, os.path.join(self.sitedir, 'site.txt')) + self.assertEqual(self.fp.read(), 'Site template') + + def test_find_domain_template(self): + filename, self.fp = find('domain.txt', self.mlist) + self.assertEqual(filename, os.path.join(self.domaindir, 'domain.txt')) + self.assertEqual(self.fp.read(), 'Domain template') + + def test_find_list_template(self): + filename, self.fp = find('list.txt', self.mlist) + self.assertEqual(filename, os.path.join(self.listdir, 'list.txt')) + self.assertEqual(self.fp.read(), 'List template') + + def test_template_not_found(self): + # Python 2.6 compatibility. + try: + find('missing.txt', self.mlist) + except TemplateNotFoundError as error: + self.assertEqual(error.template_file, 'missing.txt') + else: + raise AssertionError('TemplateNotFoundError expected') + + + +class TestMake(unittest.TestCase): + """Test template interpolation.""" + + layer = ConfigLayer + + def setUp(self): + self.template_dir = tempfile.mkdtemp() + config.push('template config', """\ + [paths.testing] + template_dir: {0} + """.format(self.template_dir)) + # The following MUST happen AFTER the push() above since pushing a new + # config also clears out the language manager. + getUtility(ILanguageManager).add('xx', 'utf-8', 'Xlandia') + self.mlist = create_list('test@example.com') + self.mlist.preferred_language = 'xx' + # Populate the template directory with some samples. + self.xxdir = os.path.join(self.template_dir, 'xx') + os.mkdir(self.xxdir) + with open(os.path.join(self.xxdir, 'nosub.txt'), 'w') as fp: + print >> fp, """\ +This is a global template. +It has no substitutions. +It will be wrapped. +""" + with open(os.path.join(self.xxdir, 'subs.txt'), 'w') as fp: + print >> fp, """\ +This is a $kind template. +It has $howmany substitutions. +It will be wrapped. +""" + with open(os.path.join(self.xxdir, 'nowrap.txt'), 'w') as fp: + print >> fp, """\ +This is a $kind template. +It has $howmany substitutions. +It will not be wrapped. +""" + + def tearDown(self): + config.pop('template config') + shutil.rmtree(self.template_dir) + + def test_no_substitutions(self): + self.assertEqual(make('nosub.txt', self.mlist), """\ +This is a global template. It has no substitutions. It will be +wrapped.""") + + def test_substitutions(self): + self.assertEqual(make('subs.txt', self.mlist, + kind='very nice', + howmany='a few'), """\ +This is a very nice template. It has a few substitutions. It will be +wrapped.""") + + def test_substitutions_no_wrap(self): + self.assertEqual(make('nowrap.txt', self.mlist, wrap=False, + kind='very nice', + howmany='a few'), """\ +This is a very nice template. +It has a few substitutions. +It will not be wrapped. +""") + + + +def test_suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(TestSearchOrder)) + suite.addTest(unittest.makeSuite(TestFind)) + suite.addTest(unittest.makeSuite(TestMake)) + return suite diff --git a/src/mailman/utilities/tests/test_wrap.py b/src/mailman/utilities/tests/test_wrap.py new file mode 100644 index 000000000..2fc6c6ccf --- /dev/null +++ b/src/mailman/utilities/tests/test_wrap.py @@ -0,0 +1,151 @@ +# Copyright (C) 2011 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test text wrapping.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'test_suite', + ] + + +import unittest + +from mailman.utilities.string import wrap + + + +class TestWrap(unittest.TestCase): + """Test text wrapping.""" + + def test_simple_wrap(self): + text = """\ +This is a single +paragraph. It consists +of several sentences +none of +which are +very long. +""" + self.assertEqual(wrap(text), """\ +This is a single paragraph. It consists of several sentences none of +which are very long.""") + + def test_two_paragraphs(self): + text = """\ +This is a single +paragraph. It consists +of several sentences +none of +which are +very long. + +And here is a second paragraph which +also consists +of several sentences. None of +these are very long +either. +""" + self.assertEqual(wrap(text), """\ +This is a single paragraph. It consists of several sentences none of +which are very long. + +And here is a second paragraph which also consists of several +sentences. None of these are very long either.""") + + def test_honor_ws(self): + text = """\ +This is a single +paragraph. It consists +of several sentences +none of +which are +very long. + + This paragraph is + indented so it + won't be filled. + +And here is a second paragraph which +also consists +of several sentences. None of +these are very long +either. +""" + self.assertEqual(wrap(text), """\ +This is a single paragraph. It consists of several sentences none of +which are very long. + + This paragraph is + indented so it + won't be filled. + +And here is a second paragraph which also consists of several +sentences. None of these are very long either.""") + + def test_dont_honor_ws(self): + text = """\ +This is a single +paragraph. It consists +of several sentences +none of +which are +very long. + + This paragraph is + indented but we don't + honor whitespace so it + will be filled. + +And here is a second paragraph which +also consists +of several sentences. None of +these are very long +either. +""" + self.assertEqual(wrap(text, honor_leading_ws=False), """\ +This is a single paragraph. It consists of several sentences none of +which are very long. + + This paragraph is indented but we don't honor whitespace so it + will be filled. + +And here is a second paragraph which also consists of several +sentences. None of these are very long either.""") + + def test_indentation_boundary(self): + text = """\ +This is a single paragraph +that consists of one sentence. + And another one that breaks + because it is indented. +Followed by one more paragraph. +""" + self.assertEqual(wrap(text), """\ +This is a single paragraph that consists of one sentence. + And another one that breaks + because it is indented. +Followed by one more paragraph.""") + + + +def test_suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(TestWrap)) + return suite |
