diff options
112 files changed, 2339 insertions, 2094 deletions
diff --git a/README.rst b/README.rst index 186062e53..c87b7eccb 100644 --- a/README.rst +++ b/README.rst @@ -38,6 +38,18 @@ Table of Contents src/mailman/docs/DATABASE src/mailman/docs/MTA src/mailman/docs/8-miles-high + src/mailman/docs/WebUIin5 + src/mailman/docs/ArchiveUIin5 + + +Mailman modules +--------------- + +These documents are generated from the doctest suite. + +.. toctree:: + :glob: + src/mailman/bin/docs/* src/mailman/commands/docs/* src/mailman/database/docs/* @@ -46,7 +58,7 @@ Table of Contents src/mailman/app/docs/* src/mailman/styles/docs/* src/mailman/runners/docs/* - src/mailman/pipeline/docs/* + src/mailman/handlers/docs/* src/mailman/rest/docs/* src/mailman/chains/docs/* src/mailman/rules/docs/* @@ -224,13 +224,18 @@ def index_html(): import errno cwd = os.getcwd() try: - os.makedirs('build/sphinx/html') + try: + os.makedirs('build/sphinx/html') + except OSError as error: + if error.errno != errno.EEXIST: + raise os.chdir('build/sphinx/html') - os.symlink('README.html', 'index.html') - print 'index.html -> README.html' - except OSError as error: - if error.errno != errno.EEXIST: - raise + try: + os.symlink('README.html', 'index.html') + print 'index.html -> README.html' + except OSError as error: + if error.errno != errno.EEXIST: + raise finally: os.chdir(cwd) @@ -102,6 +102,7 @@ case second `m'. Any other spelling is incorrect.""", 'httplib2', 'lazr.config', 'lazr.smtptest', + 'mock', 'restish', 'storm', 'zc.buildout', diff --git a/src/mailman/app/docs/pipelines.rst b/src/mailman/app/docs/pipelines.rst index 56fdeb5c6..96d9d232a 100644 --- a/src/mailman/app/docs/pipelines.rst +++ b/src/mailman/app/docs/pipelines.rst @@ -60,7 +60,7 @@ However there are currently no recipients for this message. >>> dump_msgdata(msgdata) original_sender : aperson@example.com - origsubj : My first post + original_subject: My first post recipients : set([]) stripped_subject: My first post @@ -90,7 +90,7 @@ processing queues. >>> dump_msgdata(messages[0].msgdata) _parsemsg : False original_sender : aperson@example.com - origsubj : My first post + original_subject: My first post recipients : set([]) stripped_subject: My first post version : 3 @@ -98,7 +98,7 @@ processing queues. This mailing list is not linked to an NNTP newsgroup, so there's nothing in the outgoing nntp queue. - >>> messages = get_queue_messages('news') + >>> messages = get_queue_messages('nntp') >>> len(messages) 0 @@ -128,7 +128,7 @@ delivered to end recipients. _parsemsg : False listname : test@example.com original_sender : aperson@example.com - origsubj : My first post + original_subject: My first post recipients : set([]) stripped_subject: My first post version : 3 diff --git a/src/mailman/app/registrar.py b/src/mailman/app/registrar.py index b95fda1f8..030a504f7 100644 --- a/src/mailman/app/registrar.py +++ b/src/mailman/app/registrar.py @@ -17,7 +17,7 @@ """Implementation of the IUserRegistrar interface.""" -from __future__ import unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -27,7 +27,6 @@ __all__ = [ import logging -from pkg_resources import resource_string from zope.component import getUtility from zope.interface import implements diff --git a/src/mailman/app/tests/test_subscriptions.py b/src/mailman/app/tests/test_subscriptions.py index 6ab386f7f..a63c9ac04 100644 --- a/src/mailman/app/tests/test_subscriptions.py +++ b/src/mailman/app/tests/test_subscriptions.py @@ -17,10 +17,11 @@ """Tests for the subscription service.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ + 'TestJoin' ] @@ -63,6 +64,6 @@ class TestJoin(unittest.TestCase): try: self._service.join('test@example.com', 'bogus') except InvalidEmailAddressError as exc: - self.assertEqual(exc.address, 'bogus') + self.assertEqual(exc.email, 'bogus') else: raise AssertionError('InvalidEmailAddressError expected') diff --git a/src/mailman/bin/arch.py b/src/mailman/bin/arch.py deleted file mode 100644 index 64fd235aa..000000000 --- a/src/mailman/bin/arch.py +++ /dev/null @@ -1,150 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -from __future__ import absolute_import, unicode_literals - -import os -import sys -import errno -import shutil -import optparse - -from flufl.lock import Lock - -from mailman.Archiver.HyperArch import HyperArchive -from mailman.Defaults import hours -from mailman.configuration import config -from mailman.core.i18n import _ -from mailman.initialize import initialize -from mailman.version import MAILMAN_VERSION - - - -def parseargs(): - parser = optparse.OptionParser(version=MAILMAN_VERSION, - usage=_("""\ -%%prog [options] listname [mbox] - -Rebuild a list's archive. - -Use this command to rebuild the archives for a mailing list. You may want to -do this if you edit some messages in an archive, or remove some messages from -an archive. - -Where 'mbox' is the path to a list's complete mbox archive. Usually this will -be some path in the archives/private directory. For example: - -% bin/arch mylist archives/private/mylist.mbox/mylist.mbox - -'mbox' is optional. If it is missing, it is calculated from the listname. -""")) - parser.add_option('-q', '--quiet', - dest='verbose', default=True, action='store_false', - help=_('Make the archiver output less verbose')) - parser.add_option('--wipe', - default=False, action='store_true', - help=_("""\ -First wipe out the original archive before regenerating. You usually want to -specify this argument unless you're generating the archive in chunks.""")) - parser.add_option('-s', '--start', - default=None, type='int', metavar='N', - help=_("""\ -Start indexing at article N, where article 0 is the first in the mbox. -Defaults to 0.""")) - parser.add_option('-e', '--end', - default=None, type='int', metavar='M', - help=_("""\ -End indexing at article M. This script is not very efficient with respect to -memory management, and for large archives, it may not be possible to index the -mbox entirely. For that reason, you can specify the start and end article -numbers.""")) - parser.add_option('-C', '--config', - help=_('Alternative configuration file to use')) - opts, args = parser.parse_args() - if len(args) < 1: - parser.print_help() - print >> sys.stderr, _('listname is required') - sys.exit(1) - if len(args) > 2: - parser.print_help() - print >> sys.stderr, _('Unexpected arguments') - sys.exit(1) - return parser, opts, args - - - -def main(): - parser, opts, args = parseargs() - initialize(opts.config) - - listname = args[0].lower().strip() - if len(args) < 2: - mbox = None - else: - mbox = args[1] - - # Open the mailing list object - mlist = config.list_manager.get(listname) - if mlist is None: - parser.error(_('No such list: $listname')) - if mbox is None: - mbox = mlist.ArchiveFileName() - - # Set the default language to the list's preferred language. - _.default = mlist.preferred_language - # Lay claim to the archive's lock file. This is so no other post can - # mess up the archive while we're processing it. Try to pick a - # suitably long period of time for the lock lifetime even though we - # really don't know how long it will take. - # - # XXX processUnixMailbox() should refresh the lock. - lock_path = os.path.join(mlist.data_path, '.archiver.lck') - with Lock(lock_path, lifetime=int(hours(3))): - # Try to open mbox before wiping old archive. - try: - fp = open(mbox) - except IOError, e: - if e.errno == errno.ENOENT: - print >> sys.stderr, _('Cannot open mbox file: $mbox') - else: - print >> sys.stderr, e - sys.exit(1) - # Maybe wipe the old archives - if opts.wipe: - if mlist.scrub_nondigest: - # TK: save the attachments dir because they are not in mbox - saved = False - atchdir = os.path.join(mlist.archive_dir(), 'attachments') - savedir = os.path.join(mlist.archive_dir() + '.mbox', - 'attachments') - try: - os.rename(atchdir, savedir) - saved = True - except OSError, e: - if e.errno != errno.ENOENT: - raise - shutil.rmtree(mlist.archive_dir()) - if mlist.scrub_nondigest and saved: - os.renames(savedir, atchdir) - - archiver = HyperArchive(mlist) - archiver.VERBOSE = opts.verbose - try: - archiver.processUnixMailbox(fp, opts.start, opts.end) - finally: - archiver.close() - fp.close() diff --git a/src/mailman/bin/check_perms.py b/src/mailman/bin/check_perms.py deleted file mode 100644 index 6e0d761d9..000000000 --- a/src/mailman/bin/check_perms.py +++ /dev/null @@ -1,408 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -import os -import sys -import pwd -import grp -import errno -import optparse - -from stat import * - -from mailman.configuration import config -from mailman.core.i18n import _ -from mailman.version import MAILMAN_VERSION - - -# XXX Need to check the archives/private/*/database/* files - - - -class State: - FIX = False - VERBOSE = False - ERRORS = 0 - -STATE = State() - -DIRPERMS = S_ISGID | S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH -QFILEPERMS = S_ISGID | S_IRWXU | S_IRWXG -PYFILEPERMS = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH -ARTICLEFILEPERMS = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP -MBOXPERMS = S_IRGRP | S_IWGRP | S_IRUSR | S_IWUSR -PRIVATEPERMS = QFILEPERMS - - - -def statmode(path): - return os.stat(path).st_mode - - -def statgidmode(path): - stat = os.stat(path) - return stat.st_mode, stat.st_gid - - -seen = {} - -# libc's getgrgid re-opens /etc/group each time :( -_gidcache = {} - -def getgrgid(gid): - data = _gidcache.get(gid) - if data is None: - data = grp.getgrgid(gid) - _gidcache[gid] = data - return data - - - -def checkwalk(arg, dirname, names): - # Short-circuit duplicates - if seen.has_key(dirname): - return - seen[dirname] = True - for name in names: - path = os.path.join(dirname, name) - if arg.VERBOSE: - print _(' checking gid and mode for $path') - try: - mode, gid = statgidmode(path) - except OSError, e: - if e.errno != errno.ENOENT: raise - continue - if gid != MAILMAN_GID: - try: - groupname = getgrgid(gid)[0] - except KeyError: - groupname = '<anon gid %d>' % gid - arg.ERRORS += 1 - print _( - '$path bad group (has: $groupname, expected $MAILMAN_GROUP)'), - if STATE.FIX: - print _('(fixing)') - os.chown(path, -1, MAILMAN_GID) - else: - print - # Most directories must be at least rwxrwsr-x. - # The private archive directory and database directory must be at - # least rwxrws---. Their 'other' permissions are checked in - # checkarchives() and checkarchivedbs() below. Their 'user' and - # 'group' permissions are checked here. - # The directories under qfiles should be rwxrws---. Their 'user' and - # 'group' permissions are checked here. Their 'other' permissions - # aren't checked. - private = config.PRIVATE_ARCHIVE_FILE_DIR - if path == private or ( - os.path.commonprefix((path, private)) == private - and os.path.split(path)[1] == 'database'): - # then... - targetperms = PRIVATEPERMS - elif (os.path.commonprefix((path, config.QUEUE_DIR)) - == config.QUEUE_DIR): - targetperms = QFILEPERMS - else: - targetperms = DIRPERMS - octperms = oct(targetperms) - if S_ISDIR(mode) and (mode & targetperms) != targetperms: - arg.ERRORS += 1 - print _('directory permissions must be $octperms: $path'), - if STATE.FIX: - print _('(fixing)') - os.chmod(path, mode | targetperms) - else: - print - elif os.path.splitext(path)[1] in ('.py', '.pyc', '.pyo'): - octperms = oct(PYFILEPERMS) - if mode & PYFILEPERMS != PYFILEPERMS: - print _('source perms must be $octperms: $path'), - arg.ERRORS += 1 - if STATE.FIX: - print _('(fixing)') - os.chmod(path, mode | PYFILEPERMS) - else: - print - elif path.endswith('-article'): - # Article files must be group writeable - octperms = oct(ARTICLEFILEPERMS) - if mode & ARTICLEFILEPERMS != ARTICLEFILEPERMS: - print _('article db files must be $octperms: $path'), - arg.ERRORS += 1 - if STATE.FIX: - print _('(fixing)') - os.chmod(path, mode | ARTICLEFILEPERMS) - else: - print - - - -def checkall(): - # first check PREFIX - if STATE.VERBOSE: - prefix = config.PREFIX - print _('checking mode for $prefix') - dirs = {} - for d in (config.PREFIX, config.EXEC_PREFIX, config.VAR_PREFIX, - config.LOG_DIR): - dirs[d] = True - for d in dirs.keys(): - try: - mode = statmode(d) - except OSError, e: - if e.errno != errno.ENOENT: raise - print _('WARNING: directory does not exist: $d') - continue - if (mode & DIRPERMS) != DIRPERMS: - STATE.ERRORS += 1 - print _('directory must be at least 02775: $d'), - if STATE.FIX: - print _('(fixing)') - os.chmod(d, mode | DIRPERMS) - else: - print - # check all subdirs - os.path.walk(d, checkwalk, STATE) - - - -def checkarchives(): - private = config.PRIVATE_ARCHIVE_FILE_DIR - if STATE.VERBOSE: - print _('checking perms on $private') - # private archives must not be other readable - mode = statmode(private) - if mode & S_IROTH: - STATE.ERRORS += 1 - print _('$private must not be other-readable'), - if STATE.FIX: - print _('(fixing)') - os.chmod(private, mode & ~S_IROTH) - else: - print - # In addition, on a multiuser system you may want to hide the private - # archives so other users can't read them. - if mode & S_IXOTH: - print _("""\ -Warning: Private archive directory is other-executable (o+x). - This could allow other users on your system to read private archives. - If you're on a shared multiuser system, you should consult the - installation manual on how to fix this.""") - - - -def checkmboxfile(mboxdir): - absdir = os.path.join(config.PRIVATE_ARCHIVE_FILE_DIR, mboxdir) - for f in os.listdir(absdir): - if not f.endswith('.mbox'): - continue - mboxfile = os.path.join(absdir, f) - mode = statmode(mboxfile) - if (mode & MBOXPERMS) != MBOXPERMS: - STATE.ERRORS = STATE.ERRORS + 1 - print _('mbox file must be at least 0660:'), mboxfile - if STATE.FIX: - print _('(fixing)') - os.chmod(mboxfile, mode | MBOXPERMS) - else: - print - - - -def checkarchivedbs(): - # The archives/private/listname/database file must not be other readable - # or executable otherwise those files will be accessible when the archives - # are public. That may not be a horrible breach, but let's close this off - # anyway. - for dir in os.listdir(config.PRIVATE_ARCHIVE_FILE_DIR): - if dir.endswith('.mbox'): - checkmboxfile(dir) - dbdir = os.path.join(config.PRIVATE_ARCHIVE_FILE_DIR, dir, 'database') - try: - mode = statmode(dbdir) - except OSError, e: - if e.errno not in (errno.ENOENT, errno.ENOTDIR): raise - continue - if mode & S_IRWXO: - STATE.ERRORS += 1 - print _('$dbdir "other" perms must be 000'), - if STATE.FIX: - print _('(fixing)') - os.chmod(dbdir, mode & ~S_IRWXO) - else: - print - - - -def checkcgi(): - cgidir = os.path.join(config.EXEC_PREFIX, 'cgi-bin') - if STATE.VERBOSE: - print _('checking cgi-bin permissions') - exes = os.listdir(cgidir) - for f in exes: - path = os.path.join(cgidir, f) - if STATE.VERBOSE: - print _(' checking set-gid for $path') - mode = statmode(path) - if mode & S_IXGRP and not mode & S_ISGID: - STATE.ERRORS += 1 - print _('$path must be set-gid'), - if STATE.FIX: - print _('(fixing)') - os.chmod(path, mode | S_ISGID) - else: - print - - - -def checkmail(): - wrapper = os.path.join(config.WRAPPER_DIR, 'mailman') - if STATE.VERBOSE: - print _('checking set-gid for $wrapper') - mode = statmode(wrapper) - if not mode & S_ISGID: - STATE.ERRORS += 1 - print _('$wrapper must be set-gid'), - if STATE.FIX: - print _('(fixing)') - os.chmod(wrapper, mode | S_ISGID) - - - -def checkadminpw(): - for pwfile in (os.path.join(config.DATA_DIR, 'adm.pw'), - os.path.join(config.DATA_DIR, 'creator.pw')): - targetmode = S_IFREG | S_IRUSR | S_IWUSR | S_IRGRP - if STATE.VERBOSE: - print _('checking permissions on $pwfile') - try: - mode = statmode(pwfile) - except OSError, e: - if e.errno != errno.ENOENT: - raise - return - if mode != targetmode: - STATE.ERRORS += 1 - octmode = oct(mode) - print _('$pwfile permissions must be exactly 0640 (got $octmode)'), - if STATE.FIX: - print _('(fixing)') - os.chmod(pwfile, targetmode) - else: - print - - -def checkmta(): - if config.MTA: - modname = 'mailman.MTA.' + config.MTA - __import__(modname) - try: - sys.modules[modname].checkperms(STATE) - except AttributeError: - pass - - - -def checkdata(): - targetmode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP - checkfiles = ('config.pck', 'config.pck.last', - 'config.db', 'config.db.last', - 'next-digest', 'next-digest-topics', - 'digest.mbox', 'pending.pck', - 'request.db', 'request.db.tmp') - if STATE.VERBOSE: - print _('checking permissions on list data') - for dir in os.listdir(config.LIST_DATA_DIR): - for file in checkfiles: - path = os.path.join(config.LIST_DATA_DIR, dir, file) - if STATE.VERBOSE: - print _(' checking permissions on: $path') - try: - mode = statmode(path) - except OSError, e: - if e.errno != errno.ENOENT: - raise - continue - if (mode & targetmode) != targetmode: - STATE.ERRORS += 1 - print _('file permissions must be at least 660: $path'), - if STATE.FIX: - print _('(fixing)') - os.chmod(path, mode | targetmode) - else: - print - - - -def parseargs(): - parser = optparse.OptionParser(version=MAILMAN_VERSION, - usage=_("""\ -%prog [options] - -Check the permissions of all Mailman files. With no options, just report the -permission and ownership problems found.""")) - parser.add_option('-f', '--fix', - default=False, action='store_true', help=_("""\ -Fix all permission and ownership problems found. With this option, you must -run check_perms as root.""")) - parser.add_option('-v', '--verbose', - default=False, action='store_true', - help=_('Produce more verbose output')) - parser.add_option('-C', '--config', - help=_('Alternative configuration file to use')) - opts, args = parser.parse_args() - if args: - parser.print_help() - print >> sys.stderr, _('Unexpected arguments') - sys.exit(1) - return parser, opts, args - - - -def main(): - global MAILMAN_USER, MAILMAN_GROUP, MAILMAN_UID, MAILMAN_GID - - parser, opts, args = parseargs() - STATE.FIX = opts.fix - STATE.VERBOSE = opts.verbose - - config.load(opts.config) - - MAILMAN_USER = config.MAILMAN_USER - MAILMAN_GROUP = config.MAILMAN_GROUP - # Let KeyErrors percolate - MAILMAN_GID = grp.getgrnam(MAILMAN_GROUP).gr_gid - MAILMAN_UID = pwd.getpwnam(MAILMAN_USER).pw_uid - - checkall() - checkarchives() - checkarchivedbs() - checkcgi() - checkmail() - checkdata() - checkadminpw() - checkmta() - - if not STATE.ERRORS: - print _('No problems found') - else: - print _('Problems found:'), STATE.ERRORS - print _('Re-run as $MAILMAN_USER (or root) with -f flag to fix') - - -if __name__ == '__main__': - main() diff --git a/src/mailman/bin/cleanarch.py b/src/mailman/bin/cleanarch.py deleted file mode 100644 index 48b96d191..000000000 --- a/src/mailman/bin/cleanarch.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright (C) 2001-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Clean up an .mbox archive file.""" - -import re -import sys -import mailbox -import optparse - -from mailman.core.i18n import _ -from mailman.version import MAILMAN_VERSION - - -cre = re.compile(mailbox.UnixMailbox._fromlinepattern) -# From RFC 2822, a header field name must contain only characters from 33-126 -# inclusive, excluding colon. I.e. from oct 41 to oct 176 less oct 072. Must -# use re.match() so that it's anchored at the beginning of the line. -fre = re.compile(r'[\041-\071\073-\176]+') - - - -def parseargs(): - parser = optparse.OptionParser(version=MAILMAN_VERSION, - usage=_("""\ -%prog [options] < inputfile > outputfile - -The archiver looks for Unix-From lines separating messages in an mbox archive -file. For compatibility, it specifically looks for lines that start with -'From ' -- i.e. the letters capital-F, lowercase-r, o, m, space, ignoring -everything else on the line. - -Normally, any lines that start 'From ' in the body of a message should be -escaped such that a > character is actually the first on a line. It is -possible though that body lines are not actually escaped. This script -attempts to fix these by doing a stricter test of the Unix-From lines. Any -lines that start From ' but do not pass this stricter test are escaped with a -'>' character.""")) - parser.add_option('-q', '--quiet', - default=False, action='store_true', help=_("""\ -Don't print changed line information to standard error.""")) - parser.add_option('-s', '--status', - default=-1, type='int', help=_("""\ -Print a '#' character for every n lines processed. With a number less than or -equal to zero, suppress the '#' characters.""")) - parser.add_option('-n', '--dry-run', - default=False, action='store_true', help=_("""\ -Don't actually output anything.""")) - opts, args = parser.parser_args() - if args: - parser.print_error(_('Unexpected arguments')) - return parser, opts, args - - - -def escape_line(line, lineno, quiet, output): - if output: - sys.stdout.write('>' + line) - if not quiet: - print >> sys.stderr, _('Unix-From line changed: $lineno') - print >> sys.stderr, line[:-1] - - - -def main(): - parser, opts, args = parseargs() - - lineno = 0 - statuscnt = 0 - messages = 0 - prevline = None - while True: - lineno += 1 - line = sys.stdin.readline() - if not line: - break - if line.startswith('From '): - if cre.match(line): - # This is a real Unix-From line. But it could be a message - # /about/ Unix-From lines, so as a second order test, make - # sure there's at least one RFC 2822 header following - nextline = sys.stdin.readline() - lineno += 1 - if not nextline: - # It was the last line of the mbox, so it couldn't have - # been a Unix-From - escape_line(line, lineno, quiet, output) - break - fieldname = nextline.split(':', 1) - if len(fieldname) < 2 or not fre.match(nextline): - # The following line was not a header, so this wasn't a - # valid Unix-From - escape_line(line, lineno, quiet, output) - if output: - sys.stdout.write(nextline) - else: - # It's a valid Unix-From line - messages += 1 - if output: - # Before we spit out the From_ line, make sure the - # previous line was blank. - if prevline is not None and prevline != '\n': - sys.stdout.write('\n') - sys.stdout.write(line) - sys.stdout.write(nextline) - else: - # This is a bogus Unix-From line - escape_line(line, lineno, quiet, output) - elif output: - # Any old line - sys.stdout.write(line) - if status > 0 and (lineno % status) == 0: - sys.stderr.write('#') - statuscnt += 1 - if statuscnt > 50: - print >> sys.stderr - statuscnt = 0 - prevline = line - print >> sys.stderr, _('%(messages)d messages found') diff --git a/src/mailman/bin/disabled.py b/src/mailman/bin/disabled.py index 4d7167417..c7cab4829 100644 --- a/src/mailman/bin/disabled.py +++ b/src/mailman/bin/disabled.py @@ -54,7 +54,7 @@ def who_callback(option, opt, value, parser): dest.discard(MemberAdaptor.BYBOUNCE) elif opt in ('-a', '--all'): dest.update(ALL) - +x5o def parseargs(): parser = optparse.OptionParser(version=MAILMAN_VERSION, diff --git a/src/mailman/bin/nightly_gzip.py b/src/mailman/bin/nightly_gzip.py deleted file mode 100644 index 2d1c5b8a1..000000000 --- a/src/mailman/bin/nightly_gzip.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -import os -import sys -import optparse - -try: - import gzip -except ImportError: - sys.exit(0) - -from mailman import MailList -from mailman.configuration import config -from mailman.core.i18n import _ -from mailman.initialize import initialize -from mailman.version import MAILMAN_VERSION - - - -def parseargs(): - parser = optparse.OptionParser(version=MAILMAN_VERSION, - usage=_("""\ -%prog [options] [listname ...] - -Re-generate the Pipermail gzip'd archive flat files.""")) - parser.add_option('-v', '--verbose', - default=False, action='store_true', - help=_("Print each file as it's being gzip'd")) - parser.add_option('-z', '--level', - default=6, type='int', - help=_('Specifies the compression level')) - parser.add_option('-C', '--config', - help=_('Alternative configuration file to use')) - opts, args = parser.parse_args() - if opts.level < 1 or opts.level > 9: - parser.print_help() - print >> sys.stderr, _('Illegal compression level: $opts.level') - sys.exit(1) - return opts, args, parser - - - -def compress(txtfile, opts): - if opts.verbose: - print _("gzip'ing: $txtfile") - infp = outfp = None - try: - infp = open(txtfile) - outfp = gzip.open(txtfile + '.gz', 'wb', opts.level) - outfp.write(infp.read()) - finally: - if outfp: - outfp.close() - if infp: - infp.close() - - - -def main(): - opts, args, parser = parseargs() - initialize(opts.config) - - if config.ARCHIVE_TO_MBOX not in (1, 2) or config.GZIP_ARCHIVE_TXT_FILES: - # We're only going to run the nightly archiver if messages are - # archived to the mbox, and the gzip file is not created on demand - # (i.e. for every individual post). This is the normal mode of - # operation. - return - - # Process all the specified lists - for listname in set(args or config.list_manager.names): - mlist = MailList.MailList(listname, lock=False) - if not mlist.archive: - continue - dir = mlist.archive_dir() - try: - allfiles = os.listdir(dir) - except OSError: - # Has the list received any messages? If not, last_post_time will - # be zero, so it's not really a bogus archive dir. - if mlist.last_post_time > 0: - print _('List $listname has a bogus archive_directory: $dir') - continue - if opts.verbose: - print _('Processing list: $listname') - files = [] - for f in allfiles: - if os.path.splitext(f)[1] <> '.txt': - continue - # stat both the .txt and .txt.gz files and append them only if - # the former is newer than the latter. - txtfile = os.path.join(dir, f) - gzpfile = txtfile + '.gz' - txt_mtime = os.path.getmtime(txtfile) - try: - gzp_mtime = os.path.getmtime(gzpfile) - except OSError: - gzp_mtime = -1 - if txt_mtime > gzp_mtime: - files.append(txtfile) - for f in files: - compress(f, opts) diff --git a/src/mailman/bin/show_qfiles.py b/src/mailman/bin/show_qfiles.py deleted file mode 100644 index 08f5fb04e..000000000 --- a/src/mailman/bin/show_qfiles.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (C) 2006-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -import os -import sys - -from cPickle import load - -from mailman.config import config -from mailman.core.i18n import _ -from mailman.options import Options - - - -class ScriptOptions(Options): - usage = _(""" -%%prog [options] qfiles ... - -Show the contents of one or more Mailman queue files.""") - - def add_options(self): - super(ScriptOptions, self).add_options() - self.parser.add_option( - '-q', '--quiet', - default=False, action='store_true', - help=_("Don't print 'helpful' message delimiters.")) - self.parser.add_option( - '-s', '--summary', - default=False, action='store_true', - help=_('Show a summary of queue files.')) - - - -def main(): - options = ScriptOptions() - options.initialize() - - if options.options.summary: - queue_totals = {} - files_by_queue = {} - for switchboard in config.switchboards.values(): - total = 0 - file_mappings = {} - for filename in os.listdir(switchboard.queue_directory): - base, ext = os.path.splitext(filename) - file_mappings[ext] = file_mappings.get(ext, 0) + 1 - total += 1 - files_by_queue[switchboard.queue_directory] = file_mappings - queue_totals[switchboard.queue_directory] = total - # Sort by queue name. - for queue_directory in sorted(files_by_queue): - total = queue_totals[queue_directory] - print queue_directory - print _('\tfile count: $total') - file_mappings = files_by_queue[queue_directory] - for ext in sorted(file_mappings): - print '\t{0}: {1}'.format(ext, file_mappings[ext]) - return - # No summary. - for filename in options.arguments: - if not options.options.quiet: - print '====================>', filename - with open(filename) as fp: - if filename.endswith('.pck'): - msg = load(fp) - data = load(fp) - if data.get('_parsemsg'): - sys.stdout.write(msg) - else: - sys.stdout.write(msg.as_string()) - else: - sys.stdout.write(fp.read()) - - - -if __name__ == '__main__': - main() diff --git a/src/mailman/bin/update.py b/src/mailman/bin/update.py deleted file mode 100644 index 3beda2d97..000000000 --- a/src/mailman/bin/update.py +++ /dev/null @@ -1,660 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -import os -import md5 -import sys -import time -import email -import errno -import shutil -import cPickle -import marshal -import optparse - -from flufl.lock import TimeOutError - -from mailman import MailList -from mailman import Message -from mailman import Pending -from mailman import Utils -from mailman import version -from mailman.MemberAdaptor import BYBOUNCE, ENABLED -from mailman.OldStyleMemberships import OldStyleMemberships -from mailman.Queue.Switchboard import Switchboard -from mailman.configuration import config -from mailman.core.i18n import _ -from mailman.initialize import initialize -from mailman.utilities.filesystem import makedirs - - -FRESH = 0 -NOTFRESH = -1 - - - -def parseargs(): - parser = optparse.OptionParser(version=version.MAILMAN_VERSION, - usage=_("""\ -Perform all necessary upgrades. - -%prog [options]""")) - parser.add_option('-f', '--force', - default=False, action='store_true', help=_("""\ -Force running the upgrade procedures. Normally, if the version number of the -installed Mailman matches the current version number (or a 'downgrade' is -detected), nothing will be done.""")) - parser.add_option('-C', '--config', - help=_('Alternative configuration file to use')) - opts, args = parser.parse_args() - if args: - parser.print_help() - print >> sys.stderr, _('Unexpected arguments') - sys.exit(1) - return parser, opts, args - - - -def calcversions(): - # Returns a tuple of (lastversion, thisversion). If the last version - # could not be determined, lastversion will be FRESH or NOTFRESH, - # depending on whether this installation appears to be fresh or not. The - # determining factor is whether there are files in the $var_prefix/logs - # subdir or not. The version numbers are HEX_VERSIONs. - # - # See if we stored the last updated version - lastversion = None - thisversion = version.HEX_VERSION - try: - fp = open(os.path.join(config.DATA_DIR, 'last_mailman_version')) - data = fp.read() - fp.close() - lastversion = int(data, 16) - except (IOError, ValueError): - pass - # - # try to figure out if this is a fresh install - if lastversion is None: - lastversion = FRESH - try: - if os.listdir(config.LOG_DIR): - lastversion = NOTFRESH - except OSError: - pass - return (lastversion, thisversion) - - - -def makeabs(relpath): - return os.path.join(config.PREFIX, relpath) - - -def make_varabs(relpath): - return os.path.join(config.VAR_PREFIX, relpath) - - - -def move_language_templates(mlist): - listname = mlist.internal_name() - print _('Fixing language templates: $listname') - # Mailman 2.1 has a new cascading search for its templates, defined and - # described in Utils.py:maketext(). Putting templates in the top level - # templates/ subdir or the lists/<listname> subdir is deprecated and no - # longer searched.. - # - # What this means is that most templates can live in the global templates/ - # subdirectory, and only needs to be copied into the list-, vhost-, or - # site-specific language directories when needed. - # - # Also, by default all standard (i.e. English) templates must now live in - # the templates/en directory. This update cleans up all the templates, - # deleting more-specific duplicates (as calculated by md5 checksums) in - # favor of more-global locations. - # - # First, get rid of any lists/<list> template or lists/<list>/en template - # that is identical to the global templates/* default. - for gtemplate in os.listdir(os.path.join(config.TEMPLATE_DIR, 'en')): - # BAW: get rid of old templates, e.g. admlogin.txt and - # handle_opts.html - try: - fp = open(os.path.join(config.TEMPLATE_DIR, gtemplate)) - except IOError, e: - if e.errno <> errno.ENOENT: - raise - # No global template - continue - gcksum = md5.new(fp.read()).digest() - fp.close() - # Match against the lists/<list>/* template - try: - fp = open(os.path.join(mlist.fullpath(), gtemplate)) - except IOError, e: - if e.errno <> errno.ENOENT: - raise - else: - tcksum = md5.new(fp.read()).digest() - fp.close() - if gcksum == tcksum: - os.unlink(os.path.join(mlist.fullpath(), gtemplate)) - # Match against the lists/<list>/*.prev template - try: - fp = open(os.path.join(mlist.fullpath(), gtemplate + '.prev')) - except IOError, e: - if e.errno <> errno.ENOENT: - raise - else: - tcksum = md5.new(fp.read()).digest() - fp.close() - if gcksum == tcksum: - os.unlink(os.path.join(mlist.fullpath(), gtemplate + '.prev')) - # Match against the lists/<list>/en/* templates - try: - fp = open(os.path.join(mlist.fullpath(), 'en', gtemplate)) - except IOError, e: - if e.errno <> errno.ENOENT: - raise - else: - tcksum = md5.new(fp.read()).digest() - fp.close() - if gcksum == tcksum: - os.unlink(os.path.join(mlist.fullpath(), 'en', gtemplate)) - # Match against the templates/* template - try: - fp = open(os.path.join(config.TEMPLATE_DIR, gtemplate)) - except IOError, e: - if e.errno <> errno.ENOENT: - raise - else: - tcksum = md5.new(fp.read()).digest() - fp.close() - if gcksum == tcksum: - os.unlink(os.path.join(config.TEMPLATE_DIR, gtemplate)) - # Match against the templates/*.prev template - try: - fp = open(os.path.join(config.TEMPLATE_DIR, gtemplate + '.prev')) - except IOError, e: - if e.errno <> errno.ENOENT: - raise - else: - tcksum = md5.new(fp.read()).digest() - fp.close() - if gcksum == tcksum: - os.unlink(os.path.join(config.TEMPLATE_DIR, - gtemplate + '.prev')) - - - -def situate_list(listname): - # This turns the directory called 'listname' into a directory called - # 'listname@domain'. Start by finding out what the domain should be. - # A list's domain is its email host. - mlist = MailList.MailList(listname, lock=False, check_version=False) - fullname = mlist.fqdn_listname - oldpath = os.path.join(config.VAR_PREFIX, 'lists', listname) - newpath = os.path.join(config.VAR_PREFIX, 'lists', fullname) - if os.path.exists(newpath): - print >> sys.stderr, _('WARNING: could not situate list: $listname') - else: - os.rename(oldpath, newpath) - print _('situated list $listname to $fullname') - return fullname - - - -def dolist(listname): - mlist = MailList.MailList(listname, lock=False) - try: - mlist.Lock(0.5) - except TimeOutError: - print >> sys.stderr, _( - 'WARNING: could not acquire lock for list: $listname') - return 1 - # Sanity check the invariant that every BYBOUNCE disabled member must have - # bounce information. Some earlier betas broke this. BAW: we're - # submerging below the MemberAdaptor interface, so skip this if we're not - # using OldStyleMemberships. - if isinstance(mlist._memberadaptor, OldStyleMemberships): - noinfo = {} - for addr, (reason, when) in mlist.delivery_status.items(): - if reason == BYBOUNCE and not mlist.bounce_info.has_key(addr): - noinfo[addr] = reason, when - # What to do about these folks with a BYBOUNCE delivery status and no - # bounce info? This number should be very small, and I think it's - # fine to simple re-enable them and let the bounce machinery - # re-disable them if necessary. - n = len(noinfo) - if n > 0: - print _( - 'Resetting $n BYBOUNCEs disabled addrs with no bounce info') - for addr in noinfo.keys(): - mlist.setDeliveryStatus(addr, ENABLED) - - mbox_dir = make_varabs('archives/private/%s.mbox' % (listname)) - mbox_file = make_varabs('archives/private/%s.mbox/%s' % (listname, - listname)) - o_pub_mbox_file = make_varabs('archives/public/%s' % (listname)) - o_pri_mbox_file = make_varabs('archives/private/%s' % (listname)) - html_dir = o_pri_mbox_file - o_html_dir = makeabs('public_html/archives/%s' % (listname)) - # Make the mbox directory if it's not there. - if not os.path.exists(mbox_dir): - makedirs(mbox_dir) - else: - # This shouldn't happen, but hey, just in case - if not os.path.isdir(mbox_dir): - print _("""\ -For some reason, $mbox_dir exists as a file. This won't work with b6, so I'm -renaming it to ${mbox_dir}.tmp and proceeding.""") - os.rename(mbox_dir, "%s.tmp" % (mbox_dir)) - makedirs(mbox_dir) - # Move any existing mboxes around, but watch out for both a public and a - # private one existing - if os.path.isfile(o_pri_mbox_file) and os.path.isfile(o_pub_mbox_file): - if mlist.archive_private: - print _("""\ - -$listname has both public and private mbox archives. Since this list -currently uses private archiving, I'm installing the private mbox archive -- -$o_pri_mbox_file -- as the active archive, and renaming - $o_pub_mbox_file -to - ${o_pub_mbox_file}.preb6 - -You can integrate that into the archives if you want by using the 'arch' -script. -""") % (mlist._internal_name, o_pri_mbox_file, o_pub_mbox_file, - o_pub_mbox_file) - os.rename(o_pub_mbox_file, "%s.preb6" % (o_pub_mbox_file)) - else: - print _("""\ -$mlist._internal_name has both public and private mbox archives. Since this -list currently uses public archiving, I'm installing the public mbox file -archive file ($o_pub_mbox_file) as the active one, and renaming -$o_pri_mbox_file to ${o_pri_mbox_file}.preb6 - -You can integrate that into the archives if you want by using the 'arch' -script. -""") - os.rename(o_pri_mbox_file, "%s.preb6" % (o_pri_mbox_file)) - # Move private archive mbox there if it's around - # and take into account all sorts of absurdities - print _('- updating old private mbox file') - if os.path.exists(o_pri_mbox_file): - if os.path.isfile(o_pri_mbox_file): - os.rename(o_pri_mbox_file, mbox_file) - elif not os.path.isdir(o_pri_mbox_file): - newname = "%s.mm_install-dunno_what_this_was_but_its_in_the_way" \ - % o_pri_mbox_file - os.rename(o_pri_mbox_file, newname) - print _("""\ - unknown file in the way, moving - $o_pri_mbox_file - to - $newname""") - else: - # directory - print _("""\ - looks like you have a really recent development installation... - you're either one brave soul, or you already ran me""") - # Move public archive mbox there if it's around - # and take into account all sorts of absurdities. - print _('- updating old public mbox file') - if os.path.exists(o_pub_mbox_file): - if os.path.isfile(o_pub_mbox_file): - os.rename(o_pub_mbox_file, mbox_file) - elif not os.path.isdir(o_pub_mbox_file): - newname = "%s.mm_install-dunno_what_this_was_but_its_in_the_way" \ - % o_pub_mbox_file - os.rename(o_pub_mbox_file, newname) - print _("""\ - unknown file in the way, moving - $o_pub_mbox_file - to - $newname""") - else: # directory - print _("""\ - looks like you have a really recent development installation... - you're either one brave soul, or you already ran me""") - # Move the html archives there - if os.path.isdir(o_html_dir): - os.rename(o_html_dir, html_dir) - # chmod the html archives - os.chmod(html_dir, 02775) - # BAW: Is this still necessary?! - mlist.Save() - # Check to see if pre-b4 list-specific templates are around - # and move them to the new place if there's not already - # a new one there - tmpl_dir = os.path.join(config.PREFIX, "templates") - list_dir = os.path.join(config.PREFIX, "lists") - b4_tmpl_dir = os.path.join(tmpl_dir, mlist._internal_name) - new_tmpl_dir = os.path.join(list_dir, mlist._internal_name) - if os.path.exists(b4_tmpl_dir): - print _("""\ -- This list looks like it might have <= b4 list templates around""") - for f in os.listdir(b4_tmpl_dir): - o_tmpl = os.path.join(b4_tmpl_dir, f) - n_tmpl = os.path.join(new_tmpl_dir, f) - if os.path.exists(o_tmpl): - if not os.path.exists(n_tmpl): - os.rename(o_tmpl, n_tmpl) - print _('- moved $o_tmpl to $n_tmpl') - else: - print _("""\ -- both $o_tmpl and $n_tmpl exist, leaving untouched""") - else: - print _("""\ -- $o_tmpl doesn't exist, leaving untouched""") - # Move all the templates to the en language subdirectory as required for - # Mailman 2.1 - move_language_templates(mlist) - # Avoid eating filehandles with the list lockfiles - mlist.Unlock() - return 0 - - - -def archive_path_fixer(unused_arg, dir, files): - # Passed to os.path.walk to fix the perms on old html archives. - for f in files: - abs = os.path.join(dir, f) - if os.path.isdir(abs): - if f == "database": - os.chmod(abs, 02770) - else: - os.chmod(abs, 02775) - elif os.path.isfile(abs): - os.chmod(abs, 0664) - - -def remove_old_sources(module): - # Also removes old directories. - src = '%s/%s' % (config.PREFIX, module) - pyc = src + "c" - if os.path.isdir(src): - print _('removing directory $src and everything underneath') - shutil.rmtree(src) - elif os.path.exists(src): - print _('removing $src') - try: - os.unlink(src) - except os.error, rest: - print _("Warning: couldn't remove $src -- $rest") - if module.endswith('.py') and os.path.exists(pyc): - try: - os.unlink(pyc) - except OSError, rest: - print _("couldn't remove old file $pyc -- $rest") - - - -def update_qfiles(): - print _('updating old qfiles') - prefix = `time.time()` + '+' - # Be sure the qfiles/in directory exists (we don't really need the - # switchboard object, but it's convenient for creating the directory). - sb = Switchboard(config.INQUEUE_DIR) - for filename in os.listdir(config.QUEUE_DIR): - # Updating means just moving the .db and .msg files to qfiles/in where - # it should be dequeued, converted, and processed normally. - if os.path.splitext(filename) == '.msg': - oldmsgfile = os.path.join(config.QUEUE_DIR, filename) - newmsgfile = os.path.join(config.INQUEUE_DIR, prefix + filename) - os.rename(oldmsgfile, newmsgfile) - elif os.path.splitext(filename) == '.db': - olddbfile = os.path.join(config.QUEUE_DIR, filename) - newdbfile = os.path.join(config.INQUEUE_DIR, prefix + filename) - os.rename(olddbfile, newdbfile) - # Now update for the Mailman 2.1.5 qfile format. For every filebase in - # the qfiles/* directories that has both a .pck and a .db file, pull the - # data out and re-queue them. - for dirname in os.listdir(config.QUEUE_DIR): - dirpath = os.path.join(config.QUEUE_DIR, dirname) - if dirpath == config.BADQUEUE_DIR: - # The files in qfiles/bad can't possibly be pickles - continue - sb = Switchboard(dirpath) - try: - for filename in os.listdir(dirpath): - filepath = os.path.join(dirpath, filename) - filebase, ext = os.path.splitext(filepath) - # Handle the .db metadata files as part of the handling of the - # .pck or .msg message files. - if ext not in ('.pck', '.msg'): - continue - msg, data = dequeue(filebase) - if msg is not None and data is not None: - sb.enqueue(msg, data) - except EnvironmentError, e: - if e.errno <> errno.ENOTDIR: - raise - print _('Warning! Not a directory: $dirpath') - - - -# Implementations taken from the pre-2.1.5 Switchboard -def ext_read(filename): - fp = open(filename) - d = marshal.load(fp) - # Update from version 2 files - if d.get('version', 0) == 2: - del d['filebase'] - # Do the reverse conversion (repr -> float) - for attr in ['received_time']: - try: - sval = d[attr] - except KeyError: - pass - else: - # Do a safe eval by setting up a restricted execution - # environment. This may not be strictly necessary since we - # know they are floats, but it can't hurt. - d[attr] = eval(sval, {'__builtins__': {}}) - fp.close() - return d - - -def dequeue(filebase): - # Calculate the .db and .msg filenames from the given filebase. - msgfile = os.path.join(filebase + '.msg') - pckfile = os.path.join(filebase + '.pck') - dbfile = os.path.join(filebase + '.db') - # Now we are going to read the message and metadata for the given - # filebase. We want to read things in this order: first, the metadata - # file to find out whether the message is stored as a pickle or as - # plain text. Second, the actual message file. However, we want to - # first unlink the message file and then the .db file, because the - # qrunner only cues off of the .db file - msg = None - try: - data = ext_read(dbfile) - os.unlink(dbfile) - except EnvironmentError, e: - if e.errno <> errno.ENOENT: - raise - data = {} - # Between 2.1b4 and 2.1b5, the `rejection-notice' key in the metadata - # was renamed to `rejection_notice', since dashes in the keys are not - # supported in METAFMT_ASCII. - if data.has_key('rejection-notice'): - data['rejection_notice'] = data['rejection-notice'] - del data['rejection-notice'] - msgfp = None - try: - try: - msgfp = open(pckfile) - msg = cPickle.load(msgfp) - os.unlink(pckfile) - except EnvironmentError, e: - if e.errno <> errno.ENOENT: raise - msgfp = None - try: - msgfp = open(msgfile) - msg = email.message_from_file(msgfp, Message.Message) - os.unlink(msgfile) - except EnvironmentError, e: - if e.errno <> errno.ENOENT: raise - except (email.Errors.MessageParseError, ValueError), e: - # This message was unparsable, most likely because its - # MIME encapsulation was broken. For now, there's not - # much we can do about it. - print _('message is unparsable: $filebase') - msgfp.close() - msgfp = None - if config.QRUNNER_SAVE_BAD_MESSAGES: - # Cheapo way to ensure the directory exists w/ the - # proper permissions. - sb = Switchboard(config.BADQUEUE_DIR) - os.rename(msgfile, os.path.join( - config.BADQUEUE_DIR, filebase + '.txt')) - else: - os.unlink(msgfile) - msg = data = None - except EOFError: - # For some reason the pckfile was empty. Just delete it. - print _('Warning! Deleting empty .pck file: $pckfile') - os.unlink(pckfile) - finally: - if msgfp: - msgfp.close() - return msg, data - - - -def main(): - parser, opts, args = parseargs() - initialize(opts.config) - - # calculate the versions - lastversion, thisversion = calcversions() - hexlversion = hex(lastversion) - hextversion = hex(thisversion) - if lastversion == thisversion and not opts.force: - # nothing to do - print _('No updates are necessary.') - sys.exit(0) - if lastversion > thisversion and not opts.force: - print _("""\ -Downgrade detected, from version $hexlversion to version $hextversion -This is probably not safe. -Exiting.""") - sys.exit(1) - print _('Upgrading from version $hexlversion to $hextversion') - errors = 0 - # get rid of old stuff - print _('getting rid of old source files') - for mod in ('mailman/Archiver.py', 'mailman/HyperArch.py', - 'mailman/HyperDatabase.py', 'mailman/pipermail.py', - 'mailman/smtplib.py', 'mailman/Cookie.py', - 'bin/update_to_10b6', 'scripts/mailcmd', - 'scripts/mailowner', 'mail/wrapper', 'mailman/pythonlib', - 'cgi-bin/archives', 'mailman/MailCommandHandler'): - remove_old_sources(mod) - if not config.list_manager.names: - print _('no lists == nothing to do, exiting') - return - # For people with web archiving, make sure the directories - # in the archiving are set with proper perms for b6. - if os.path.isdir("%s/public_html/archives" % config.PREFIX): - print _("""\ -fixing all the perms on your old html archives to work with b6 -If your archives are big, this could take a minute or two...""") - os.path.walk("%s/public_html/archives" % config.PREFIX, - archive_path_fixer, "") - print _('done') - for listname in config.list_manager.names: - # With 2.2.0a0, all list names grew an @domain suffix. If you find a - # list without that, move it now. - if not '@' in listname: - listname = situate_list(listname) - print _('Updating mailing list: $listname') - errors += dolist(listname) - print - print _('Updating Usenet watermarks') - wmfile = os.path.join(config.DATA_DIR, 'gate_watermarks') - try: - fp = open(wmfile) - except IOError: - print _('- nothing to update here') - else: - d = marshal.load(fp) - fp.close() - for listname in d.keys(): - if listname not in listnames: - # this list no longer exists - continue - mlist = MailList.MailList(listname, lock=0) - try: - mlist.Lock(0.5) - except TimeOutError: - print >> sys.stderr, _( - 'WARNING: could not acquire lock for list: $listname') - errors = errors + 1 - else: - # Pre 1.0b7 stored 0 in the gate_watermarks file to indicate - # that no gating had been done yet. Without coercing this to - # None, the list could now suddenly get flooded. - mlist.usenet_watermark = d[listname] or None - mlist.Save() - mlist.Unlock() - os.unlink(wmfile) - print _('- usenet watermarks updated and gate_watermarks removed') - # In Mailman 2.1, the qfiles directory has a different structure and a - # different content. Also, in Mailman 2.1.5 we collapsed the message - # files from separate .msg (pickled Message objects) and .db (marshalled - # dictionaries) to a shared .pck file containing two pickles. - update_qfiles() - # This warning was necessary for the upgrade from 1.0b9 to 1.0b10. - # There's no good way of figuring this out for releases prior to 2.0beta2 - # :( - if lastversion == NOTFRESH: - print _(""" - -NOTE NOTE NOTE NOTE NOTE - - You are upgrading an existing Mailman installation, but I can't tell what - version you were previously running. - - If you are upgrading from Mailman 1.0b9 or earlier you will need to - manually update your mailing lists. For each mailing list you need to - copy the file templates/options.html lists/<listname>/options.html. - - However, if you have edited this file via the Web interface, you will have - to merge your changes into this file, otherwise you will lose your - changes. - -NOTE NOTE NOTE NOTE NOTE - -""") - if not errors: - # Record the version we just upgraded to - fp = open(os.path.join(config.DATA_DIR, 'last_mailman_version'), 'w') - fp.write(hex(config.HEX_VERSION) + '\n') - fp.close() - else: - lockdir = config.LOCK_DIR - print _('''\ - -ERROR: - -The locks for some lists could not be acquired. This means that either -Mailman was still active when you upgraded, or there were stale locks in the -$lockdir directory. - -You must put Mailman into a quiescent state and remove all stale locks, then -re-run "make update" manually. See the INSTALL and UPGRADE files for details. -''') diff --git a/src/mailman/chains/accept.py b/src/mailman/chains/accept.py index b1e4b1cf0..4b326142b 100644 --- a/src/mailman/chains/accept.py +++ b/src/mailman/chains/accept.py @@ -62,7 +62,6 @@ class AcceptChain(TerminalChainBase): rule_misses = msgdata.get('rule_misses') if rule_misses: msg['X-Mailman-Rule-Misses'] = SEMISPACE.join(rule_misses) - accept_queue = config.switchboards['pipeline'] - accept_queue.enqueue(msg, msgdata) + config.switchboards['pipeline'].enqueue(msg, msgdata) log.info('ACCEPT: %s', msg.get('message-id', 'n/a')) notify(AcceptNotification(mlist, msg, msgdata, self)) diff --git a/src/mailman/chains/owner.py b/src/mailman/chains/owner.py new file mode 100644 index 000000000..ad0a04cea --- /dev/null +++ b/src/mailman/chains/owner.py @@ -0,0 +1,56 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""The standard -owner posting chain.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'BuiltInOwnerChain', + ] + + +import logging + +from zope.event import notify + +from mailman.chains.base import ChainNotification, TerminalChainBase +from mailman.config import config +from mailman.core.i18n import _ + + +log = logging.getLogger('mailman.vette') + + + +class OwnerNotification(ChainNotification): + """An event signaling that a message is accepted to the -owner address.""" + + + +class BuiltInOwnerChain(TerminalChainBase): + """Default built-in -owner address chain.""" + + name = 'default-owner-chain' + description = _('The built-in -owner posting chain.') + + def _process(self, mlist, msg, msgdata): + # At least for now, everything posted to -owners goes through. + config.switchboards['pipeline'].enqueue(msg, msgdata) + log.info('OWNER: %s', msg.get('message-id', 'n/a')) + notify(OwnerNotification(mlist, msg, msgdata, self)) diff --git a/src/mailman/chains/tests/test_owner.py b/src/mailman/chains/tests/test_owner.py new file mode 100644 index 000000000..db85d4967 --- /dev/null +++ b/src/mailman/chains/tests/test_owner.py @@ -0,0 +1,75 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test the owner chain.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestOwnerChain', + ] + + +import unittest + +from mailman.app.lifecycle import create_list +from mailman.chains.owner import BuiltInOwnerChain, OwnerNotification +from mailman.core.chains import process +from mailman.testing.helpers import ( + event_subscribers, + get_queue_messages, + specialized_message_from_string as mfs) +from mailman.testing.layers import ConfigLayer + + + +class TestOwnerChain(unittest.TestCase): + """Test the owner chain.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._msg = mfs("""\ +From: anne@example.com +To: test@example.com +Message-ID: <ant> + +""") + + def test_owner_pipeline(self): + # Messages processed through the default owners chain end up in the + # pipeline queue, and an event gets sent. + # + # This event subscriber records the event that occurs when the message + # is processed by the owner chain. + events = [] + def catch_event(event): + events.append(event) + with event_subscribers(catch_event): + process(self._mlist, self._msg, {}, 'default-owner-chain') + self.assertEqual(len(events), 1) + event = events[0] + self.assertTrue(isinstance(event, OwnerNotification)) + self.assertEqual(event.mlist, self._mlist) + self.assertEqual(event.msg['message-id'], '<ant>') + self.assertTrue(isinstance(event.chain, BuiltInOwnerChain)) + messages = get_queue_messages('pipeline') + self.assertEqual(len(messages), 1) + message = messages[0].msg + self.assertEqual(message['message-id'], '<ant>') diff --git a/src/mailman/commands/docs/inject.rst b/src/mailman/commands/docs/inject.rst index 7150beac7..e8987128c 100644 --- a/src/mailman/commands/docs/inject.rst +++ b/src/mailman/commands/docs/inject.rst @@ -35,7 +35,7 @@ It's easy to find out which queues are available. digest in lmtp - news + nntp out pipeline rest diff --git a/src/mailman/commands/docs/membership.rst b/src/mailman/commands/docs/membership.rst index 638705e91..3faccfe6a 100644 --- a/src/mailman/commands/docs/membership.rst +++ b/src/mailman/commands/docs/membership.rst @@ -292,8 +292,8 @@ Once Anne has verified her alternative address though, it can be used to unsubscribe her from the list. :: - >>> from datetime import datetime - >>> address.verified_on = datetime.now() + >>> from mailman.utilities.datetime import now + >>> address.verified_on = now() >>> results = Results() >>> print leave.process(mlist, msg, {}, (), results) diff --git a/src/mailman/commands/docs/qfile.rst b/src/mailman/commands/docs/qfile.rst index 74ede1b64..8ec0a3952 100644 --- a/src/mailman/commands/docs/qfile.rst +++ b/src/mailman/commands/docs/qfile.rst @@ -59,7 +59,6 @@ Once we've figured out the file name of the shunted message, we can print it. 'bad': u'yes', 'bar': u'baz', 'foo': 7, - u'received_time': ... u'version': 3} [----- end pickle -----] diff --git a/src/mailman/config/mailman.cfg b/src/mailman/config/mailman.cfg index 0d37ceed9..10ef0ba17 100644 --- a/src/mailman/config/mailman.cfg +++ b/src/mailman/config/mailman.cfg @@ -61,8 +61,8 @@ class: mailman.runners.incoming.IncomingRunner [runner.lmtp] class: mailman.runners.lmtp.LMTPRunner -[runner.news] -class: mailman.runners.news.NewsRunner +[runner.nntp] +class: mailman.runners.nntp.NNTPRunner [runner.out] class: mailman.runners.outgoing.OutgoingRunner diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg index 3344e965a..44cbf6f4e 100644 --- a/src/mailman/config/schema.cfg +++ b/src/mailman/config/schema.cfg @@ -540,6 +540,22 @@ recipient: archive@archive.example.com # command to call. command: /bin/echo +# When sending the message to the archiver, you have the option of +# "clobbering" the Date: header, specifically to make it more sane. Some +# archivers can't handle dates that are wildly off from reality. This does +# not change the Date: header for any other delivery vector except this +# specific archive. +# +# When the original Date header is clobbered, it will always be stored in +# X-Original-Date. The new Date header will always be set to the date at +# which the messages was received by the Mailman server, in UTC. +# +# Your options here are: +# * never -- Leaves the original Date header alone. +# * always -- Always override the Date header. +# * maybe -- Override the Date only if it is outside the clobber_skew period. +clobber_date: maybe +clobber_skew: 1d [archiver.mhonarc] # This is the stock MHonArc archiver. @@ -585,13 +601,15 @@ plain_digest_keep_headers: [nntp] # Set these variables if you need to authenticate to your NNTP server for -# Usenet posting or reading. If no authentication is necessary, specify None -# for both variables. -username: +# Usenet posting or reading. Leave these blank if no authentication is +# necessary. +user: password: -# Set this if you have an NNTP server you prefer gatewayed lists to use. +# Host and port of the NNTP server to connect to. Leave these blank to use +# the default localhost:119. host: +port: # This controls how headers must be cleansed in order to be accepted by your # NNTP server. Some servers like INN reject messages containing prohibited diff --git a/src/mailman/core/pipelines.py b/src/mailman/core/pipelines.py index d5cee588b..25bb68030 100644 --- a/src/mailman/core/pipelines.py +++ b/src/mailman/core/pipelines.py @@ -15,12 +15,16 @@ # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see <http://www.gnu.org/licenses/>. -"""Pipeline processor.""" +"""Built-in pipelines.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ + 'BasePipeline', + 'OwnerPipeline', + 'PostingPipeline', + 'VirginPipeline', 'initialize', 'process', ] @@ -89,16 +93,29 @@ class BasePipeline: yield handler -class BuiltInPipeline(BasePipeline): - """The built-in pipeline.""" + +class OwnerPipeline(BasePipeline): + """The built-in owner pipeline.""" + + name = 'default-owner-pipeline' + description = _('The built-in owner pipeline.') + + _default_handlers = ( + 'owner-recipients', + 'to-outgoing', + ) + + +class PostingPipeline(BasePipeline): + """The built-in posting pipeline.""" name = 'default-posting-pipeline' - description = _('The built-in pipeline.') + description = _('The built-in posting pipeline.') _default_handlers = ( 'mime-delete', 'tagger', - 'calculate-recipients', + 'member-recipients', 'avoid-duplicates', 'cleanse', 'cleanse-dkim', @@ -131,7 +148,7 @@ class VirginPipeline(BasePipeline): def initialize(): """Initialize the pipelines.""" # Find all handlers in the registered plugins. - for handler_class in find_components('mailman.pipeline', IHandler): + for handler_class in find_components('mailman.handlers', IHandler): handler = handler_class() verifyObject(IHandler, handler) assert handler.name not in config.handlers, ( @@ -139,6 +156,6 @@ def initialize(): handler.name, handler_class)) config.handlers[handler.name] = handler # Set up some pipelines. - for pipeline_class in (BuiltInPipeline, VirginPipeline): + for pipeline_class in (OwnerPipeline, PostingPipeline, VirginPipeline): pipeline = pipeline_class() config.pipelines[pipeline.name] = pipeline diff --git a/src/mailman/core/switchboard.py b/src/mailman/core/switchboard.py index ba8b2ec3f..7cab4f4ad 100644 --- a/src/mailman/core/switchboard.py +++ b/src/mailman/core/switchboard.py @@ -137,8 +137,7 @@ class Switchboard: # file name consists of two parts separated by a '+': the received # time for this message (i.e. when it first showed up on this system) # and the sha hex digest. - rcvtime = data.setdefault('received_time', now) - filebase = repr(rcvtime) + '+' + hashlib.sha1(hashfood).hexdigest() + filebase = repr(now) + '+' + hashlib.sha1(hashfood).hexdigest() filename = os.path.join(self.queue_directory, filebase + '.pck') tmpfile = filename + '.tmp' # Always add the metadata schema version number diff --git a/src/mailman/core/tests/test_pipelines.py b/src/mailman/core/tests/test_pipelines.py index 0cf3732c9..8f851de95 100644 --- a/src/mailman/core/tests/test_pipelines.py +++ b/src/mailman/core/tests/test_pipelines.py @@ -21,11 +21,14 @@ from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ + 'TestOwnerPipeline', + 'TestPostingPipeline', ] import unittest +from zope.component import getUtility from zope.interface import implements from mailman.app.lifecycle import create_list @@ -33,7 +36,9 @@ from mailman.config import config from mailman.core.errors import DiscardMessage, RejectMessage from mailman.core.pipelines import process from mailman.interfaces.handler import IHandler +from mailman.interfaces.member import MemberRole from mailman.interfaces.pipeline import IPipeline +from mailman.interfaces.usermanager import IUserManager from mailman.testing.helpers import ( LogFileMark, get_queue_messages, @@ -78,7 +83,7 @@ class RejectingPipeline: -class TestBuiltinPipeline(unittest.TestCase): +class TestPostingPipeline(unittest.TestCase): """Test various aspects of the built-in postings pipeline.""" layer = ConfigLayer @@ -133,3 +138,42 @@ testing messages = get_queue_messages('virgin') self.assertEqual(len(messages), 1) self.assertEqual(str(messages[0].msg['subject']), 'a test') + + + +class TestOwnerPipeline(unittest.TestCase): + """Test various aspects of the built-in owner pipeline.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + user_manager = getUtility(IUserManager) + anne = user_manager.create_address('anne@example.com') + bart = user_manager.create_address('bart@example.com') + self._mlist.subscribe(anne, MemberRole.owner) + self._mlist.subscribe(bart, MemberRole.moderator) + self._msg = mfs("""\ +From: Anne Person <anne@example.org> +To: test-owner@example.com + +""") + + def test_calculate_recipients(self): + # Recipients are the administrators of the mailing list. + msgdata = dict(listname='test@example.com', + to_owner=True) + process(self._mlist, self._msg, msgdata, + pipeline_name='default-owner-pipeline') + self.assertEqual(msgdata['recipients'], set(('anne@example.com', + 'bart@example.com'))) + + def test_to_outgoing(self): + # The message, with the calculated recipients, gets put in the + # outgoing queue. + process(self._mlist, self._msg, {}, + pipeline_name='default-owner-pipeline') + messages = get_queue_messages('out', sort_on='to') + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0].msgdata['recipients'], + set(('anne@example.com', 'bart@example.com'))) diff --git a/src/mailman/database/schema/postgres.sql b/src/mailman/database/schema/postgres.sql index bd7ef3f6b..2e9ba249f 100644 --- a/src/mailman/database/schema/postgres.sql +++ b/src/mailman/database/schema/postgres.sql @@ -84,6 +84,8 @@ CREATE TABLE mailinglist ( nondigestable BOOLEAN, nonmember_rejection_notice TEXT, obscure_addresses BOOLEAN, + owner_chain TEXT, + owner_pipeline TEXT, personalize INTEGER, post_id INTEGER, posting_chain TEXT, diff --git a/src/mailman/database/schema/sqlite.sql b/src/mailman/database/schema/sqlite.sql index 37b6ed8f2..e6211bf53 100644 --- a/src/mailman/database/schema/sqlite.sql +++ b/src/mailman/database/schema/sqlite.sql @@ -180,6 +180,8 @@ CREATE TABLE mailinglist ( nondigestable BOOLEAN, nonmember_rejection_notice TEXT, obscure_addresses BOOLEAN, + owner_chain TEXT, + owner_pipeline TEXT, personalize INTEGER, post_id INTEGER, posting_chain TEXT, diff --git a/src/mailman/docs/ArchiveUIin5.rst b/src/mailman/docs/ArchiveUIin5.rst new file mode 100644 index 000000000..58b48cd3d --- /dev/null +++ b/src/mailman/docs/ArchiveUIin5.rst @@ -0,0 +1,97 @@ +===================================== +Set up the archive ui in five minutes +===================================== + +The `hyperkitty`_ application aims at providing an interface to visualize and +explore Mailman archives. + +This is a `Django`_ project. + +Requirements +============ + +- A mail archive in `maildir format`_ (no, you don't need a running Mailman + 3!) Eventually hyperkitty will support `mbox format`_ for backward + compatibility with *Pipermail*, and *zipped maildirs* seem like a good idea + to save space. **Beware:** Although you'd think that we would be able to + manipulate the venerable *mbox* format safely and efficiently, that doesn't + seem to be the case. *Maildir* archives are **strongly** preferred, because + they are more robust to program bugs (whether in Mailman, hyperkitty, or in + the originating MUA!) +- Django is the web framework that supports the UI. +- `bunch`_ DOES WHAT? +- The `notmuch mail indexer`_ is used to generate indexes (and requires + `Xapian`_). +- hyperkitty itself, which is a UI, and not responsible for maintaining the + message archive itself. (Since the archive is in `maildir format`_, any + modern MTA or MDA can build one for you.) + + +Get it running (under virtualenv): +---------------------------------- + +It is generally a good idea to use `virtualenv`_ to create a stable +environment for your Python applications. + +- Create the virtualenv:: + + % virtualenv mailman3 + +- Activate the virtualenv:: + + % cd mailman3 + % source bin/activate + +You don't *have* to use virtualenv, though, and if you don't want to, just +omit the preceding steps. Continue with these steps. + +- Install Django and dependencies:: + + % easy_install django + % easy_install bunch + +- Install notmuch -- these are bindings that come with the notmuch C library. + The easiest way is probably to install them for your OS vendor and then + symlink them into the virtualenv similar to this:: + + % yum install -y python-notmuch + +- Note: on a multiarch system like Fedora, the directories may be lib64 rather + than lib on 64 bit systems. Next:: + + % cd lib/python2.7/site-packages + % ln -s /usr/lib/python2.7/site-packages/notmuch . + +- Note: this is the version of notmuch I tested with; others may work:: + + % ln -s /usr/lib/python2.7/site-packages/notmuch-0.11-py2.7.egg-info . + +- Install the hyperkitty sources:: + + % git clone http://ambre.pingoured.fr/cgit/hyperkitty.git/ + + +Running hyperkitty +------------------ + +- Start it:: + + % cd hyperkitty + +- Put the static content where it should be:: + + % python manage.py collectstatic + +- Run the Django server:: + + % python manage.py runserver + + +.. _`hyperkitty`: https://launchpad.net/hyperkitty +.. _`Django`: http://djangoproject.org/ +.. _`notmuch mail indexer`: http://notmuchmail.org +.. _`bunch`: http://pypi.python.org/pypi/bunch +.. _`Xapian`: http://xapian.org/ +.. _`maildir format`: http://en.wikipedia.org/wiki/Maildir +.. _`mbox format`: http://en.wikipedia.org/wiki/Mbox +.. _`virtualenv`: http://pypi.python.org/pypi/virtualenv diff --git a/src/mailman/docs/NEWS.rst b/src/mailman/docs/NEWS.rst index 21deef544..71a2d0cdc 100644 --- a/src/mailman/docs/NEWS.rst +++ b/src/mailman/docs/NEWS.rst @@ -8,9 +8,48 @@ Copyright (C) 1998-2012 by the Free Software Foundation, Inc. Here is a history of user visible changes to Mailman. +3.0 beta 2 -- "Freeze" +====================== +(20XX-XX-XX) + +Architecture +------------ + * Internally, all datetimes are kept in the UTC timezone, however because of + LP: #280708, they are stored in the database in naive format. + * `received_time` is now added to the message metadata by the LMTP runner + instead of by `Switchboard.enqueue()`. This latter no longer depends on + `received_time` in the metadata. + * The `ArchiveRunner` no longer acquires a lock before it calls the + individual archiver implementations, since not all of them need a lock. If + they do, the implementations must acquire said lock themselves. + * The `news` runner and queue has been renamed to the more accurate `nntp`. + The runner has also been ported to Mailman 3 (LP: #967409). Beta testers + can can safely remove `$var_dir/queue/news`. + +Configuration +------------- + * New configuration variables `clobber_date` and `clobber_skew` supported in + every `[archiver.<name>]` section. These are used to determine under what + circumstances a message destined for a specific archiver should have its + `Date:` header clobbered. (LP: #963612) + * Configuration schema variable changes: + [nntp]username -> [nntp]user + [nntp]port (added) + +Documentation +------------- + * Some additional documentation on related components such as Postorius and + hyperkitty have been added, given by Stephen J Turnbull. + +Bug fixes +--------- + * Fixed a UnicodeError with non-ascii message bodies in the `approved` rule, + given by Mark Sapiro. (LP: #949924) + + 3.0 beta 1 -- "The Twilight Zone" ================================= -(20XX-XX-XX) +(2012-03-23) Architecture ------------ @@ -75,6 +114,8 @@ Database - real_name -> display_name (mailinglist, user, address) * Schema additions: - mailinglist.filter_action + - mailinglist.owner_chain + - mailinglist.owner_pipeline REST ---- @@ -126,8 +167,6 @@ Commands Bug fixes --------- - * Fixed a UnicodeError with non-ascii message bodies in rules/approved.py - (LP: #949924) * Subscription disabled probe warning notification messages are now sent without a `Precedence:` header. Given by Mark Sapiro. (LP: #808821) * Fixed KeyError in retry runner, contributed by Stephen A. Goss. diff --git a/src/mailman/docs/START.rst b/src/mailman/docs/START.rst index ab2717956..57df48e30 100644 --- a/src/mailman/docs/START.rst +++ b/src/mailman/docs/START.rst @@ -2,39 +2,61 @@ Getting started with GNU Mailman ================================ -Copyright (C) 2008-2011 by the Free Software Foundation, Inc. +Copyright (C) 2008-2012 by the Free Software Foundation, Inc. -Alpha Release -============= +Beta Release +============ -The Mailman 3 alpha releases are being provided to give developers and other -interested people an early look at the next major version. As such, some -things may not work yet. Your participation is encouraged. Your feedback and -contributions are welcome. Please submit bug reports on the Mailman bug +This is a beta release. The developers believe it has sufficient +functionality to provide full services to a mailing list, but it is not ready +for production yet. Interfaces and administration may differ substantially +from the alpha series, but changes should be incremental going forward from +beta 1. Changes from the alpha series will be described in notes to the main +text. + +The Mailman 3 beta releases are being provided to give developers and other +interested people an early look at the next major version, and site +administrators a chance to prepare for an eventual upgrade. The core list +management and post distribution functionality is now complete. However, +unlike Mailman 2 whose web interface and archives were tightly integrated with +the core, Mailman 3 exposes a REST administrative interface to the web, +communicates with archivers via decoupled interfaces, and leaves summary, +search, and retrieval of archived messages to a separate application (a simple +implementation is provided). As of beta 1 the web interface and archiver are +still at an early stage of development. As such, some things may not work. + +Contributions are welcome. Please submit bug reports on the Mailman bug tracker at https://bugs.launchpad.net/mailman though you will currently need to have a login on Launchpad to do so. You can also send email to the mailman-developers@python.org mailing list. -Using the Alpha -=============== +Requirements +============ Python 2.6 or 2.7 is required. It can either be the default 'python' on your $PATH or it can be accessible via the ``python2.6`` or ``python2.7`` binary. If your operating system does not include Python, see http://www.python.org downloading and installing it from source. Python 3 is not yet supported. - -Building Mailman 3 -================== +In this documentation, a bare ``python`` refers to the python used to invoke +``bootstrap.py``, which might be ``python2.6`` or ``python2.7``, as well as +the system ``python`` or an absolute path. Mailman 3 is now based on the `zc.buildout`_ infrastructure, which greatly simplifies building and testing Mailman. -You do not need anything other than Python and an internet connection to get -all the other Mailman 3 dependencies. Here are the commands to build -everything:: +During the beta program, you may need some additional dependencies, such as a +C compiler and the Python development headers and libraries. You will need an +internet connection. Also `Postorius`_ (the web UI) and the `archive UI`_ are +distributed and installed separately. + + +Building Mailman 3 +================== + +Here are the commands to build everything in core Mailman:: % python bootstrap.py % bin/buildout @@ -50,16 +72,16 @@ You should see no failures. Build the online docs by running:: - % bin/docs + % python setup.py build_sphinx -(You might get warnings which you can safely ignore.) Then visit +(You might get warnings which you can safely ignore.) Then visit:: - parts/docs/mailman/build/mailman/docs/README.html + build/sphinx/html/README.html in your browser to start reading the documentation. Or you can just read the doctests by looking in all the 'doc' directories under the 'mailman' package. Doctests are documentation first, so they should give you a pretty good idea -how various components of Mailman 3 works. +how various components of Mailman 3 work. Running Mailman 3 @@ -88,6 +110,7 @@ The first existing file found wins. * ``./mailman.cfg`` * ``~/.mailman.cfg`` * ``/etc/mailman.cfg`` + * ``argv[0]/../../etc/mailman.cfg`` Run the ``bin/mailman info`` command to see which configuration file Mailman will use, and where it will put its database file. The first time you run @@ -98,10 +121,55 @@ Try ``bin/mailman --help`` for more details. You can use the commands ``bin/mailman start`` to start the runner subprocess daemons, and of course ``bin/mailman stop`` to stop them. -The `web ui`_ is being developed as a separate, Django-based project. For -now, all configuration happens via the command line and REST API. +Postorius is being developed as a separate, Django-based project. For now, +all configuration happens via the command line and REST API. + + +Mailman Web UI +-------------- + +The Mailman 3 web UI, called *Postorius*, interfaces to core Mailman engine +via the REST client API. It is expected that this architecture will make it +possible for users with other needs to adapt the web UI, or even replace it +entirely, with a reasonable amount of effort. However, as a core feature of +Mailman, the web UI will emphasize usability over modularity at first, so most +users should use the web UI described here. + +Postorius was prototyped at the `Pycon 2012 sprint`_, so it is "very alpha" as +of Mailman 3 beta 1, and comes in several components. In particular, it +requires a `Django`_ installation, and Bazaar checkouts of the `REST client +module`_ and `Postorius`_ itself. Building it is fairly straightforward, +however, given Florian Fuchs' `Five Minute Guide` from his `blog post`_ on the +Mailman wiki. (Check the `blog post`_ for the most recent version!) + + +The List Archiver +----------------- + +In Mailman 3, the archivers are decoupled from the core engine. It is useful +to provide a simple, standard interface for third-party archiving tools and +services. For this reason, Mailman 3 defines a formal interface to insert +messages into any of a number of configured archivers, using whatever protocol +is appropriate for that archiver. Summary, search, and retrieval of archived +posts are handled by a separate application. + +A new `archive UI`_ called Hyperkitty, based on the `notmuch mail indexer`_ +and `Django`_, was prototyped at the PyCon 2012 sprint by Toshio Kuratomi, and +like the web UI it is also in early alpha as of Mailman 3 beta 1. The +"hyperkitty" archiver is very loosely coupled to Mailman 3 core. In fact, any +email application that speaks LMTP or SMTP will be able to use hyperkitty. + +A `five minute guide to hyperkitty`_ is based on Toshio Kuratomi's README. .. _`zc.buildout`: http://pypi.python.org/pypi/zc.buildout .. _`lazr.config`: http://pypi.python.org/pypi/lazr.config -.. _`web ui`: https://launchpad.net/mailmanweb +.. _`Postorius`: https://launchpad.net/postorius +.. _`archive UI`: https://launchpad.net/hyperkitty +.. _`Django`: http://djangoproject.org/ +.. _`REST client module`: https://launchpad.net/mailman.client +.. _`Five Minute Guide the Web UI`: WebUIin5.html +.. _`blog post`: http://wiki.list.org/display/DEV/A+5+minute+guide+to+get+the+Mailman+web+UI+running +.. _`notmuch mail indexer`: http://notmuchmail.org +.. _`five minute guide to hyperkitty`: ArchiveUIin5.html +.. _`Pycon 2012 sprint`: https://us.pycon.org/2012/community/sprints/projects/ diff --git a/src/mailman/docs/WebUIin5.rst b/src/mailman/docs/WebUIin5.rst new file mode 100644 index 000000000..56f2df9fb --- /dev/null +++ b/src/mailman/docs/WebUIin5.rst @@ -0,0 +1,113 @@ +================================ +Set up Postorius in five minutes +================================ + +This is a quick guide for setting up a development environment to work on +Mailman 3's web UI, called Postorius. If all goes as planned, you should be +done within 5 minutes. This has been tested on Ubuntu 11.04. + +In order to download the components necessary you need to have the `Bazaar`_ +version control system installed on your system. Mailman and mailman.client +need at least Python version 2.6. + +It's probably a good idea to set up a virtual Python environment using +`virtualenv`_. `Here is a brief HOWTO`_. + +.. _`virtualenv`: http://pypi.python.org/pypi/virtualenv +.. _`Here is a brief HOWTO`: ./ArchiveUIin5.html#get-it-running-under-virtualenv +.. _`Bazaar`: http://bazaar.canonical.com/en/ + + +GNU Mailman 3 +============= + +First download the latest revision of Mailman 3 from Launchpad. +:: + + $ bzr branch lp:mailman + +Install and test:: + + $ cd mailman + $ python bootstrap.py + $ bin/buildout + $ bin/test + +If you get no errors you can now start Mailman:: + + $ bin/mailman start + $ cd .. + +At this point Mailman will not send nor receive any real emails. But that's +fine as long as you only want to work on the components related to the REST +client or the web ui. + + +mailman.client (the Python bindings for Mailman's REST API) +=========================================================== + +Download from Launchpad:: + + $ bzr branch lp:mailman.client + +Install in development mode to be able to change the code without working +directly on the PYTHONPATH. +:: + + $ cd mailman.client + $ sudo python setup.py develop + $ cd .. + + +Django >= 1.3 +============= + +Postorius is a pluggable Django application. Therefore you need to have +Django (at least version 1.3) installed. +:: + + $ wget http://www.djangoproject.com/download/1.3.1/tarball/ -O Django-1.3.1.tar.gz + $ tar xzf Django-1.3.1.tar.gz + $ cd Django-1.3.1 + $ sudo python setup.py install + $ cd .. + + +Postorius +========= + +:: + + $ bzr branch lp:postorius + $ cd postorius + $ sudo python setup.py develop + + +Start the development server +============================ + +:: + + $ cd dev_setup + $ python manage.py syncdb + $ python manage.py runserver + +The last command will start the dev server on http://localhost:8000. + + +A note for MacOS X users (and possibly others running python 2.7) +================================================================= + +*Note: These paragraphs are struck-through on the Mailman wiki.* + +On an OS X 10.7 (Lion) system, some of these steps needed to be modified to +use python2.6 instead of python. (In particular, bzr is known to behave badly +when used python2.7 on OS X 10.7 at the moment -- hopefully this will be fixed +and no longer an issue soon.) + +You will need to install the latest version of XCode on MacOS 10.7, which is +available for free from the App Store. If you had a previous version of XCode +installed when you upgraded to 10.7, it will no longer work and will not have +automatically been upgraded, so be prepared to install again. Once you have +it installed from the App Store, you will still need to go run the installer +from ``/Applications`` to complete the installation. diff --git a/src/mailman/email/validate.py b/src/mailman/email/validate.py index d0df7592f..1861a8121 100644 --- a/src/mailman/email/validate.py +++ b/src/mailman/email/validate.py @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see <http://www.gnu.org/licenses/>. -"""Module stuff.""" +"""Email address validation.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ diff --git a/src/mailman/pipeline/__init__.py b/src/mailman/handlers/__init__.py index e69de29bb..e69de29bb 100644 --- a/src/mailman/pipeline/__init__.py +++ b/src/mailman/handlers/__init__.py diff --git a/src/mailman/pipeline/acknowledge.py b/src/mailman/handlers/acknowledge.py index 0e0916337..0e0916337 100644 --- a/src/mailman/pipeline/acknowledge.py +++ b/src/mailman/handlers/acknowledge.py diff --git a/src/mailman/pipeline/after_delivery.py b/src/mailman/handlers/after_delivery.py index 46007092b..a964804b5 100644 --- a/src/mailman/pipeline/after_delivery.py +++ b/src/mailman/handlers/after_delivery.py @@ -25,12 +25,11 @@ __all__ = [ ] -import datetime - from zope.interface import implements from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler +from mailman.utilities.datetime import now @@ -44,5 +43,5 @@ class AfterDelivery: def process(self, mlist, msg, msgdata): """See `IHander`.""" - mlist.last_post_time = datetime.datetime.now() + mlist.last_post_time = now() mlist.post_id += 1 diff --git a/src/mailman/pipeline/avoid_duplicates.py b/src/mailman/handlers/avoid_duplicates.py index ffbc80c85..ffbc80c85 100644 --- a/src/mailman/pipeline/avoid_duplicates.py +++ b/src/mailman/handlers/avoid_duplicates.py diff --git a/src/mailman/pipeline/cleanse.py b/src/mailman/handlers/cleanse.py index 90f2a892a..605b843d0 100644 --- a/src/mailman/pipeline/cleanse.py +++ b/src/mailman/handlers/cleanse.py @@ -31,8 +31,8 @@ from email.utils import formataddr from zope.interface import implements from mailman.core.i18n import _ +from mailman.handlers.cook_headers import uheader from mailman.interfaces.handler import IHandler -from mailman.pipeline.cook_headers import uheader log = logging.getLogger('mailman.post') diff --git a/src/mailman/pipeline/cleanse_dkim.py b/src/mailman/handlers/cleanse_dkim.py index d2cd32636..d2cd32636 100644 --- a/src/mailman/pipeline/cleanse_dkim.py +++ b/src/mailman/handlers/cleanse_dkim.py diff --git a/src/mailman/pipeline/cook_headers.py b/src/mailman/handlers/cook_headers.py index 2d117429c..5d1e416a6 100644 --- a/src/mailman/pipeline/cook_headers.py +++ b/src/mailman/handlers/cook_headers.py @@ -190,7 +190,7 @@ def prefix_subject(mlist, msg, msgdata): ws = '\t' if len(lines) > 1 and lines[1] and lines[1][0] in ' \t': ws = lines[1][0] - msgdata['origsubj'] = subject + msgdata['original_subject'] = subject # The subject may be multilingual but we take the first charset as major # one and try to decode. If it is decodable, returned subject is in one # line and cset is properly set. If fail, subject is mime-encoded and diff --git a/src/mailman/pipeline/decorate.py b/src/mailman/handlers/decorate.py index d6d156048..d6d156048 100644 --- a/src/mailman/pipeline/decorate.py +++ b/src/mailman/handlers/decorate.py diff --git a/src/mailman/pipeline/docs/ack-headers.rst b/src/mailman/handlers/docs/ack-headers.rst index dba2169e2..e700e2fd1 100644 --- a/src/mailman/pipeline/docs/ack-headers.rst +++ b/src/mailman/handlers/docs/ack-headers.rst @@ -21,7 +21,7 @@ added. ... A message of great import. ... """) - >>> from mailman.pipeline.cook_headers import process + >>> from mailman.handlers.cook_headers import process >>> process(mlist, msg, dict(noack=True)) >>> print msg.as_string() From: aperson@example.com diff --git a/src/mailman/pipeline/docs/acknowledge.rst b/src/mailman/handlers/docs/acknowledge.rst index 479aa4ea6..479aa4ea6 100644 --- a/src/mailman/pipeline/docs/acknowledge.rst +++ b/src/mailman/handlers/docs/acknowledge.rst diff --git a/src/mailman/pipeline/docs/after-delivery.rst b/src/mailman/handlers/docs/after-delivery.rst index c3e393cf2..b65b0e77b 100644 --- a/src/mailman/pipeline/docs/after-delivery.rst +++ b/src/mailman/handlers/docs/after-delivery.rst @@ -6,9 +6,10 @@ After a message is delivered, or more correctly, after it has been processed by the rest of the handlers in the incoming queue pipeline, a couple of bookkeeping pieces of information are updated. - >>> import datetime + >>> from datetime import timedelta + >>> from mailman.utilities.datetime import now >>> mlist = create_list('_xtest@example.com') - >>> post_time = datetime.datetime.now() - datetime.timedelta(minutes=10) + >>> post_time = now() - timedelta(minutes=10) >>> mlist.last_post_time = post_time >>> mlist.post_id = 10 diff --git a/src/mailman/pipeline/docs/archives.rst b/src/mailman/handlers/docs/archives.rst index 323d121e8..323d121e8 100644 --- a/src/mailman/pipeline/docs/archives.rst +++ b/src/mailman/handlers/docs/archives.rst diff --git a/src/mailman/pipeline/docs/avoid-duplicates.rst b/src/mailman/handlers/docs/avoid-duplicates.rst index 1e46793c2..1e46793c2 100644 --- a/src/mailman/pipeline/docs/avoid-duplicates.rst +++ b/src/mailman/handlers/docs/avoid-duplicates.rst diff --git a/src/mailman/pipeline/docs/cleanse.rst b/src/mailman/handlers/docs/cleanse.rst index 61dfa8f52..61dfa8f52 100644 --- a/src/mailman/pipeline/docs/cleanse.rst +++ b/src/mailman/handlers/docs/cleanse.rst diff --git a/src/mailman/pipeline/docs/cook-headers.rst b/src/mailman/handlers/docs/cook-headers.rst index e0313f53a..948628d54 100644 --- a/src/mailman/pipeline/docs/cook-headers.rst +++ b/src/mailman/handlers/docs/cook-headers.rst @@ -26,7 +26,7 @@ will place the sender in the message metadata for safe keeping. ... """) >>> msgdata = {} - >>> from mailman.pipeline.cook_headers import process + >>> from mailman.handlers.cook_headers import process >>> process(mlist, msg, msgdata) >>> print msgdata['original_sender'] aperson@example.com diff --git a/src/mailman/pipeline/docs/decorate.rst b/src/mailman/handlers/docs/decorate.rst index 6fa8212ac..eae8ea904 100644 --- a/src/mailman/pipeline/docs/decorate.rst +++ b/src/mailman/handlers/docs/decorate.rst @@ -22,7 +22,7 @@ Digest messages get decorated during the digest creation phase so no extra decorations are added for digest messages. :: - >>> from mailman.pipeline.decorate import process + >>> from mailman.handlers.decorate import process >>> process(mlist, msg, dict(isdigest=True)) >>> print msg.as_string() From: aperson@example.org diff --git a/src/mailman/pipeline/docs/digests.rst b/src/mailman/handlers/docs/digests.rst index d4d563180..d4d563180 100644 --- a/src/mailman/pipeline/docs/digests.rst +++ b/src/mailman/handlers/docs/digests.rst diff --git a/src/mailman/pipeline/docs/file-recips.rst b/src/mailman/handlers/docs/file-recips.rst index 7d157ccc5..7d157ccc5 100644 --- a/src/mailman/pipeline/docs/file-recips.rst +++ b/src/mailman/handlers/docs/file-recips.rst diff --git a/src/mailman/pipeline/docs/filtering.rst b/src/mailman/handlers/docs/filtering.rst index fd0b33d3b..fd0b33d3b 100644 --- a/src/mailman/pipeline/docs/filtering.rst +++ b/src/mailman/handlers/docs/filtering.rst diff --git a/src/mailman/pipeline/docs/calc-recips.rst b/src/mailman/handlers/docs/member-recips.rst index 6dca85816..1439e978f 100644 --- a/src/mailman/pipeline/docs/calc-recips.rst +++ b/src/mailman/handlers/docs/member-recips.rst @@ -6,10 +6,10 @@ Every message that makes it through to the list membership gets sent to a set of recipient addresses. These addresses are calculated by one of the handler modules and depends on a host of factors. - >>> mlist = create_list('_xtest@example.com') + >>> mlist = create_list('test@example.com') -Recipients are calculate from the list members, so add a bunch of members to -start out with. First, create a bunch of addresses... +Recipients are calculate from the list membership, so first some people +subscribe to the mailing list... :: >>> from mailman.interfaces.usermanager import IUserManager @@ -35,42 +35,25 @@ start out with. First, create a bunch of addresses... ...then make some of the members digest members. - >>> from mailman.core.constants import DeliveryMode + >>> from mailman.interfaces.member import DeliveryMode >>> member_d.preferences.delivery_mode = DeliveryMode.plaintext_digests >>> member_e.preferences.delivery_mode = DeliveryMode.mime_digests >>> member_f.preferences.delivery_mode = DeliveryMode.summary_digests -Short-circuiting -================ +Regular delivery recipients +=========================== -Sometimes, the list of recipients already exists in the message metadata. -This can happen for example, when a message was previously delivered to some -but not all of the recipients. -:: +Regular delivery recipients are those people who get messages from the list as +soon as they are posted. In other words, these folks are not digest members. >>> msg = message_from_string("""\ ... From: Xavier Person <xperson@example.com> ... ... Something of great import. ... """) - >>> recipients = set(('qperson@example.com', 'zperson@example.com')) - >>> msgdata = dict(recipients=recipients) - - >>> handler = config.handlers['calculate-recipients'] - >>> handler.process(mlist, msg, msgdata) - >>> dump_list(msgdata['recipients']) - qperson@example.com - zperson@example.com - - -Regular delivery recipients -=========================== - -Regular delivery recipients are those people who get messages from the list as -soon as they are posted. In other words, these folks are not digest members. - >>> msgdata = {} + >>> handler = config.handlers['member-recipients'] >>> handler.process(mlist, msg, msgdata) >>> dump_list(msgdata['recipients']) aperson@example.com diff --git a/src/mailman/pipeline/docs/nntp.rst b/src/mailman/handlers/docs/nntp.rst index 874712397..c298fcb14 100644 --- a/src/mailman/pipeline/docs/nntp.rst +++ b/src/mailman/handlers/docs/nntp.rst @@ -3,15 +3,14 @@ NNTP Gateway ============ Mailman has an NNTP gateway, whereby messages posted to the mailing list can -be forwarded onto an NNTP newsgroup. Typically this means Usenet, but since -NNTP is to Usenet as IP is to the web, it's more general than that. +be forwarded onto an NNTP newsgroup. - >>> mlist = create_list('_xtest@example.com') + >>> mlist = create_list('test@example.com') Gatewaying from the mailing list to the newsgroup happens through a separate ``nntp`` queue and happen immediately when the message is posted through to the list. Note that gatewaying from the newsgroup to the list happens via a -cronjob (currently not shown). +separate process. There are several situations which prevent a message from being gatewayed to the newsgroup. The feature could be disabled, as is the default. @@ -26,43 +25,43 @@ the newsgroup. The feature could be disabled, as is the default. >>> handler = config.handlers['to-usenet'] >>> handler.process(mlist, msg, {}) - - >>> switchboard = config.switchboards['news'] - >>> switchboard.files + >>> from mailman.testing.helpers import get_queue_messages + >>> get_queue_messages('nntp') [] Even if enabled, messages that came from the newsgroup are never gated back to the newsgroup. >>> mlist.gateway_to_news = True - >>> handler.process(mlist, msg, {'fromusenet': True}) - >>> switchboard.files + >>> handler.process(mlist, msg, dict(fromusenet=True)) + >>> get_queue_messages('nntp') [] Neither are digests ever gated to the newsgroup. - >>> handler.process(mlist, msg, {'isdigest': True}) - >>> switchboard.files + >>> handler.process(mlist, msg, dict(isdigest=True)) + >>> get_queue_messages('nntp') [] However, other posted messages get gated to the newsgroup via the nntp queue. The list owner can set the linked newsgroup and the nntp host that its messages are gated to. +:: >>> mlist.linked_newsgroup = 'comp.lang.thing' >>> mlist.nntp_host = 'news.example.com' >>> handler.process(mlist, msg, {}) - >>> len(switchboard.files) + >>> messages = get_queue_messages('nntp') + >>> len(messages) 1 - >>> filebase = switchboard.files[0] - >>> msg, msgdata = switchboard.dequeue(filebase) - >>> switchboard.finish(filebase) - >>> print msg.as_string() + + >>> print messages[0].msg.as_string() Subject: An important message <BLANKLINE> Something of great import. <BLANKLINE> - >>> dump_msgdata(msgdata) + + >>> dump_msgdata(messages[0].msgdata) _parsemsg: False - listname : _xtest@example.com + listname : test@example.com version : 3 diff --git a/src/mailman/handlers/docs/owner-recips.rst b/src/mailman/handlers/docs/owner-recips.rst new file mode 100644 index 000000000..e62551ba6 --- /dev/null +++ b/src/mailman/handlers/docs/owner-recips.rst @@ -0,0 +1,63 @@ +===================== +List owner recipients +===================== + +When a message is posted to a mailing list's `-owners` address, all of the +list's administrators will receive a copy. The administrators are defined as +the set of owners and moderators. + + >>> mlist_1 = create_list('alpha@example.com') + +Anne is the owner of the list and Bart is a moderator of the list. + + >>> from mailman.interfaces.usermanager import IUserManager + >>> from zope.component import getUtility + >>> user_manager = getUtility(IUserManager) + >>> anne_addr = user_manager.create_address('anne@example.com') + >>> bart_addr = user_manager.create_address('bart@example.com') + >>> from mailman.interfaces.member import MemberRole + >>> anne = mlist_1.subscribe(anne_addr, MemberRole.owner) + >>> bart = mlist_1.subscribe(bart_addr, MemberRole.moderator) + +The recipients list for the `-owners` address includes both Anne and Bart. + + >>> msg = message_from_string("""\ + ... From: Xavier Person <xperson@example.com> + ... To: alpha@example.com + ... + ... """) + >>> msgdata = {} + >>> handler = config.handlers['owner-recipients'] + >>> handler.process(mlist_1, msg, msgdata) + >>> dump_list(msgdata['recipients']) + anne@example.com + bart@example.com + +Anne disables her owner delivery, so she will not receive `-owner` emails. + + >>> from mailman.interfaces.member import DeliveryStatus + >>> anne.preferences.delivery_status = DeliveryStatus.by_user + >>> msgdata = {} + >>> handler.process(mlist_1, msg, msgdata) + >>> dump_list(msgdata['recipients']) + bart@example.com + +If Bart also disables his owner delivery, then no one could contact the list's +owners. Since this is unacceptable, the site owner is used as a fallback. + + >>> bart.preferences.delivery_status = DeliveryStatus.by_user + >>> msgdata = {} + >>> handler.process(mlist_1, msg, msgdata) + >>> dump_list(msgdata['recipients']) + noreply@example.com + +For mailing lists which have no owners at all, the site owner is also used as +a fallback. + + >>> mlist_2 = create_list('beta@example.com') + >>> mlist_2.administrators.member_count + 0 + >>> msgdata = {} + >>> handler.process(mlist_2, msg, msgdata) + >>> dump_list(msgdata['recipients']) + noreply@example.com diff --git a/src/mailman/pipeline/docs/reply-to.rst b/src/mailman/handlers/docs/reply-to.rst index e08fea81d..d421e2dc5 100644 --- a/src/mailman/pipeline/docs/reply-to.rst +++ b/src/mailman/handlers/docs/reply-to.rst @@ -46,7 +46,7 @@ original message, the list's posting address simply gets inserted. ... ... """) - >>> from mailman.pipeline.cook_headers import process + >>> from mailman.handlers.cook_headers import process >>> process(mlist, msg, {}) >>> len(msg.get_all('reply-to')) 1 diff --git a/src/mailman/pipeline/docs/replybot.rst b/src/mailman/handlers/docs/replybot.rst index 7cdd7c928..2793e4f75 100644 --- a/src/mailman/pipeline/docs/replybot.rst +++ b/src/mailman/handlers/docs/replybot.rst @@ -21,11 +21,11 @@ automatic response grace period which specifies how much time must pass before a second response will be sent, with 0 meaning "there is no grace period". :: - >>> import datetime + >>> from datetime import timedelta >>> from mailman.interfaces.autorespond import ResponseAction >>> mlist.autorespond_owner = ResponseAction.respond_and_continue - >>> mlist.autoresponse_grace_period = datetime.timedelta() + >>> mlist.autoresponse_grace_period = timedelta() >>> mlist.autoresponse_owner_text = 'owner autoresponse text' >>> msg = message_from_string("""\ @@ -242,7 +242,7 @@ Automatic responses have a grace period, during which no additional responses will be sent. This is so as not to bombard the sender with responses. The grace period is measured in days. - >>> mlist.autoresponse_grace_period = datetime.timedelta(days=10) + >>> mlist.autoresponse_grace_period = timedelta(days=10) When a response is sent to a person via any of the owner, request, or postings addresses, the response date is recorded. The grace period is usually diff --git a/src/mailman/pipeline/docs/rfc-2369.rst b/src/mailman/handlers/docs/rfc-2369.rst index 1b89f2354..0461f27ba 100644 --- a/src/mailman/pipeline/docs/rfc-2369.rst +++ b/src/mailman/handlers/docs/rfc-2369.rst @@ -28,7 +28,7 @@ headers generally start with the `List-` prefix. The `rfc-2369` handler adds the `List-` headers. `List-Id` is always added. - >>> from mailman.pipeline.rfc_2369 import process + >>> from mailman.handlers.rfc_2369 import process >>> msg = message_from_string("""\ ... From: aperson@example.com ... diff --git a/src/mailman/pipeline/docs/subject-munging.rst b/src/mailman/handlers/docs/subject-munging.rst index e7a6553ce..f9e3b9abb 100644 --- a/src/mailman/pipeline/docs/subject-munging.rst +++ b/src/mailman/handlers/docs/subject-munging.rst @@ -8,7 +8,7 @@ transformations. Some headers get added, others get changed. Some of these changes depend on mailing list settings and others depend on how the message is getting sent through the system. We'll take things one-by-one. - >>> mlist = create_list('_xtest@example.com') + >>> mlist = create_list('test@example.com') Inserting a prefix @@ -29,14 +29,12 @@ subject munging, a mailing list must have a preferred language. ... """) >>> msgdata = {} - >>> from mailman.pipeline.cook_headers import process + >>> from mailman.handlers.cook_headers import process >>> process(mlist, msg, msgdata) -The original subject header is stored in the message metadata. We must print -the new ``Subject`` header because it gets converted from a string to an -``email.header.Header`` instance which has an unhelpful ``repr``. +The original subject header is stored in the message metadata. - >>> msgdata['origsubj'] + >>> msgdata['original_subject'] u'' >>> print msg['subject'] [XTest] (no subject) @@ -52,7 +50,7 @@ at the beginning of the header's value. ... """) >>> msgdata = {} >>> process(mlist, msg, msgdata) - >>> print msgdata['origsubj'] + >>> print msgdata['original_subject'] Something important >>> print msg['subject'] [XTest] Something important diff --git a/src/mailman/pipeline/docs/tagger.rst b/src/mailman/handlers/docs/tagger.rst index 80e682119..b64b05c54 100644 --- a/src/mailman/pipeline/docs/tagger.rst +++ b/src/mailman/handlers/docs/tagger.rst @@ -26,7 +26,7 @@ are defined. ... """) >>> msgdata = {} - >>> from mailman.pipeline.tagger import process + >>> from mailman.handlers.tagger import process >>> process(mlist, msg, msgdata) >>> print msg.as_string() Subject: foobar diff --git a/src/mailman/pipeline/docs/to-outgoing.rst b/src/mailman/handlers/docs/to-outgoing.rst index 816aa4ca6..816aa4ca6 100644 --- a/src/mailman/pipeline/docs/to-outgoing.rst +++ b/src/mailman/handlers/docs/to-outgoing.rst diff --git a/src/mailman/pipeline/file_recipients.py b/src/mailman/handlers/file_recipients.py index d087ff2bb..d087ff2bb 100644 --- a/src/mailman/pipeline/file_recipients.py +++ b/src/mailman/handlers/file_recipients.py diff --git a/src/mailman/pipeline/calculate_recipients.py b/src/mailman/handlers/member_recipients.py index 15be07ecd..956ea6adc 100644 --- a/src/mailman/pipeline/calculate_recipients.py +++ b/src/mailman/handlers/member_recipients.py @@ -23,13 +23,14 @@ on the `recipients' attribute of the message. This attribute is used by the SendmailDeliver and BulkDeliver modules. """ -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ - 'CalculateRecipients', + 'MemberRecipients', ] + from zope.interface import implements from mailman.config import config @@ -41,12 +42,12 @@ from mailman.utilities.string import wrap -class CalculateRecipients: +class MemberRecipients: """Calculate the regular (i.e. non-digest) recipients of the message.""" implements(IHandler) - name = 'calculate-recipients' + name = 'member-recipients' description = _('Calculate the regular recipients of the message.') def process(self, mlist, msg, msgdata): @@ -82,10 +83,9 @@ class CalculateRecipients: # Bad Urgent: password, so reject it instead of passing it on. # I think it's better that the sender know they screwed up # than to deliver it normally. - listname = mlist.display_name text = _("""\ -Your urgent message to the $listname mailing list was not authorized for -delivery. The original message as received by Mailman is attached. +Your urgent message to the $mlist.display_name mailing list was not authorized +for delivery. The original message as received by Mailman is attached. """) raise errors.RejectMessage(wrap(text)) # Calculate the regular recipients of the message diff --git a/src/mailman/pipeline/mime_delete.py b/src/mailman/handlers/mime_delete.py index c9c1eb408..c9c1eb408 100644 --- a/src/mailman/pipeline/mime_delete.py +++ b/src/mailman/handlers/mime_delete.py diff --git a/src/mailman/handlers/owner_recipients.py b/src/mailman/handlers/owner_recipients.py new file mode 100644 index 000000000..e431d00cf --- /dev/null +++ b/src/mailman/handlers/owner_recipients.py @@ -0,0 +1,67 @@ +# Copyright (C) 2001-2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Calculate the list owner recipients (includes moderators).""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'OwnerRecipients', + ] + + +from zope.interface import implements + +from mailman.config import config +from mailman.core.i18n import _ +from mailman.interfaces.handler import IHandler +from mailman.interfaces.member import DeliveryStatus + + + +class OwnerRecipients: + """Calculate the owner (and moderator) recipients for -owner postings.""" + + implements(IHandler) + + name = 'owner-recipients' + description = _('Calculate the owner and moderator recipients.') + + def process(self, mlist, msg, msgdata): + """See `IHandler`.""" + # Short circuit if we've already calculated the recipients list, + # regardless of whether the list is empty or not. + if 'recipients' in msgdata: + return + # -owner messages go to both the owners and moderators, which is most + # conveniently accessed via the administrators roster. + recipients = set(admin.address.email + for admin in mlist.administrators.members + if admin.delivery_status == DeliveryStatus.enabled) + # To prevent -owner messages from going into a black hole, if there + # are no administrators available, the message goes to the site owner. + if len(recipients) == 0: + msgdata['recipients'] = set((config.mailman.site_owner,)) + else: + msgdata['recipients'] = recipients + # Don't decorate these messages with the header/footers. Eventually + # we should support unique decorations for owner emails. + msgdata['nodecorate'] = True + # We should probably always VERP deliveries to the owners. We + # *really* want to know if they are bouncing. + msgdata['verp'] = True diff --git a/src/mailman/pipeline/replybot.py b/src/mailman/handlers/replybot.py index 83aa40214..83aa40214 100644 --- a/src/mailman/pipeline/replybot.py +++ b/src/mailman/handlers/replybot.py diff --git a/src/mailman/pipeline/rfc_2369.py b/src/mailman/handlers/rfc_2369.py index 26bfe094c..ece4e83cb 100644 --- a/src/mailman/pipeline/rfc_2369.py +++ b/src/mailman/handlers/rfc_2369.py @@ -30,8 +30,9 @@ from zope.interface import implements from mailman.config import config from mailman.core.i18n import _ +from mailman.handlers.cook_headers import uheader from mailman.interfaces.handler import IHandler -from mailman.pipeline.cook_headers import uheader + CONTINUATION = ',\n\t' diff --git a/src/mailman/pipeline/tagger.py b/src/mailman/handlers/tagger.py index 49e004a12..49e004a12 100644 --- a/src/mailman/pipeline/tagger.py +++ b/src/mailman/handlers/tagger.py diff --git a/src/mailman/pipeline/tests/__init__.py b/src/mailman/handlers/tests/__init__.py index e69de29bb..e69de29bb 100644 --- a/src/mailman/pipeline/tests/__init__.py +++ b/src/mailman/handlers/tests/__init__.py diff --git a/src/mailman/pipeline/tests/test_mimedel.py b/src/mailman/handlers/tests/test_mimedel.py index 566c1a40c..6ca34b17b 100644 --- a/src/mailman/pipeline/tests/test_mimedel.py +++ b/src/mailman/handlers/tests/test_mimedel.py @@ -32,10 +32,10 @@ from zope.component import getUtility from mailman.app.lifecycle import create_list from mailman.config import config from mailman.core import errors +from mailman.handlers import mime_delete from mailman.interfaces.action import FilterAction from mailman.interfaces.member import MemberRole from mailman.interfaces.usermanager import IUserManager -from mailman.pipeline import mime_delete from mailman.testing.helpers import ( LogFileMark, get_queue_messages, diff --git a/src/mailman/handlers/tests/test_recipients.py b/src/mailman/handlers/tests/test_recipients.py new file mode 100644 index 000000000..29cd5b64a --- /dev/null +++ b/src/mailman/handlers/tests/test_recipients.py @@ -0,0 +1,200 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Testing various recipients stuff.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestMemberRecipients', + 'TestOwnerRecipients', + ] + + +import unittest + +from zope.component import getUtility +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.interfaces.member import DeliveryMode, DeliveryStatus, MemberRole +from mailman.interfaces.usermanager import IUserManager +from mailman.testing.helpers import specialized_message_from_string as mfs +from mailman.testing.layers import ConfigLayer + + + +class TestMemberRecipients(unittest.TestCase): + """Test regular member recipient calculation.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._manager = getUtility(IUserManager) + anne = self._manager.create_address('anne@example.com') + bart = self._manager.create_address('bart@example.com') + cris = self._manager.create_address('cris@example.com') + dave = self._manager.create_address('dave@example.com') + self._anne = self._mlist.subscribe(anne, MemberRole.member) + self._bart = self._mlist.subscribe(bart, MemberRole.member) + self._cris = self._mlist.subscribe(cris, MemberRole.member) + self._dave = self._mlist.subscribe(dave, MemberRole.member) + self._process = config.handlers['member-recipients'].process + self._msg = mfs("""\ +From: Elle Person <elle@example.com> +To: test@example.com + +""") + + def test_shortcircuit(self): + # When there are already recipients in the message metadata, those are + # used instead of calculating them from the list membership. + recipients = set(('zperson@example.com', 'yperson@example.com')) + msgdata = dict(recipients=recipients) + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], recipients) + + def test_calculate_recipients(self): + # The normal path just adds the list's regular members. + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('anne@example.com', + 'bart@example.com', + 'cris@example.com', + 'dave@example.com'))) + + def test_digest_members_not_included(self): + # Digest members are not included in the recipients calculated by this + # handler. + self._cris.preferences.delivery_mode = DeliveryMode.mime_digests + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('anne@example.com', + 'bart@example.com', + 'dave@example.com'))) + + + +class TestOwnerRecipients(unittest.TestCase): + """Test owner recipient calculation.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._manager = getUtility(IUserManager) + anne = self._manager.create_address('anne@example.com') + bart = self._manager.create_address('bart@example.com') + cris = self._manager.create_address('cris@example.com') + dave = self._manager.create_address('dave@example.com') + # Make Cris and Dave owners of the mailing list. + self._anne = self._mlist.subscribe(anne, MemberRole.member) + self._bart = self._mlist.subscribe(bart, MemberRole.member) + self._cris = self._mlist.subscribe(cris, MemberRole.owner) + self._dave = self._mlist.subscribe(dave, MemberRole.owner) + self._process = config.handlers['owner-recipients'].process + self._msg = mfs("""\ +From: Elle Person <elle@example.com> +To: test-owner@example.com + +""") + + def test_shortcircuit(self): + # When there are already recipients in the message metadata, those are + # used instead of calculating them from the owner membership. + recipients = set(('zperson@example.com', 'yperson@example.com')) + msgdata = dict(recipients=recipients) + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], recipients) + + def test_calculate_recipients(self): + # The normal path just adds the list's owners. + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('cris@example.com', + 'dave@example.com'))) + + def test_with_moderators(self): + # Moderators are included in the owner recipient list. + elle = self._manager.create_address('elle@example.com') + fred = self._manager.create_address('fred@example.com') + gwen = self._manager.create_address('gwen@example.com') + self._mlist.subscribe(elle, MemberRole.moderator) + self._mlist.subscribe(fred, MemberRole.moderator) + self._mlist.subscribe(gwen, MemberRole.owner) + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('cris@example.com', + 'dave@example.com', + 'elle@example.com', + 'fred@example.com', + 'gwen@example.com'))) + + def test_dont_decorate(self): + # Messages to the administrators don't get decorated. + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertTrue(msgdata['nodecorate']) + + def test_omit_disabled_owners(self): + # Owner memberships can be disabled, and these folks will not get the + # messages. + self._dave.preferences.delivery_status = DeliveryStatus.by_user + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('cris@example.com',))) + + def test_include_membership_disabled_owner_enabled(self): + # If an address is subscribed to a mailing list as both an owner and a + # member, and their membership is disabled but their ownership + # subscription is not, they still get owner email. + dave = self._manager.get_address('dave@example.com') + member = self._mlist.subscribe(dave, MemberRole.member) + member.preferences.delivery_status = DeliveryStatus.by_user + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('cris@example.com', + 'dave@example.com'))) + # Dave disables his owner membership but re-enables his list + # membership. He will not get the owner emails now. + member.preferences.delivery_status = DeliveryStatus.enabled + self._dave.preferences.delivery_status = DeliveryStatus.by_user + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('cris@example.com',))) + + def test_all_owners_disabled(self): + # If all the owners are disabled, then the site owner gets the + # message. This prevents a list's -owner address from going into a + # black hole. + self._cris.preferences.delivery_status = DeliveryStatus.by_user + self._dave.preferences.delivery_status = DeliveryStatus.by_user + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('noreply@example.com',))) + + def test_no_owners(self): + # If a list has no owners or moderators, then the site owner gets the + # message. This prevents a list's -owner address from going into a + # black hole. + self._cris.unsubscribe() + self._dave.unsubscribe() + self.assertEqual(self._mlist.administrators.member_count, 0) + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('noreply@example.com',))) diff --git a/src/mailman/pipeline/to_archive.py b/src/mailman/handlers/to_archive.py index fd5259a14..fd5259a14 100644 --- a/src/mailman/pipeline/to_archive.py +++ b/src/mailman/handlers/to_archive.py diff --git a/src/mailman/pipeline/to_digest.py b/src/mailman/handlers/to_digest.py index 698f16e1e..71511f136 100644 --- a/src/mailman/pipeline/to_digest.py +++ b/src/mailman/handlers/to_digest.py @@ -26,7 +26,6 @@ __all__ = [ import os -import datetime from zope.interface import implements @@ -35,6 +34,7 @@ from mailman.core.i18n import _ from mailman.email.message import Message from mailman.interfaces.digests import DigestFrequency from mailman.interfaces.handler import IHandler +from mailman.utilities.datetime import now as right_now from mailman.utilities.mailbox import Mailbox @@ -85,7 +85,7 @@ class ToDigest: def bump_digest_number_and_volume(mlist): """Bump the digest number and volume.""" - now = datetime.datetime.now() + now = right_now() if mlist.digest_last_sent_at is None: # There has been no previous digest. bump = False diff --git a/src/mailman/pipeline/to_outgoing.py b/src/mailman/handlers/to_outgoing.py index 971f87757..971f87757 100644 --- a/src/mailman/pipeline/to_outgoing.py +++ b/src/mailman/handlers/to_outgoing.py diff --git a/src/mailman/pipeline/to_usenet.py b/src/mailman/handlers/to_usenet.py index 26a383c64..021f8f9e5 100644 --- a/src/mailman/pipeline/to_usenet.py +++ b/src/mailman/handlers/to_usenet.py @@ -17,7 +17,7 @@ """Move the message to the mail->news queue.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -50,11 +50,12 @@ class ToUsenet: def process(self, mlist, msg, msgdata): """See `IHandler`.""" # Short circuits. - if not mlist.gateway_to_news or \ - msgdata.get('isdigest') or \ - msgdata.get('fromusenet'): + if (not mlist.gateway_to_news or + msgdata.get('isdigest') or + msgdata.get('fromusenet')): + # Short-circuit. return - # sanity checks + # Sanity checks. error = [] if not mlist.linked_newsgroup: error.append('no newsgroup') @@ -65,5 +66,5 @@ class ToUsenet: COMMASPACE.join(error)) return # Put the message in the news runner's queue. - config.switchboards['news'].enqueue( + config.switchboards['nntp'].enqueue( msg, msgdata, listname=mlist.fqdn_listname) diff --git a/src/mailman/interfaces/address.py b/src/mailman/interfaces/address.py index 6c371e1c0..cf2c50bf4 100644 --- a/src/mailman/interfaces/address.py +++ b/src/mailman/interfaces/address.py @@ -17,13 +17,14 @@ """Interface for email address related information.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ 'AddressAlreadyLinkedError', 'AddressError', 'AddressNotLinkedError', + 'EmailError', 'ExistingAddressError', 'IAddress', 'IEmailValidator', @@ -37,15 +38,30 @@ from mailman.interfaces.errors import MailmanError +class EmailError(MailmanError): + """A generic text email address-related error occurred.""" + + def __init__(self, email): + super(EmailError, self).__init__() + self.email = email + + def __str__(self): + # This is a workaround for Python 2.6 support. When self.email + # contains non-ascii characters, this will cause unprintable output in + # doctests. Python 2.7 can handle it but we haven't dropped support + # for 2.6 yet. + return self.email.encode('us-ascii', 'backslashreplace') + + class AddressError(MailmanError): - """A general address-related error occurred.""" + """A generic IAddress-related error occurred.""" def __init__(self, address): super(AddressError, self).__init__() self.address = address def __str__(self): - return self.address + return str(self.address) class ExistingAddressError(AddressError): @@ -60,7 +76,7 @@ class AddressNotLinkedError(AddressError): """The address is not linked to the user.""" -class InvalidEmailAddressError(AddressError): +class InvalidEmailAddressError(EmailError): """Email address is invalid.""" diff --git a/src/mailman/interfaces/archiver.py b/src/mailman/interfaces/archiver.py index a06bbdede..f3edc7719 100644 --- a/src/mailman/interfaces/archiver.py +++ b/src/mailman/interfaces/archiver.py @@ -21,14 +21,23 @@ from __future__ import absolute_import, unicode_literals __metaclass__ = type __all__ = [ + 'ClobberDate', 'IArchiver', ] +from flufl.enum import Enum from zope.interface import Interface, Attribute +class ClobberDate(Enum): + never = 1 + maybe = 2 + always = 3 + + + class IArchiver(Interface): """An interface to the archiver.""" diff --git a/src/mailman/interfaces/mailinglist.py b/src/mailman/interfaces/mailinglist.py index d92bae464..bced070d3 100644 --- a/src/mailman/interfaces/mailinglist.py +++ b/src/mailman/interfaces/mailinglist.py @@ -382,7 +382,7 @@ class IMailingList(Interface): # Processing. posting_chain = Attribute( - """This mailing list's moderation chain. + """This mailing list's posting moderation chain. When messages are posted to a mailing list, it first goes through a moderation chain to determine whether the message will be accepted. @@ -397,6 +397,24 @@ class IMailingList(Interface): This attribute names a pipeline for postings, which must exist. """) + owner_chain = Attribute( + """This mailing list's owner moderation chain. + + When messages are posted to the owners of a mailing list, it first + goes through a moderation chain to determine whether the message will + be accepted. This attribute names a chain for postings, which must + exist. + """) + + owner_pipeline = Attribute( + """This mailing list's owner posting pipeline. + + Every mailing list has a processing pipeline that messages flow + through once they've been accepted for posting to the owners of a + mailing list. This attribute names a pipeline for postings, which + must exist. + """) + data_path = Attribute( """The file system path to list-specific data. diff --git a/src/mailman/interfaces/nntp.py b/src/mailman/interfaces/nntp.py index 099e14a07..d5d08d3f0 100644 --- a/src/mailman/interfaces/nntp.py +++ b/src/mailman/interfaces/nntp.py @@ -26,9 +26,9 @@ from flufl.enum import Enum class NewsModeration(Enum): - # The newsgroup is not moderated + # The newsgroup is not moderated. none = 0 # The newsgroup is moderated, but allows for an open posting policy. open_moderated = 1 - # The newsgroup is moderated + # The newsgroup is moderated. moderated = 2 diff --git a/src/mailman/interfaces/user.py b/src/mailman/interfaces/user.py index aba9bcbba..8d0cbfb54 100644 --- a/src/mailman/interfaces/user.py +++ b/src/mailman/interfaces/user.py @@ -17,7 +17,7 @@ """Interface describing the basics of a user.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -28,20 +28,13 @@ __all__ = [ from zope.interface import Interface, Attribute -from mailman.interfaces.errors import MailmanError +from mailman.interfaces.address import AddressError -class UnverifiedAddressError(MailmanError): +class UnverifiedAddressError(AddressError): """Unverified address cannot be used as a user's preferred address.""" - def __init__(self, address): - super(UnverifiedAddressError, self).__init__() - self.address = address - - def __str__(self): - return self.address - class IUser(Interface): diff --git a/src/mailman/model/docs/registration.rst b/src/mailman/model/docs/registration.rst index 112864f21..eecb3a8cd 100644 --- a/src/mailman/model/docs/registration.rst +++ b/src/mailman/model/docs/registration.rst @@ -275,12 +275,13 @@ different except that the new address will still need to be verified before it can be used. :: + >>> from mailman.utilities.datetime import now >>> dperson = user_manager.create_user( ... 'dperson@example.com', 'Dave Person') >>> dperson <User "Dave Person" (...) at ...> >>> address = user_manager.get_address('dperson@example.com') - >>> address.verified_on = datetime.now() + >>> address.verified_on = now() >>> from operator import attrgetter >>> dump_list(repr(address) for address in dperson.addresses) diff --git a/src/mailman/model/docs/requests.rst b/src/mailman/model/docs/requests.rst index 1e461e36c..a20823a91 100644 --- a/src/mailman/model/docs/requests.rst +++ b/src/mailman/model/docs/requests.rst @@ -230,8 +230,7 @@ We can also hold a message with some additional metadata. # collisions in the message storage. >>> del msg['message-id'] >>> msgdata = dict(sender='aperson@example.com', - ... approved=True, - ... received_time=123.45) + ... approved=True) >>> id_2 = moderator.hold_message(mlist, msg, msgdata, 'Feeling ornery') >>> requests.get_request(id_2) is not None True @@ -664,7 +663,7 @@ The admin message is sent to the moderators. Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Subject: A Test List subscription notification - From: changeme@example.com + From: noreply@example.com To: alist-owner@example.com Message-ID: ... Date: ... @@ -676,7 +675,7 @@ The admin message is sent to the moderators. >>> dump_msgdata(messages[0].msgdata) _parsemsg : False - envsender : changeme@example.com + envsender : noreply@example.com listname : alist@example.com nodecorate : True recipients : set([]) @@ -888,7 +887,7 @@ The goodbye message... Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Subject: A Test List unsubscription notification - From: changeme@example.com + From: noreply@example.com To: alist-owner@example.com Message-ID: ... Date: ... @@ -899,7 +898,7 @@ The goodbye message... >>> dump_msgdata(messages[1].msgdata) _parsemsg : False - envsender : changeme@example.com + envsender : noreply@example.com listname : alist@example.com nodecorate : True recipients : set([]) diff --git a/src/mailman/model/listmanager.py b/src/mailman/model/listmanager.py index cb56a36b6..0ea87a082 100644 --- a/src/mailman/model/listmanager.py +++ b/src/mailman/model/listmanager.py @@ -25,8 +25,6 @@ __all__ = [ ] -import datetime - from zope.event import notify from zope.interface import implements @@ -36,6 +34,7 @@ from mailman.interfaces.listmanager import ( IListManager, ListAlreadyExistsError, ListCreatedEvent, ListCreatingEvent, ListDeletedEvent, ListDeletingEvent) from mailman.model.mailinglist import MailingList +from mailman.utilities.datetime import now @@ -57,7 +56,7 @@ class ListManager: if mlist: raise ListAlreadyExistsError(fqdn_listname) mlist = MailingList(fqdn_listname) - mlist.created_at = datetime.datetime.now() + mlist.created_at = now() config.db.store.add(mlist) notify(ListCreatedEvent(mlist)) return mlist diff --git a/src/mailman/model/mailinglist.py b/src/mailman/model/mailinglist.py index 76f88caa7..e397d59d6 100644 --- a/src/mailman/model/mailinglist.py +++ b/src/mailman/model/mailinglist.py @@ -168,6 +168,8 @@ class MailingList(Model): nondigestable = Bool() nonmember_rejection_notice = Unicode() obscure_addresses = Bool() + owner_chain = Unicode() + owner_pipeline = Unicode() personalize = Enum(Personalization) post_id = Int() posting_chain = Unicode() diff --git a/src/mailman/model/pending.py b/src/mailman/model/pending.py index 72746295b..557361c6f 100644 --- a/src/mailman/model/pending.py +++ b/src/mailman/model/pending.py @@ -29,7 +29,6 @@ __all__ = [ import time import random import hashlib -import datetime from lazr.config import as_timedelta from storm.locals import DateTime, Int, RawStr, ReferenceSet, Unicode @@ -40,6 +39,7 @@ from mailman.config import config from mailman.database.model import Model from mailman.interfaces.pending import ( IPendable, IPended, IPendedKeyValue, IPendings) +from mailman.utilities.datetime import now from mailman.utilities.modules import call_name @@ -98,8 +98,8 @@ class Pendings: # does the hash calculation. The integral parts of the time values # are discarded because they're the most predictable bits. for attempts in range(3): - now = time.time() - x = random.random() + now % 1.0 + time.clock() % 1.0 + right_now = time.time() + x = random.random() + right_now % 1.0 + time.clock() % 1.0 # Use sha1 because it produces shorter strings. token = hashlib.sha1(repr(x)).hexdigest() # In practice, we'll never get a duplicate, but we'll be anal @@ -111,7 +111,7 @@ class Pendings: # Create the record, and then the individual key/value pairs. pending = Pended( token=token, - expiration_date=datetime.datetime.now() + lifetime) + expiration_date=now() + lifetime) for key, value in pendable.items(): if isinstance(key, str): key = unicode(key, 'utf-8') @@ -160,9 +160,9 @@ class Pendings: def evict(self): store = config.db.store - now = datetime.datetime.now() + right_now = now() for pending in store.find(Pended): - if pending.expiration_date < now: + if pending.expiration_date < right_now: # Find all PendedKeyValue entries that are associated with the # pending object's ID. q = store.find(PendedKeyValue, diff --git a/src/mailman/mta/postfix.py b/src/mailman/mta/postfix.py index 14f19635a..32bdb8268 100644 --- a/src/mailman/mta/postfix.py +++ b/src/mailman/mta/postfix.py @@ -27,7 +27,6 @@ __all__ = [ import os import logging -import datetime from flufl.lock import Lock from operator import attrgetter @@ -38,6 +37,7 @@ from mailman.config import config from mailman.interfaces.listmanager import IListManager from mailman.interfaces.mta import ( IMailTransportAgentAliases, IMailTransportAgentLifecycle) +from mailman.utilities.datetime import now log = logging.getLogger('mailman.error') @@ -124,7 +124,7 @@ class LMTP: # file. YOU SHOULD NOT MANUALLY EDIT THIS FILE unless you know what you're # doing, and can keep the two files properly in sync. If you screw it up, # you're on your own. -""".format(datetime.datetime.now().replace(microsecond=0)) +""".format(now().replace(microsecond=0)) sort_key = attrgetter('list_name') for domain in sorted(by_domain): print >> fp, """\ diff --git a/src/mailman/pipeline/owner_recipients.py b/src/mailman/pipeline/owner_recipients.py deleted file mode 100644 index 9e4bbf174..000000000 --- a/src/mailman/pipeline/owner_recipients.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (C) 2001-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Calculate the list owner recipients (includes moderators).""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'process', - ] - - - -def process(mlist, msg, msgdata): - """Add owner recipients.""" - # The recipients are the owner and the moderator - msgdata['recipients'] = mlist.owner + mlist.moderator - # Don't decorate these messages with the header/footers - msgdata['nodecorate'] = True - msgdata['personalize'] = False diff --git a/src/mailman/rules/approved.py b/src/mailman/rules/approved.py index b2a04bb05..927a96ee5 100644 --- a/src/mailman/rules/approved.py +++ b/src/mailman/rules/approved.py @@ -73,8 +73,8 @@ class Approved: break payload = part.get_payload(decode=True) if payload is not None: - cset = part.get_content_charset() or 'us-ascii' - payload = payload.decode(cset, 'replace') + charset = part.get_content_charset('us-ascii') + payload = payload.decode(charset, 'replace') line = '' lines = payload.splitlines(True) for lineno, line in enumerate(lines): diff --git a/src/mailman/rules/tests/test_approved.py b/src/mailman/rules/tests/test_approved.py index 79e3c84e2..8ffe68aa9 100644 --- a/src/mailman/rules/tests/test_approved.py +++ b/src/mailman/rules/tests/test_approved.py @@ -27,18 +27,9 @@ __all__ = [ import unittest -from zope.component import getUtility - from mailman.app.lifecycle import create_list -from mailman.config import config -from mailman.core import errors -from mailman.interfaces.action import FilterAction -from mailman.interfaces.member import MemberRole -from mailman.interfaces.usermanager import IUserManager from mailman.rules import approved from mailman.testing.helpers import ( - LogFileMark, - get_queue_messages, specialized_message_from_string as mfs) from mailman.testing.layers import ConfigLayer @@ -65,10 +56,8 @@ This is a message body with a non-ascii character =E4 """) - def test_approved_nonascii(self): - result = True - try: - result = self._rule.check(self._mlist, self._msg, {}) - except (UnicodeError, UnicodeWarning): - raise AssertionError('Non-ascii message raised UnicodeError') - self.assertEqual(result, False) + def test_nonascii_body_missing_header(self): + # When the message body contains non-ascii, the rule should not throw + # unicode errors. LP: #949924. + result = self._rule.check(self._mlist, self._msg, {}) + self.assertFalse(result) diff --git a/src/mailman/runners/archive.py b/src/mailman/runners/archive.py index ea85281b1..7295a5c57 100644 --- a/src/mailman/runners/archive.py +++ b/src/mailman/runners/archive.py @@ -25,67 +25,83 @@ __all__ = [ ] -import os +import copy import logging +from email.utils import parsedate_tz, mktime_tz from datetime import datetime -from email.utils import parsedate_tz, mktime_tz, formatdate -from flufl.lock import Lock from lazr.config import as_timedelta from mailman.config import config from mailman.core.runner import Runner +from mailman.interfaces.archiver import ClobberDate +from mailman.utilities.datetime import RFC822_DATE_FMT, now + log = logging.getLogger('mailman.error') +def _should_clobber(msg, msgdata, archiver): + """Should the Date header in the original message get clobbered?""" + # Calculate the Date header of the message as a datetime. What if there + # are multiple Date headers, even in violation of the RFC? For now, take + # the first one. If there are no Date headers, then definitely clobber. + original_date = msg.get('date') + if original_date is None: + return True + section = getattr(config.archiver, archiver, None) + if section is None: + log.error('No archiver config section found: {0}'.format(archiver)) + return False + try: + clobber = ClobberDate[section.clobber_date] + except ValueError: + log.error('Invalid clobber_date for "{0}": {1}'.format( + archiver, section.clobber_date)) + return False + if clobber is ClobberDate.always: + return True + elif clobber is ClobberDate.never: + return False + # Maybe we'll clobber the date. Let's see if it's farther off from now + # than the skew period. + skew = as_timedelta(section.clobber_skew) + try: + time_tuple = parsedate_tz(original_date) + except (ValueError, OverflowError): + # The likely cause of this is that the year in the Date: field is + # horribly incorrect, e.g. (from SF bug # 571634): + # + # Date: Tue, 18 Jun 0102 05:12:09 +0500 + # + # Obviously clobber such dates. + return True + if time_tuple is None: + # There was some other bogosity in the Date header. + return True + claimed_date = datetime.fromtimestamp(mktime_tz(time_tuple)) + return (abs(now() - claimed_date) > skew) + + + class ArchiveRunner(Runner): """The archive runner.""" def _dispose(self, mlist, msg, msgdata): - # Support clobber_date, i.e. setting the date in the archive to the - # received date, not the (potentially bogus) Date: header of the - # original message. - clobber = False - original_date = msg.get('date') - received_time = formatdate(msgdata['received_time']) - if not original_date: - clobber = True - elif int(config.archiver.pipermail.clobber_date_policy) == 1: - clobber = True - elif int(config.archiver.pipermail.clobber_date_policy) == 2: - # What's the timestamp on the original message? - timetup = parsedate_tz(original_date) - now = datetime.now() + received_time = msgdata.get('received_time', now(strip_tzinfo=False)) + for archiver in config.archivers: + msg_copy = copy.deepcopy(msg) + if _should_clobber(msg, msgdata, archiver.name): + original_date = msg_copy['date'] + del msg_copy['date'] + del msg_copy['x-original-date'] + msg_copy['Date'] = received_time.strftime(RFC822_DATE_FMT) + if original_date: + msg_copy['X-Original-Date'] = original_date + # A problem in one archiver should not prevent other archivers + # from running. try: - if not timetup: - clobber = True - else: - utc_timestamp = datetime.fromtimestamp(mktime_tz(timetup)) - date_skew = as_timedelta( - config.archiver.pipermail.allowable_sane_date_skew) - clobber = (abs(now - utc_timestamp) > date_skew) - except (ValueError, OverflowError): - # The likely cause of this is that the year in the Date: field - # is horribly incorrect, e.g. (from SF bug # 571634): - # Date: Tue, 18 Jun 0102 05:12:09 +0500 - # Obviously clobber such dates. - clobber = True - if clobber: - del msg['date'] - del msg['x-original-date'] - msg['Date'] = received_time - if original_date: - msg['X-Original-Date'] = original_date - # Always put an indication of when we received the message. - msg['X-List-Received-Date'] = received_time - # While a list archiving lock is acquired, archive the message. - with Lock(os.path.join(mlist.data_path, 'archive.lck')): - for archiver in config.archivers: - # A problem in one archiver should not prevent other archivers - # from running. - try: - archiver.archive_message(mlist, msg) - except Exception: - log.exception('Broken archiver: %s' % archiver.name) + archiver.archive_message(mlist, msg_copy) + except Exception: + log.exception('Broken archiver: %s' % archiver.name) diff --git a/src/mailman/runners/digest.py b/src/mailman/runners/digest.py index 513a20322..afc11f732 100644 --- a/src/mailman/runners/digest.py +++ b/src/mailman/runners/digest.py @@ -41,11 +41,10 @@ from email.utils import formatdate, getaddresses, make_msgid from urllib2 import URLError from mailman.config import config -from mailman.core.errors import DiscardMessage from mailman.core.i18n import _ from mailman.core.runner import Runner +from mailman.handlers.decorate import decorate from mailman.interfaces.member import DeliveryMode, DeliveryStatus -from mailman.pipeline.decorate import decorate from mailman.utilities.i18n import make from mailman.utilities.mailbox import Mailbox from mailman.utilities.string import oneline, wrap diff --git a/src/mailman/runners/docs/lmtp.rst b/src/mailman/runners/docs/lmtp.rst index 2b6c4b42b..3ce145907 100644 --- a/src/mailman/runners/docs/lmtp.rst +++ b/src/mailman/runners/docs/lmtp.rst @@ -306,7 +306,7 @@ Messages to the `-owner` address also go to the incoming processor. 1 >>> dump_msgdata(messages[0].msgdata) _parsemsg : False - envsender : changeme@example.com + envsender : noreply@example.com listname : mylist@example.com original_size: ... subaddress : owner diff --git a/src/mailman/runners/docs/news.rst b/src/mailman/runners/docs/news.rst deleted file mode 100644 index 71febf95c..000000000 --- a/src/mailman/runners/docs/news.rst +++ /dev/null @@ -1,161 +0,0 @@ -=============== -The news runner -=============== - -The news runner gateways mailing list messages to an NNTP newsgroup. One of -the most important things this runner does is prepare the message for Usenet -(yes, I know that NNTP is not Usenet, but this runner was originally written -to gate to Usenet, which has its own rules). - - >>> mlist = create_list('_xtest@example.com') - >>> mlist.linked_newsgroup = 'comp.lang.python' - -Some NNTP servers such as INN reject messages containing a set of prohibited -headers, so one of the things that the news runner does is remove these -prohibited headers. -:: - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: _xtest@example.com - ... NNTP-Posting-Host: news.example.com - ... NNTP-Posting-Date: today - ... X-Trace: blah blah - ... X-Complaints-To: abuse@dom.ain - ... Xref: blah blah - ... Xref: blah blah - ... Date-Received: yesterday - ... Posted: tomorrow - ... Posting-Version: 99.99 - ... Relay-Version: 88.88 - ... Received: blah blah - ... - ... A message - ... """) - >>> msgdata = {} - - >>> from mailman.runners.news import prepare_message - >>> prepare_message(mlist, msg, msgdata) - >>> msgdata['prepped'] - True - >>> print msg.as_string() - From: aperson@example.com - To: _xtest@example.com - Newsgroups: comp.lang.python - Message-ID: ... - Lines: 1 - <BLANKLINE> - A message - <BLANKLINE> - -Some NNTP servers will reject messages where certain headers are duplicated, -so the news runner must collapse or move these duplicate headers to an -``X-Original-*`` header that the news server doesn't care about. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: _xtest@example.com - ... To: two@example.com - ... Cc: three@example.com - ... Cc: four@example.com - ... Cc: five@example.com - ... Content-Transfer-Encoding: yes - ... Content-Transfer-Encoding: no - ... Content-Transfer-Encoding: maybe - ... - ... A message - ... """) - >>> msgdata = {} - >>> prepare_message(mlist, msg, msgdata) - >>> msgdata['prepped'] - True - >>> print msg.as_string() - From: aperson@example.com - Newsgroups: comp.lang.python - Message-ID: ... - Lines: 1 - To: _xtest@example.com - X-Original-To: two@example.com - CC: three@example.com - X-Original-CC: four@example.com - X-Original-CC: five@example.com - Content-Transfer-Encoding: yes - X-Original-Content-Transfer-Encoding: no - X-Original-Content-Transfer-Encoding: maybe - <BLANKLINE> - A message - <BLANKLINE> - -But if no headers are duplicated, then the news runner doesn't need to modify -the message. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: _xtest@example.com - ... Cc: someother@example.com - ... Content-Transfer-Encoding: yes - ... - ... A message - ... """) - >>> msgdata = {} - >>> prepare_message(mlist, msg, msgdata) - >>> msgdata['prepped'] - True - >>> print msg.as_string() - From: aperson@example.com - To: _xtest@example.com - Cc: someother@example.com - Content-Transfer-Encoding: yes - Newsgroups: comp.lang.python - Message-ID: ... - Lines: 1 - <BLANKLINE> - A message - <BLANKLINE> - - -Newsgroup moderation -==================== - -When the newsgroup is moderated, an ``Approved:`` header with the list's -posting address is added for the benefit of the Usenet system. -:: - - >>> from mailman.interfaces.nntp import NewsModeration - >>> mlist.news_moderation = NewsModeration.open_moderated - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: _xtest@example.com - ... Approved: this gets deleted - ... - ... """) - >>> prepare_message(mlist, msg, {}) - >>> print msg['approved'] - _xtest@example.com - - >>> mlist.news_moderation = NewsModeration.moderated - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: _xtest@example.com - ... Approved: this gets deleted - ... - ... """) - >>> prepare_message(mlist, msg, {}) - >>> print msg['approved'] - _xtest@example.com - -But if the newsgroup is not moderated, the ``Approved:`` header is not changed. - - >>> mlist.news_moderation = NewsModeration.none - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: _xtest@example.com - ... Approved: this doesn't get deleted - ... - ... """) - >>> prepare_message(mlist, msg, {}) - >>> msg['approved'] - u"this doesn't get deleted" - - -XXX More of the NewsRunner should be tested. diff --git a/src/mailman/runners/docs/nntp.rst b/src/mailman/runners/docs/nntp.rst new file mode 100644 index 000000000..31c7210cf --- /dev/null +++ b/src/mailman/runners/docs/nntp.rst @@ -0,0 +1,56 @@ +=============== +The NNTP runner +=============== + +The NNTP runner gateways mailing list messages to an NNTP newsgroup. + + >>> mlist = create_list('test@example.com') + >>> mlist.linked_newsgroup = 'comp.lang.python' + +Get a handle on the NNTP server, which we'll use later to verify the posted +messages. + + >>> from mailman.testing.helpers import get_nntp_server + >>> nntpd = get_nntp_server(cleanups) + +A message gets posted to the mailing list. It may contain some headers which +are prohibited by NNTP servers such as INN. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: test@example.com + ... NNTP-Posting-Host: news.example.com + ... NNTP-Posting-Date: today + ... X-Trace: blah blah + ... X-Complaints-To: abuse@dom.ain + ... Xref: blah blah + ... Xref: blah blah + ... Date-Received: yesterday + ... Posted: tomorrow + ... Posting-Version: 99.99 + ... Relay-Version: 88.88 + ... Received: blah blah + ... + ... A message + ... """) + +The message gets copied to the NNTP queue for preparation and posting. + + >>> filebase = config.switchboards['nntp'].enqueue( + ... msg, listname='test@example.com') + >>> from mailman.testing.helpers import make_testable_runner + >>> from mailman.runners.nntp import NNTPRunner + >>> runner = make_testable_runner(NNTPRunner, 'nntp') + >>> runner.run() + +The message was successfully posted the NNTP server. + + >>> print nntpd.get_message().as_string() + From: aperson@example.com + To: test@example.com + Newsgroups: comp.lang.python + Message-ID: ... + Lines: 1 + <BLANKLINE> + A message + <BLANKLINE> diff --git a/src/mailman/runners/docs/outgoing.rst b/src/mailman/runners/docs/outgoing.rst index b1ffb06a0..d8963ae00 100644 --- a/src/mailman/runners/docs/outgoing.rst +++ b/src/mailman/runners/docs/outgoing.rst @@ -33,7 +33,7 @@ move messages to the 'retry queue' for handling delivery failures. Normally, messages would show up in the outgoing queue after the message has been processed by the rule set and pipeline. But we can simulate that here by injecting a message directly into the outgoing queue. First though, we must -call the ``calculate-recipients`` handler so that the message metadata will be +call the ``member-recipients`` handler so that the message metadata will be populated with the list of addresses to deliver the message to. :: @@ -47,7 +47,7 @@ populated with the list of addresses to deliver the message to. ... """) >>> msgdata = {} - >>> handler = config.handlers['calculate-recipients'] + >>> handler = config.handlers['member-recipients'] >>> handler.process(mlist, msg, msgdata) >>> outgoing_queue = config.switchboards['out'] diff --git a/src/mailman/runners/incoming.py b/src/mailman/runners/incoming.py index 7072f9bcc..d8db926c7 100644 --- a/src/mailman/runners/incoming.py +++ b/src/mailman/runners/incoming.py @@ -61,6 +61,9 @@ class IncomingRunner(Runner): pass config.db.commit() # Process the message through the mailing list's start chain. - process(mlist, msg, msgdata, mlist.posting_chain) + start_chain = (mlist.owner_chain + if msgdata.get('to_owner', False) + else mlist.posting_chain) + process(mlist, msg, msgdata, start_chain) # Do not keep this message queued. return False diff --git a/src/mailman/runners/lmtp.py b/src/mailman/runners/lmtp.py index bee111ad1..45fa5a783 100644 --- a/src/mailman/runners/lmtp.py +++ b/src/mailman/runners/lmtp.py @@ -44,8 +44,10 @@ from mailman.core.runner import Runner from mailman.database.transaction import txn from mailman.email.message import Message from mailman.interfaces.listmanager import IListManager +from mailman.utilities.datetime import now from mailman.utilities.email import add_message_hash + elog = logging.getLogger('mailman.error') qlog = logging.getLogger('mailman.runner') slog = logging.getLogger('mailman.smtp') @@ -181,6 +183,7 @@ class LMTPRunner(Runner, smtpd.SMTPServer): # see if it's destined for a valid mailing list. If so, then queue # the message to the appropriate place and record a 250 status for # that recipient. If not, record a failure status for that recipient. + received_time = now() for to in rcpttos: try: to = parseaddr(to)[1].lower() @@ -196,7 +199,8 @@ class LMTPRunner(Runner, smtpd.SMTPServer): # queue. queue = None msgdata = dict(listname=listname, - original_size=msg.original_size) + original_size=msg.original_size, + received_time=received_time) canonical_subaddress = SUBADDRESS_NAMES.get(subaddress) queue = SUBADDRESS_QUEUES.get(canonical_subaddress) if subaddress is None: diff --git a/src/mailman/runners/news.py b/src/mailman/runners/nntp.py index e3c6f060d..8339c735e 100644 --- a/src/mailman/runners/news.py +++ b/src/mailman/runners/nntp.py @@ -17,6 +17,14 @@ """NNTP runner.""" +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'NNTPRunner', + ] + + import re import email import socket @@ -24,12 +32,12 @@ import logging import nntplib from cStringIO import StringIO -from lazr.config import as_host_port from mailman.config import config from mailman.core.runner import Runner from mailman.interfaces.nntp import NewsModeration +COMMA = ',' COMMASPACE = ', ' log = logging.getLogger('mailman.error') @@ -49,39 +57,46 @@ mcre = re.compile(r""" -class NewsRunner(Runner): +class NNTPRunner(Runner): def _dispose(self, mlist, msg, msgdata): + # Get NNTP server connection information. + host = config.nntp.host.strip() + port = config.nntp.port.strip() + if len(port) == 0: + port = 119 + else: + try: + port = int(port) + except (TypeError, ValueError): + log.exception('Bad [nntp]port value: {0}'.format(port)) + port = 119 # Make sure we have the most up-to-date state - mlist.Load() if not msgdata.get('prepped'): prepare_message(mlist, msg, msgdata) + # Flatten the message object, sticking it in a StringIO object + fp = StringIO(msg.as_string()) + conn = None try: - # Flatten the message object, sticking it in a StringIO object - fp = StringIO(msg.as_string()) - conn = None - try: - try: - nntp_host, nntp_port = as_host_port( - mlist.nntp_host, default_port=119) - conn = nntplib.NNTP(nntp_host, nntp_port, - readermode=True, - user=config.nntp.username, - password=config.nntp.password) - conn.post(fp) - except nntplib.error_temp, e: - log.error('(NNTPDirect) NNTP error for list "%s": %s', - mlist.internal_name(), e) - except socket.error, e: - log.error('(NNTPDirect) socket error for list "%s": %s', - mlist.internal_name(), e) - finally: - if conn: - conn.quit() - except Exception, e: + conn = nntplib.NNTP(host, port, + readermode=True, + user=config.nntp.user, + password=config.nntp.password) + conn.post(fp) + except nntplib.error_temp: + log.exception('{0} NNTP error for {1}'.format( + msg.get('message-id', 'n/a'), mlist.fqdn_listname)) + except socket.error: + log.exception('{0} NNTP socket error for {1}'.format( + msg.get('message-id', 'n/a'), mlist.fqdn_listname)) + except Exception: # Some other exception occurred, which we definitely did not # expect, so set this message up for requeuing. - self._log(e) + log.exception('{0} NNTP unexpected exception for {1}'.format( + msg.get('message-id', 'n/a'), mlist.fqdn_listname)) return True + finally: + if conn: + conn.quit() return False @@ -99,35 +114,41 @@ def prepare_message(mlist, msg, msgdata): # messages? TK: We use stripped_subject (prefix stripped) which was # crafted in CookHeaders.py to ensure prefix was stripped from the subject # came from mailing list user. - stripped_subject = msgdata.get('stripped_subject') \ - or msgdata.get('origsubj') + stripped_subject = msgdata.get('stripped_subject', + msgdata.get('original_subject')) + # XXX 2012-03-31 BAW: rename news_prefix_subject_too to nntp_. This + # requires a schema change. if not mlist.news_prefix_subject_too and stripped_subject is not None: del msg['subject'] msg['subject'] = stripped_subject - # Add the appropriate Newsgroups: header - ngheader = msg['newsgroups'] - if ngheader is not None: + # Add the appropriate Newsgroups header. Multiple Newsgroups headers are + # generally not allowed so we're not testing for them. + header = msg.get('newsgroups') + if header is None: + msg['Newsgroups'] = mlist.linked_newsgroup + else: # See if the Newsgroups: header already contains our linked_newsgroup. # If so, don't add it again. If not, append our linked_newsgroup to # the end of the header list - ngroups = [s.strip() for s in ngheader.split(',')] - if mlist.linked_newsgroup not in ngroups: - ngroups.append(mlist.linked_newsgroup) + newsgroups = [value.strip() for value in header.split(COMMA)] + if mlist.linked_newsgroup not in newsgroups: + newsgroups.append(mlist.linked_newsgroup) # Subtitute our new header for the old one. del msg['newsgroups'] - msg['Newsgroups'] = COMMASPACE.join(ngroups) - else: - # Newsgroups: isn't in the message - msg['Newsgroups'] = mlist.linked_newsgroup + msg['Newsgroups'] = COMMASPACE.join(newsgroups) # Note: We need to be sure two messages aren't ever sent to the same list # in the same process, since message ids need to be unique. Further, if - # messages are crossposted to two Usenet-gated mailing lists, they each - # need to have unique message ids or the nntpd will only accept one of - # them. The solution here is to substitute any existing message-id that - # isn't ours with one of ours, so we need to parse it to be sure we're not - # looping. + # messages are crossposted to two gated mailing lists, they must each have + # unique message ids or the nntpd will only accept one of them. The + # solution here is to substitute any existing message-id that isn't ours + # with one of ours, so we need to parse it to be sure we're not looping. # # Our Message-ID format is <mailman.secs.pid.listname@hostname> + # + # XXX 2012-03-31 BAW: What we really want to do is try posting the message + # to the nntpd first, and only if that fails substitute a unique + # Message-ID. The following should get moved out of prepare_message() and + # into _dispose() above. msgid = msg['message-id'] hackmsgid = True if msgid: @@ -139,29 +160,37 @@ def prepare_message(mlist, msg, msgdata): if hackmsgid: del msg['message-id'] msg['Message-ID'] = email.utils.make_msgid() - # Lines: is useful + # Lines: is useful. if msg['Lines'] is None: # BAW: is there a better way? - count = len(list(email.Iterators.body_line_iterator(msg))) + count = len(list(email.iterators.body_line_iterator(msg))) msg['Lines'] = str(count) # Massage the message headers by remove some and rewriting others. This - # woon't completely sanitize the message, but it will eliminate the bulk - # of the rejections based on message headers. The NNTP server may still + # won't completely sanitize the message, but it will eliminate the bulk of + # the rejections based on message headers. The NNTP server may still # reject the message because of other problems. for header in config.nntp.remove_headers.split(): del msg[header] - for rewrite_pairs in config.nntp.rewrite_duplicate_headers.splitlines(): - if len(rewrite_pairs.strip()) == 0: - continue - header, rewrite = rewrite_pairs.split() - values = msg.get_all(header, []) + dup_headers = config.nntp.rewrite_duplicate_headers.split() + if len(dup_headers) % 2 != 0: + # There are an odd number of headers; ignore the last one. + bad_header = dup_headers.pop() + log.error('Ignoring odd [nntp]rewrite_duplicate_headers: {0}'.format( + bad_header)) + dup_headers.reverse() + while dup_headers: + source = dup_headers.pop() + target = dup_headers.pop() + values = msg.get_all(source, []) if len(values) < 2: - # We only care about duplicates + # We only care about duplicates. continue - del msg[header] - # But keep the first one... - msg[header] = values[0] - for v in values[1:]: - msg[rewrite] = v - # Mark this message as prepared in case it has to be requeued + # Delete all the original headers. + del msg[source] + # Put the first value back on the original header. + msg[source] = values[0] + # And put all the subsequent values on the destination header. + for value in values[1:]: + msg[target] = value + # Mark this message as prepared in case it has to be requeued. msgdata['prepped'] = True diff --git a/src/mailman/runners/pipeline.py b/src/mailman/runners/pipeline.py index 8bee2c4cb..b031662b3 100644 --- a/src/mailman/runners/pipeline.py +++ b/src/mailman/runners/pipeline.py @@ -30,6 +30,9 @@ from mailman.core.runner import Runner class PipelineRunner(Runner): def _dispose(self, mlist, msg, msgdata): # Process the message through the mailing list's pipeline. - process(mlist, msg, msgdata, mlist.posting_pipeline) + pipeline = (mlist.owner_pipeline + if msgdata.get('to_owner', False) + else mlist.posting_pipeline) + process(mlist, msg, msgdata, pipeline) # Do not keep this message queued. return False diff --git a/src/mailman/runners/tests/test_archiver.py b/src/mailman/runners/tests/test_archiver.py index 274aba5ec..6f5804cae 100644 --- a/src/mailman/runners/tests/test_archiver.py +++ b/src/mailman/runners/tests/test_archiver.py @@ -36,9 +36,11 @@ from mailman.config import config from mailman.interfaces.archiver import IArchiver from mailman.runners.archive import ArchiveRunner from mailman.testing.helpers import ( + configuration, make_testable_runner, specialized_message_from_string as mfs) from mailman.testing.layers import ConfigLayer +from mailman.utilities.datetime import RFC822_DATE_FMT, factory, now @@ -54,7 +56,7 @@ class DummyArchiver: def permalink(mlist, msg): filename = msg['x-message-id-hash'] return 'http://archive.example.com/' + filename - + @staticmethod def archive_message(mlist, msg): filename = msg['x-message-id-hash'] @@ -73,11 +75,12 @@ class TestArchiveRunner(unittest.TestCase): def setUp(self): self._mlist = create_list('test@example.com') + self._now = now() # Enable just the dummy archiver. config.push('dummy', """ [archiver.dummy] class: mailman.runners.tests.test_archiver.DummyArchiver - enable: yes + enable: no [archiver.prototype] enable: no [archiver.mhonarc] @@ -85,6 +88,7 @@ class TestArchiveRunner(unittest.TestCase): [archiver.mail_archive] enable: no """) + self._archiveq = config.switchboards['archive'] self._msg = mfs("""\ From: aperson@example.com To: test@example.com @@ -99,10 +103,131 @@ First post! def tearDown(self): config.pop('dummy') + @configuration('archiver.dummy', enable='yes') def test_archive_runner(self): # Ensure that the archive runner ends up archiving the message. - config.switchboards['archive'].enqueue( - self._msg, {}, listname=self._mlist.fqdn_listname) + self._archiveq.enqueue( + self._msg, {}, + listname=self._mlist.fqdn_listname, + received_time=now()) + self._runner.run() + # There should now be a copy of the message in the file system. + filename = os.path.join( + config.MESSAGES_DIR, '4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB') + with open(filename) as fp: + archived = message_from_file(fp) + self.assertEqual(archived['message-id'], '<first>') + + @configuration('archiver.dummy', enable='yes') + def test_archive_runner_with_dated_message(self): + # Date headers don't throw off the archiver runner. + self._msg['Date'] = now(strip_tzinfo=False).strftime(RFC822_DATE_FMT) + self._archiveq.enqueue( + self._msg, {}, + listname=self._mlist.fqdn_listname, + received_time=now()) + self._runner.run() + # There should now be a copy of the message in the file system. + filename = os.path.join( + config.MESSAGES_DIR, '4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB') + with open(filename) as fp: + archived = message_from_file(fp) + self.assertEqual(archived['message-id'], '<first>') + self.assertEqual(archived['date'], 'Mon, 01 Aug 2005 07:49:23 +0000') + + @configuration('archiver.dummy', enable='yes', clobber_date='never') + def test_clobber_date_never(self): + # Even if the Date header is insanely off from the received time of + # the message, if clobber_date is 'never', the header is not clobbered. + self._msg['Date'] = now(strip_tzinfo=False).strftime(RFC822_DATE_FMT) + self._archiveq.enqueue( + self._msg, {}, + listname=self._mlist.fqdn_listname, + received_time=now()) + self._runner.run() + # There should now be a copy of the message in the file system. + filename = os.path.join( + config.MESSAGES_DIR, '4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB') + with open(filename) as fp: + archived = message_from_file(fp) + self.assertEqual(archived['message-id'], '<first>') + self.assertEqual(archived['date'], 'Mon, 01 Aug 2005 07:49:23 +0000') + + @configuration('archiver.dummy', enable='yes') + def test_clobber_dateless(self): + # A message with no Date header will always get clobbered. + self.assertEqual(self._msg['date'], None) + # Now, before enqueuing the message (well, really, calling 'now()' + # again), fast forward a few days. + self._archiveq.enqueue( + self._msg, {}, + listname=self._mlist.fqdn_listname, + received_time=now(strip_tzinfo=False)) + self._runner.run() + # There should now be a copy of the message in the file system. + filename = os.path.join( + config.MESSAGES_DIR, '4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB') + with open(filename) as fp: + archived = message_from_file(fp) + self.assertEqual(archived['message-id'], '<first>') + self.assertEqual(archived['date'], 'Mon, 01 Aug 2005 07:49:23 +0000') + + @configuration('archiver.dummy', enable='yes', clobber_date='always') + def test_clobber_date_always(self): + # The date always gets clobbered with the current received time. + self._msg['Date'] = now(strip_tzinfo=False).strftime(RFC822_DATE_FMT) + # Now, before enqueuing the message (well, really, calling 'now()' + # again as will happen in the runner), fast forward a few days. + self._archiveq.enqueue( + self._msg, {}, + listname=self._mlist.fqdn_listname) + factory.fast_forward(days=4) + self._runner.run() + # There should now be a copy of the message in the file system. + filename = os.path.join( + config.MESSAGES_DIR, '4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB') + with open(filename) as fp: + archived = message_from_file(fp) + self.assertEqual(archived['message-id'], '<first>') + self.assertEqual(archived['date'], 'Fri, 05 Aug 2005 07:49:23 +0000') + self.assertEqual(archived['x-original-date'], + 'Mon, 01 Aug 2005 07:49:23 +0000') + + @configuration('archiver.dummy', + enable='yes', clobber_date='maybe', clobber_skew='1d') + def test_clobber_date_maybe_when_insane(self): + # The date is clobbered if it's farther off from now than its skew + # period. + self._msg['Date'] = now(strip_tzinfo=False).strftime(RFC822_DATE_FMT) + # Now, before enqueuing the message (well, really, calling 'now()' + # again as will happen in the runner), fast forward a few days. + self._archiveq.enqueue( + self._msg, {}, + listname=self._mlist.fqdn_listname) + factory.fast_forward(days=4) + self._runner.run() + # There should now be a copy of the message in the file system. + filename = os.path.join( + config.MESSAGES_DIR, '4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB') + with open(filename) as fp: + archived = message_from_file(fp) + self.assertEqual(archived['message-id'], '<first>') + self.assertEqual(archived['date'], 'Fri, 05 Aug 2005 07:49:23 +0000') + self.assertEqual(archived['x-original-date'], + 'Mon, 01 Aug 2005 07:49:23 +0000') + + @configuration('archiver.dummy', + enable='yes', clobber_date='maybe', clobber_skew='10d') + def test_clobber_date_maybe_when_sane(self): + # The date is not clobbered if it's nearer to now than its skew + # period. + self._msg['Date'] = now(strip_tzinfo=False).strftime(RFC822_DATE_FMT) + # Now, before enqueuing the message (well, really, calling 'now()' + # again as will happen in the runner), fast forward a few days. + self._archiveq.enqueue( + self._msg, {}, + listname=self._mlist.fqdn_listname) + factory.fast_forward(days=4) self._runner.run() # There should now be a copy of the message in the file system. filename = os.path.join( @@ -110,3 +235,5 @@ First post! with open(filename) as fp: archived = message_from_file(fp) self.assertEqual(archived['message-id'], '<first>') + self.assertEqual(archived['date'], 'Mon, 01 Aug 2005 07:49:23 +0000') + self.assertEqual(archived['x-original-date'], None) diff --git a/src/mailman/runners/tests/test_confirm.py b/src/mailman/runners/tests/test_confirm.py index ea972c17f..d2b24a2d1 100644 --- a/src/mailman/runners/tests/test_confirm.py +++ b/src/mailman/runners/tests/test_confirm.py @@ -21,13 +21,13 @@ from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ + 'TestConfirm', ] import unittest from datetime import datetime -from email.iterators import body_line_iterator from zope.component import getUtility from mailman.app.lifecycle import create_list @@ -36,6 +36,7 @@ from mailman.interfaces.registrar import IRegistrar from mailman.interfaces.usermanager import IUserManager from mailman.runners.command import CommandRunner from mailman.testing.helpers import ( + body_line_iterator, get_queue_messages, make_testable_runner, specialized_message_from_string as mfs) diff --git a/src/mailman/runners/tests/test_incoming.py b/src/mailman/runners/tests/test_incoming.py new file mode 100644 index 000000000..5a0d82765 --- /dev/null +++ b/src/mailman/runners/tests/test_incoming.py @@ -0,0 +1,94 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test the incoming queue runner.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestIncoming', + ] + + +import unittest + +from mailman.app.lifecycle import create_list +from mailman.chains.base import TerminalChainBase +from mailman.config import config +from mailman.runners.incoming import IncomingRunner +from mailman.testing.helpers import ( + get_queue_messages, + make_testable_runner, + specialized_message_from_string as mfs) +from mailman.testing.layers import ConfigLayer + + + +class Chain(TerminalChainBase): + name = 'test' + description = 'a test chain' + + def __init__(self, marker): + self._marker = marker + + def _process(self, mlist, msg, msgdata): + msgdata['marker'] = self._marker + config.switchboards['out'].enqueue(msg, msgdata) + + + +class TestIncoming(unittest.TestCase): + """Test the incoming queue runner.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._mlist.posting_chain = 'test posting' + self._mlist.owner_chain = 'test owner' + config.chains['test posting'] = Chain('posting') + config.chains['test owner'] = Chain('owner') + self._in = make_testable_runner(IncomingRunner, 'in') + self._msg = mfs("""\ +From: anne@example.com +To: test@example.com + +""") + + def tearDown(self): + del config.chains['test posting'] + del config.chains['test owner'] + + def test_posting(self): + # A message posted to the list goes through the posting chain. + msgdata = dict(listname='test@example.com') + config.switchboards['in'].enqueue(self._msg, msgdata) + self._in.run() + messages = get_queue_messages('out') + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0].msgdata.get('marker'), 'posting') + + def test_owner(self): + # A message posted to the list goes through the posting chain. + msgdata = dict(listname='test@example.com', + to_owner=True) + config.switchboards['in'].enqueue(self._msg, msgdata) + self._in.run() + messages = get_queue_messages('out') + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0].msgdata.get('marker'), 'owner') diff --git a/src/mailman/runners/tests/test_join.py b/src/mailman/runners/tests/test_join.py index 8cbd8659f..a584fd2c2 100644 --- a/src/mailman/runners/tests/test_join.py +++ b/src/mailman/runners/tests/test_join.py @@ -21,12 +21,13 @@ from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ + 'TestJoin', + 'TestJoinWithDigests', ] import unittest -from email.iterators import body_line_iterator from zope.component import getUtility from mailman.app.lifecycle import create_list @@ -37,6 +38,7 @@ from mailman.interfaces.subscriptions import ISubscriptionService from mailman.interfaces.usermanager import IUserManager from mailman.runners.command import CommandRunner from mailman.testing.helpers import ( + body_line_iterator, get_queue_messages, make_testable_runner, reset_the_world, diff --git a/src/mailman/runners/tests/test_lmtp.py b/src/mailman/runners/tests/test_lmtp.py index 2c4defe59..87b69c7e4 100644 --- a/src/mailman/runners/tests/test_lmtp.py +++ b/src/mailman/runners/tests/test_lmtp.py @@ -28,6 +28,8 @@ __all__ = [ import smtplib import unittest +from datetime import datetime + from mailman.app.lifecycle import create_list from mailman.config import config from mailman.testing.helpers import get_lmtp_client, get_queue_messages @@ -96,3 +98,17 @@ Subject: This has a Message-ID but no X-Message-ID-Hash self.assertEqual(len(all_headers), 1) self.assertEqual(messages[0].msg['x-message-id-hash'], 'MS6QLWERIJLGCRF44J7USBFDELMNT2BW') + + def test_received_time(self): + # The LMTP runner adds a `received_time` key to the metadata. + self._lmtp.sendmail('anne@example.com', ['test@example.com'], """\ +From: anne@example.com +To: test@example.com +Subject: This has no Message-ID header +Message-ID: <ant> + +""") + messages = get_queue_messages('in') + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0].msgdata['received_time'], + datetime(2005, 8, 1, 7, 49, 23)) diff --git a/src/mailman/runners/tests/test_nntp.py b/src/mailman/runners/tests/test_nntp.py new file mode 100644 index 000000000..426e829d8 --- /dev/null +++ b/src/mailman/runners/tests/test_nntp.py @@ -0,0 +1,370 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test the NNTP runner and related utilities.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestPrepareMessage', + 'TestNNTPRunner', + ] + + +import mock +import socket +import nntplib +import unittest + +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.interfaces.nntp import NewsModeration +from mailman.runners import nntp +from mailman.testing.helpers import ( + LogFileMark, + configuration, + get_queue_messages, + make_testable_runner, + specialized_message_from_string as mfs) +from mailman.testing.layers import ConfigLayer + + + +class TestPrepareMessage(unittest.TestCase): + """Test message preparation.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._mlist.linked_newsgroup = 'example.test' + self._msg = mfs("""\ +From: anne@example.com +To: test@example.com +Subject: A newsgroup posting +Message-ID: <ant> + +Testing +""") + + def test_moderated_approved_header(self): + # When the mailing list is moderated , the message will get an + # Approved header, which NNTP software uses to forward to the + # newsgroup. The message would not have gotten to the mailing list if + # it wasn't already approved. + self._mlist.news_moderation = NewsModeration.moderated + nntp.prepare_message(self._mlist, self._msg, {}) + self.assertEqual(self._msg['approved'], 'test@example.com') + + def test_open_moderated_approved_header(self): + # When the mailing list is moderated using an open posting policy, the + # message will get an Approved header, which NNTP software uses to + # forward to the newsgroup. The message would not have gotten to the + # mailing list if it wasn't already approved. + self._mlist.news_moderation = NewsModeration.open_moderated + nntp.prepare_message(self._mlist, self._msg, {}) + self.assertEqual(self._msg['approved'], 'test@example.com') + + def test_moderation_removes_previous_approved_header(self): + # Any existing Approved header is removed from moderated messages. + self._msg['Approved'] = 'a bogus approval' + self._mlist.news_moderation = NewsModeration.moderated + nntp.prepare_message(self._mlist, self._msg, {}) + headers = self._msg.get_all('approved') + self.assertEqual(len(headers), 1) + self.assertEqual(headers[0], 'test@example.com') + + def test_open_moderation_removes_previous_approved_header(self): + # Any existing Approved header is removed from moderated messages. + self._msg['Approved'] = 'a bogus approval' + self._mlist.news_moderation = NewsModeration.open_moderated + nntp.prepare_message(self._mlist, self._msg, {}) + headers = self._msg.get_all('approved') + self.assertEqual(len(headers), 1) + self.assertEqual(headers[0], 'test@example.com') + + def test_stripped_subject(self): + # The cook-headers handler adds the original and/or stripped (of the + # prefix) subject to the metadata. Assume that handler's been run; + # check the Subject header. + self._mlist.news_prefix_subject_too = False + del self._msg['subject'] + self._msg['subject'] = 'Re: Your test' + msgdata = dict(stripped_subject='Your test') + nntp.prepare_message(self._mlist, self._msg, msgdata) + headers = self._msg.get_all('subject') + self.assertEqual(len(headers), 1) + self.assertEqual(headers[0], 'Your test') + + def test_original_subject(self): + # The cook-headers handler adds the original and/or stripped (of the + # prefix) subject to the metadata. Assume that handler's been run; + # check the Subject header. + self._mlist.news_prefix_subject_too = False + del self._msg['subject'] + self._msg['subject'] = 'Re: Your test' + msgdata = dict(original_subject='Your test') + nntp.prepare_message(self._mlist, self._msg, msgdata) + headers = self._msg.get_all('subject') + self.assertEqual(len(headers), 1) + self.assertEqual(headers[0], 'Your test') + + def test_stripped_subject_prefix_okay(self): + # The cook-headers handler adds the original and/or stripped (of the + # prefix) subject to the metadata. Assume that handler's been run; + # check the Subject header. + self._mlist.news_prefix_subject_too = True + del self._msg['subject'] + self._msg['subject'] = 'Re: Your test' + msgdata = dict(stripped_subject='Your test') + nntp.prepare_message(self._mlist, self._msg, msgdata) + headers = self._msg.get_all('subject') + self.assertEqual(len(headers), 1) + self.assertEqual(headers[0], 'Re: Your test') + + def test_original_subject_prefix_okay(self): + # The cook-headers handler adds the original and/or stripped (of the + # prefix) subject to the metadata. Assume that handler's been run; + # check the Subject header. + self._mlist.news_prefix_subject_too = True + del self._msg['subject'] + self._msg['subject'] = 'Re: Your test' + msgdata = dict(original_subject='Your test') + nntp.prepare_message(self._mlist, self._msg, msgdata) + headers = self._msg.get_all('subject') + self.assertEqual(len(headers), 1) + self.assertEqual(headers[0], 'Re: Your test') + + def test_add_newsgroups_header(self): + # Prepared messages get a Newsgroups header. + msgdata = dict(original_subject='Your test') + nntp.prepare_message(self._mlist, self._msg, msgdata) + self.assertEqual(self._msg['newsgroups'], 'example.test') + + def test_add_newsgroups_header_to_existing(self): + # If the message already has a Newsgroups header, the linked newsgroup + # gets appended to that value, using comma-space separated lists. + self._msg['Newsgroups'] = 'foo.test, bar.test' + msgdata = dict(original_subject='Your test') + nntp.prepare_message(self._mlist, self._msg, msgdata) + headers = self._msg.get_all('newsgroups') + self.assertEqual(len(headers), 1) + self.assertEqual(headers[0], 'foo.test, bar.test, example.test') + + def test_add_lines_header(self): + # A Lines: header seems useful. + nntp.prepare_message(self._mlist, self._msg, {}) + self.assertEqual(self._msg['lines'], '1') + + def test_the_message_has_been_prepared(self): + # A key gets added to the metadata so that a retry won't try to + # re-apply all the preparations. + msgdata = {} + nntp.prepare_message(self._mlist, self._msg, msgdata) + self.assertTrue(msgdata.get('prepped')) + + @configuration('nntp', remove_headers='x-complaints-to') + def test_remove_headers(self): + # During preparation, headers which cause problems with certain NNTP + # servers such as INN get removed. + self._msg['X-Complaints-To'] = 'arguments@example.com' + nntp.prepare_message(self._mlist, self._msg, {}) + self.assertEqual(self._msg['x-complaints-to'], None) + + @configuration('nntp', rewrite_duplicate_headers=""" + To X-Original-To + X-Fake X-Original-Fake + """) + def test_rewrite_headers(self): + # Some NNTP servers are very strict about duplicate headers. What we + # can do is look at some headers and if they is more than one of that + # header in the message, all the headers are deleted except the first + # one, and then the other values are moved to the destination header. + # + # In this example, we'll create multiple To headers, which will all + # get moved to X-Original-To. However, because there will only be one + # X-Fake header, it doesn't get rewritten. + self._msg['To'] = 'test@example.org' + self._msg['To'] = 'test@example.net' + self._msg['X-Fake'] = 'ignore me' + self.assertEqual(len(self._msg.get_all('to')), 3) + self.assertEqual(len(self._msg.get_all('x-fake')), 1) + nntp.prepare_message(self._mlist, self._msg, {}) + tos = self._msg.get_all('to') + self.assertEqual(len(tos), 1) + self.assertEqual(tos[0], 'test@example.com') + original_tos = self._msg.get_all('x-original-to') + self.assertEqual(len(original_tos), 2) + self.assertEqual(original_tos, + ['test@example.org', 'test@example.net']) + fakes = self._msg.get_all('x-fake') + self.assertEqual(len(fakes), 1) + self.assertEqual(fakes[0], 'ignore me') + self.assertEqual(self._msg.get_all('x-original-fake'), None) + + @configuration('nntp', rewrite_duplicate_headers=""" + To X-Original-To + X-Fake + """) + def test_odd_duplicates(self): + # This is just a corner case, where there is an odd number of rewrite + # headers. In that case, the odd-one-out does not get rewritten. + self._msg['x-fake'] = 'one' + self._msg['x-fake'] = 'two' + self._msg['x-fake'] = 'three' + self.assertEqual(len(self._msg.get_all('x-fake')), 3) + nntp.prepare_message(self._mlist, self._msg, {}) + fakes = self._msg.get_all('x-fake') + self.assertEqual(len(fakes), 3) + self.assertEqual(fakes, ['one', 'two', 'three']) + + + +class TestNNTPRunner(unittest.TestCase): + """The NNTP runner hands messages off to the NNTP server.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._mlist.linked_newsgroup = 'example.test' + self._msg = mfs("""\ +From: anne@example.com +To: test@example.com +Subject: A newsgroup posting +Message-ID: <ant> + +Testing +""") + self._runner = make_testable_runner(nntp.NNTPRunner, 'nntp') + self._nntpq = config.switchboards['nntp'] + + @mock.patch('nntplib.NNTP') + def test_connect(self, class_mock): + # Test connection to the NNTP server with default values. + self._nntpq.enqueue(self._msg, {}, listname='test@example.com') + self._runner.run() + class_mock.assert_called_once_with( + '', 119, user='', password='', readermode=True) + + @configuration('nntp', user='alpha', password='beta', + host='nntp.example.com', port='2112') + @mock.patch('nntplib.NNTP') + def test_connect_with_configuration(self, class_mock): + # Test connection to the NNTP server with specific values. + self._nntpq.enqueue(self._msg, {}, listname='test@example.com') + self._runner.run() + class_mock.assert_called_once_with( + 'nntp.example.com', 2112, + user='alpha', password='beta', readermode=True) + + @mock.patch('nntplib.NNTP') + def test_post(self, class_mock): + # Test that the message is posted to the NNTP server. + self._nntpq.enqueue(self._msg, {}, listname='test@example.com') + self._runner.run() + # Get the mocked instance, which was used in the runner. + conn_mock = class_mock() + # The connection object's post() method was called once with a + # file-like object containing the message's bytes. Read those bytes + # and make some simple checks that the message is what we expected. + args = conn_mock.post.call_args + # One positional argument. + self.assertEqual(len(args[0]), 1) + # No keyword arguments. + self.assertEqual(len(args[1]), 0) + msg = mfs(args[0][0].read()) + self.assertEqual(msg['subject'], 'A newsgroup posting') + + @mock.patch('nntplib.NNTP') + def test_connection_got_quit(self, class_mock): + # The NNTP connection gets closed after a successful post. + # Test that the message is posted to the NNTP server. + self._nntpq.enqueue(self._msg, {}, listname='test@example.com') + self._runner.run() + # Get the mocked instance, which was used in the runner. + conn_mock = class_mock() + # The connection object's post() method was called once with a + # file-like object containing the message's bytes. Read those bytes + # and make some simple checks that the message is what we expected. + conn_mock.quit.assert_called_once_with() + + @mock.patch('nntplib.NNTP', side_effect=nntplib.error_temp) + def test_connect_with_nntplib_failure(self, class_mock): + self._nntpq.enqueue(self._msg, {}, listname='test@example.com') + mark = LogFileMark('mailman.error') + self._runner.run() + log_message = mark.readline()[:-1] + self.assertTrue(log_message.endswith( + 'NNTP error for test@example.com')) + + @mock.patch('nntplib.NNTP', side_effect=socket.error) + def test_connect_with_socket_failure(self, class_mock): + self._nntpq.enqueue(self._msg, {}, listname='test@example.com') + mark = LogFileMark('mailman.error') + self._runner.run() + log_message = mark.readline()[:-1] + self.assertTrue(log_message.endswith( + 'NNTP socket error for test@example.com')) + + @mock.patch('nntplib.NNTP', side_effect=RuntimeError) + def test_connect_with_other_failure(self, class_mock): + # In this failure mode, the message stays queued, so we can only run + # the nntp runner once. + def once(runner): + # I.e. stop immediately, since the queue will not be empty. + return True + runner = make_testable_runner(nntp.NNTPRunner, 'nntp', predicate=once) + self._nntpq.enqueue(self._msg, {}, listname='test@example.com') + mark = LogFileMark('mailman.error') + runner.run() + log_message = mark.readline()[:-1] + self.assertTrue(log_message.endswith( + 'NNTP unexpected exception for test@example.com')) + messages = get_queue_messages('nntp') + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0].msgdata['listname'], 'test@example.com') + self.assertEqual(messages[0].msg['subject'], 'A newsgroup posting') + + @mock.patch('nntplib.NNTP', side_effect=nntplib.error_temp) + def test_connection_never_gets_quit_after_failures(self, class_mock): + # The NNTP connection doesn't get closed after a unsuccessful + # connection, since there's nothing to close. + self._nntpq.enqueue(self._msg, {}, listname='test@example.com') + self._runner.run() + # Get the mocked instance, which was used in the runner. Turn off the + # exception raising side effect first though! + class_mock.side_effect = None + conn_mock = class_mock() + # The connection object's post() method was called once with a + # file-like object containing the message's bytes. Read those bytes + # and make some simple checks that the message is what we expected. + self.assertEqual(conn_mock.quit.call_count, 0) + + @mock.patch('nntplib.NNTP') + def test_connection_got_quit_after_post_failure(self, class_mock): + # The NNTP connection does get closed after a unsuccessful post. + # Add a side-effect to the instance mock's .post() method. + conn_mock = class_mock() + conn_mock.post.side_effect = nntplib.error_temp + self._nntpq.enqueue(self._msg, {}, listname='test@example.com') + self._runner.run() + # The connection object's post() method was called once with a + # file-like object containing the message's bytes. Read those bytes + # and make some simple checks that the message is what we expected. + conn_mock.quit.assert_called_once_with() diff --git a/src/mailman/runners/tests/test_owner.py b/src/mailman/runners/tests/test_owner.py new file mode 100644 index 000000000..622bb2255 --- /dev/null +++ b/src/mailman/runners/tests/test_owner.py @@ -0,0 +1,142 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test posting to a mailing list's -owner address.""" + +# XXX 2012-03-23 BAW: This is not necessarily the best place for this test. +# We really need a better place to collect these sort of end-to-end posting +# tests. They're not exactly integration tests, but they do touch lots of +# parts of the system. + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestEmailToOwner', + ] + + +import unittest + +from operator import itemgetter +from zope.component import getUtility + +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.interfaces.member import MemberRole +from mailman.interfaces.usermanager import IUserManager +from mailman.testing.helpers import ( + TestableMaster, + get_lmtp_client, + make_testable_runner) +from mailman.runners.incoming import IncomingRunner +from mailman.runners.outgoing import OutgoingRunner +from mailman.runners.pipeline import PipelineRunner +from mailman.testing.layers import SMTPLayer + + + +class TestEmailToOwner(unittest.TestCase): + """Test emailing a mailing list's -owner address.""" + + layer = SMTPLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + # Add some owners, moderators, and members + manager = getUtility(IUserManager) + anne = manager.create_address('anne@example.com') + bart = manager.create_address('bart@example.com') + cris = manager.create_address('cris@example.com') + dave = manager.create_address('dave@example.com') + self._mlist.subscribe(anne, MemberRole.member) + self._mlist.subscribe(anne, MemberRole.owner) + self._mlist.subscribe(bart, MemberRole.moderator) + self._mlist.subscribe(bart, MemberRole.owner) + self._mlist.subscribe(cris, MemberRole.moderator) + self._mlist.subscribe(dave, MemberRole.member) + config.db.commit() + self._inq = make_testable_runner(IncomingRunner, 'in') + self._pipelineq = make_testable_runner(PipelineRunner, 'pipeline') + self._outq = make_testable_runner(OutgoingRunner, 'out') + # Python 2.7 has assertMultiLineEqual. Let this work without bounds. + self.maxDiff = None + self.eq = getattr(self, 'assertMultiLineEqual', self.assertEqual) + + def test_owners_get_email(self): + # XXX 2012-03-23 BAW: We can't use a layer here because we need both + # the SMTPLayer and LMTPLayer and these are incompatible. There's no + # way to make zope.test* happy without causing errors or worse. Live + # with this hack until we can rip all that layer crap out and use + # something like testresources. + def wait(): + get_lmtp_client(quiet=True) + lmtpd = TestableMaster(wait) + lmtpd.start('lmtp') + # Post a message to the list's -owner address, and all the owners will + # get a copy of the message. + lmtp = get_lmtp_client(quiet=True) + lmtp.lhlo('remote.example.org') + lmtp.sendmail('zuzu@example.org', ['test-owner@example.com'], """\ +From: Zuzu Person <zuzu@example.org> +To: test-owner@example.com +Message-ID: <ant> + +Can you help me? +""") + lmtpd.stop() + # There should now be one message sitting in the incoming queue. + # Check that, then process it. Don't use get_queue_messages() since + # that will empty the queue. + self.assertEqual(len(config.switchboards['in'].files), 1) + self._inq.run() + # There should now be one message sitting in the pipeline queue. + # Process that one too. + self.assertEqual(len(config.switchboards['pipeline'].files), 1) + self._pipelineq.run() + # The message has made its way to the outgoing queue. Again, check + # and process that one. + self.assertEqual(len(config.switchboards['out'].files), 1) + self._outq.run() + # The SMTP server has now received three messages, one for each of the + # owners and moderators. Of course, Bart is both an owner and a + # moderator, so he'll get only one copy of the message. Dave does not + # get a copy of the message. + messages = sorted(SMTPLayer.smtpd.messages, key=itemgetter('x-rcptto')) + self.assertEqual(len(messages), 3) + self.assertEqual(messages[0]['x-rcptto'], 'anne@example.com') + self.assertEqual(messages[1]['x-rcptto'], 'bart@example.com') + self.assertEqual(messages[2]['x-rcptto'], 'cris@example.com') + # And yet, all three messages are addressed to the -owner address. + for message in messages: + self.assertEqual(message['to'], 'test-owner@example.com') + # All three messages will have two X-MailFrom headers. One is added + # by the LMTP server accepting Zuzu's original message, and will + # contain her posting address, i.e. zuzu@example.com. The second one + # is added by the lazr.smtptest server that accepts Mailman's VERP'd + # message to the individual recipient. By verifying both, we prove + # that Zuzu sent the original message, and that Mailman is VERP'ing + # the copy to all the owners. + self.assertEqual( + messages[0].get_all('x-mailfrom'), + ['zuzu@example.org', 'test-bounces+anne=example.com@example.com']) + self.assertEqual( + messages[1].get_all('x-mailfrom'), + ['zuzu@example.org', 'test-bounces+bart=example.com@example.com']) + self.assertEqual( + messages[2].get_all('x-mailfrom'), + ['zuzu@example.org', 'test-bounces+cris=example.com@example.com']) diff --git a/src/mailman/runners/tests/test_pipeline.py b/src/mailman/runners/tests/test_pipeline.py new file mode 100644 index 000000000..8776bf844 --- /dev/null +++ b/src/mailman/runners/tests/test_pipeline.py @@ -0,0 +1,115 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test the pipeline runner.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestPipelineRunner', + ] + + +import unittest + +from zope.interface import implements + +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.interfaces.handler import IHandler +from mailman.interfaces.pipeline import IPipeline +from mailman.runners.pipeline import PipelineRunner +from mailman.testing.helpers import ( + make_testable_runner, + specialized_message_from_string as mfs) +from mailman.testing.layers import ConfigLayer + + + +class MyTestHandler: + implements(IHandler) + name = 'test handler' + description = 'A test handler' + + def __init__(self, marker, test): + self._marker = marker + self._test = test + + def process(self, mlist, msg, msgdata): + self._test.mark(self._marker) + + +class MyTestPipeline: + implements(IPipeline) + name = 'test' + description = 'a test pipeline' + + def __init__(self, marker, test): + self._marker = marker + self._test = test + + def __iter__(self): + yield MyTestHandler(self._marker, self._test) + + + +class TestPipelineRunner(unittest.TestCase): + """Test the pipeline runner.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._mlist.posting_pipeline = 'test posting' + self._mlist.owner_pipeline = 'test owner' + config.pipelines['test posting'] = MyTestPipeline('posting', self) + config.pipelines['test owner'] = MyTestPipeline('owner', self) + self._pipeline = make_testable_runner(PipelineRunner, 'pipeline') + self._markers = [] + self._msg = mfs("""\ +From: anne@example.com +To: test@example.com + +""") + + def tearDown(self): + del config.pipelines['test posting'] + del config.pipelines['test owner'] + + def mark(self, marker): + # Record a marker seen by a handler. + self._markers.append(marker) + + def test_posting(self): + # A message accepted for posting gets processed through the posting + # pipeline. + msgdata = dict(listname='test@example.com') + config.switchboards['pipeline'].enqueue(self._msg, msgdata) + self._pipeline.run() + self.assertEqual(len(self._markers), 1) + self.assertEqual(self._markers[0], 'posting') + + def test_owner(self): + # A message accepted for posting to a list's owners gets processed + # through the owner pipeline. + msgdata = dict(listname='test@example.com', + to_owner=True) + config.switchboards['pipeline'].enqueue(self._msg, msgdata) + self._pipeline.run() + self.assertEqual(len(self._markers), 1) + self.assertEqual(self._markers[0], 'owner') diff --git a/src/mailman/styles/default.py b/src/mailman/styles/default.py index e64bbe40b..b6900dca6 100644 --- a/src/mailman/styles/default.py +++ b/src/mailman/styles/default.py @@ -27,8 +27,7 @@ __all__ = [ # XXX Styles need to be reconciled with lazr.config. -import datetime - +from datetime import timedelta from zope.interface import implements from mailman.core.i18n import _ @@ -153,16 +152,15 @@ from: .*@uplinkpro.com mlist.autoresponse_postings_text = '' mlist.autorespond_requests = ResponseAction.none mlist.autoresponse_request_text = '' - mlist.autoresponse_grace_period = datetime.timedelta(days=90) + mlist.autoresponse_grace_period = timedelta(days=90) # Bounces mlist.forward_unrecognized_bounces_to = ( UnrecognizedBounceDisposition.administrators) mlist.process_bounces = True mlist.bounce_score_threshold = 5.0 - mlist.bounce_info_stale_after = datetime.timedelta(days=7) + mlist.bounce_info_stale_after = timedelta(days=7) mlist.bounce_you_are_disabled_warnings = 3 - mlist.bounce_you_are_disabled_warnings_interval = ( - datetime.timedelta(days=7)) + mlist.bounce_you_are_disabled_warnings_interval = timedelta(days=7) mlist.bounce_notify_owner_on_disable = True mlist.bounce_notify_owner_on_removal = True # This holds legacy member related information. It's keyed by the @@ -218,6 +216,11 @@ from: .*@uplinkpro.com # The default pipeline to send accepted messages through to the # mailing list's members. mlist.posting_pipeline = 'default-posting-pipeline' + # The processing chain that messages posted to this mailing list's + # -owner address gets processed by. + mlist.owner_chain = 'default-owner-chain' + # The default pipeline to send -owner email through. + mlist.owner_pipeline = 'default-owner-pipeline' def match(self, mailing_list, styles): """See `IStyle`.""" diff --git a/src/mailman/testing/helpers.py b/src/mailman/testing/helpers.py index 58c72d6d9..8295ef3a8 100644 --- a/src/mailman/testing/helpers.py +++ b/src/mailman/testing/helpers.py @@ -1,3 +1,4 @@ +# Copyright (C) 2008-2012 by the Free Software Foundation, Inc. # # This file is part of GNU Mailman. # @@ -22,10 +23,13 @@ __metaclass__ = type __all__ = [ 'LogFileMark', 'TestableMaster', + 'body_line_iterator', 'call_api', + 'configuration', 'digest_mbox', 'event_subscribers', 'get_lmtp_client', + 'get_nntp_server', 'get_queue_messages', 'make_testable_runner', 'reset_the_world', @@ -37,7 +41,9 @@ __all__ = [ import os import json +import mock import time +import uuid import errno import signal import socket @@ -66,6 +72,9 @@ from mailman.interfaces.usermanager import IUserManager from mailman.utilities.mailbox import Mailbox +NL = '\n' + + def make_testable_runner(runner_class, name=None, predicate=None): """Create a runner that runs until its queue is empty. @@ -246,6 +255,24 @@ def get_lmtp_client(quiet=False): +def get_nntp_server(cleanups): + """Create and start an NNTP server mock. + + This can be used to retrieve the posted message for verification. + """ + patcher = mock.patch('nntplib.NNTP') + server_class = patcher.start() + cleanups.append(patcher.stop) + nntpd = server_class() + # A class for more convenient access to the posted message. + class NNTPProxy: + def get_message(self): + args = nntpd.post.call_args + return specialized_message_from_string(args[0][0].read()) + return NNTPProxy() + + + def wait_for_webservice(): """Wait for the REST server to start serving requests.""" until = datetime.datetime.now() + as_timedelta(config.devmode.wait) @@ -329,6 +356,42 @@ def event_subscribers(*subscribers): +class configuration: + """A decorator/context manager for temporarily setting configurations.""" + + def __init__(self, section, **kws): + self._section = section + self._values = kws.copy() + self._uuid = uuid.uuid4().hex + + def _apply(self): + lines = ['[{0}]'.format(self._section)] + for key, value in self._values.items(): + lines.append('{0}: {1}'.format(key, value)) + config.push(self._uuid, NL.join(lines)) + + def _remove(self): + config.pop(self._uuid) + + def __enter__(self): + self._apply() + + def __exit__(self, *exc_info): + self._remove() + # Do not suppress exceptions. + return False + + def __call__(self, func): + def wrapper(*args, **kws): + self._apply() + try: + return func(*args, **kws) + finally: + self._remove() + return wrapper + + + def subscribe(mlist, first_name, role=MemberRole.member): """Helper for subscribing a sample person to a mailing list.""" user_manager = getUtility(IUserManager) @@ -412,3 +475,19 @@ class LogFileMark: with open(self._filename) as fp: fp.seek(self._filepos) return fp.readline() + + + +# In Python 2.6, body_line_iterator() uses a cStringIO.StringIO() which cannot +# handle unicode. In Python 2.7 this works fine. I hate version checks but +# this is the easiest way to handle it. OTOH, we could just use the manual +# way for all Python versions instead. +import sys +if sys.hexversion >= 0x2070000: + from email.iterators import body_line_iterator +else: + def body_line_iterator(msg, decode=False): + payload = msg.get_payload(decode=decode) + bytes_payload = payload.encode('utf-8') + for line in bytes_payload.splitlines(): + yield line diff --git a/src/mailman/testing/layers.py b/src/mailman/testing/layers.py index 04ab8f91f..41ef86935 100644 --- a/src/mailman/testing/layers.py +++ b/src/mailman/testing/layers.py @@ -17,6 +17,14 @@ """Mailman test layers.""" +# XXX 2012-03-23 BAW: Layers really really suck. For example, the +# test_owners_get_email() test requires that both the SMTPLayer and LMTPLayer +# be set up, but there's apparently no way to do that and make zope.testing +# happy. This causes no tests failures, but it does cause errors at the end +# of the full test run. For now, I'll ignore that, but I do want to +# eventually get rid of the zope.test* dependencies and use something like +# testresources or some such. + from __future__ import absolute_import, unicode_literals __metaclass__ = type diff --git a/src/mailman/testing/testing.cfg b/src/mailman/testing/testing.cfg index d503247de..91613cc8d 100644 --- a/src/mailman/testing/testing.cfg +++ b/src/mailman/testing/testing.cfg @@ -22,6 +22,9 @@ #class: mailman.database.postgresql.PostgreSQLDatabase #url: postgres://barry:barry@localhost/mailman +[mailman] +site_owner: noreply@example.com + [mta] smtp_port: 9025 lmtp_port: 9024 @@ -45,7 +48,7 @@ max_restarts: 1 [runner.lmtp] max_restarts: 1 -[runner.news] +[runner.nntp] max_restarts: 1 [runner.out] diff --git a/src/mailman/tests/test_documentation.py b/src/mailman/tests/test_documentation.py index f459f3a92..a2c1ab592 100644 --- a/src/mailman/tests/test_documentation.py +++ b/src/mailman/tests/test_documentation.py @@ -178,6 +178,14 @@ def setup(testobj): testobj.globs['smtpd'] = SMTPLayer.smtpd testobj.globs['stop'] = stop testobj.globs['transaction'] = config.db + # Add this so that cleanups can be automatically added by the doctest. + testobj.globs['cleanups'] = [] + + + +def teardown(testobj): + for cleanup in testobj.globs['cleanups']: + cleanup() @@ -223,7 +231,8 @@ def test_suite(): path, package='mailman', optionflags=flags, - setUp=setup) + setUp=setup, + tearDown=teardown) test.layer = layer suite.addTest(test) return suite diff --git a/src/mailman/utilities/datetime.py b/src/mailman/utilities/datetime.py index 96d14fce5..3f451efaf 100644 --- a/src/mailman/utilities/datetime.py +++ b/src/mailman/utilities/datetime.py @@ -28,9 +28,12 @@ from __future__ import absolute_import, unicode_literals __metaclass__ = type __all__ = [ 'DateFactory', + 'RFC822_DATE_FMT', + 'UTC', 'factory', 'now', 'today', + 'utc', ] @@ -39,6 +42,30 @@ import datetime from mailman.testing import layers +# Python always sets the locale to 'C' locale unless the user explicitly calls +# locale.setlocale(locale.LC_ALL, ''). Since we never do this in Mailman (and +# no library better do it either!) this will safely give us expected RFC 5322 +# Date headers. +RFC822_DATE_FMT = '%a, %d %b %Y %H:%M:%S %z' + + + +# Definition of UTC timezone, taken from +# http://docs.python.org/library/datetime.html +ZERO = datetime.timedelta(0) + +class UTC(datetime.tzinfo): + def utcoffset(self, dt): + return ZERO + def tzname(self, dt): + return 'UTC' + def dst(self, dt): + return ZERO + +utc = UTC() +_missing = object() + + class DateFactory: """A factory for today() and now() that works with testing.""" @@ -47,12 +74,21 @@ class DateFactory: predictable_now = None predictable_today = None - def now(self, tz=None): + def now(self, tz=_missing, strip_tzinfo=True): # We can't automatically fast-forward because some tests require us to # stay on the same day for a while, e.g. autorespond.txt. - return (self.predictable_now - if layers.is_testing() - else datetime.datetime.now(tz)) + if tz is _missing: + tz = utc + # Storm cannot yet handle datetimes with tz suffixes. Assume we're + # using UTC datetimes everywhere, so set the tzinfo to None. This + # does *not* change the actual time values. LP: #280708 + tz_now = (self.predictable_now + if layers.is_testing() + else datetime.datetime.now(tz)) + return (tz_now.replace(tzinfo=None) + if strip_tzinfo + else tz_now) + def today(self): return (self.predictable_today @@ -61,7 +97,8 @@ class DateFactory: @classmethod def reset(cls): - cls.predictable_now = datetime.datetime(2005, 8, 1, 7, 49, 23) + cls.predictable_now = datetime.datetime(2005, 8, 1, 7, 49, 23, + tzinfo=utc) cls.predictable_today = cls.predictable_now.date() @classmethod diff --git a/src/mailman/version.py b/src/mailman/version.py index 8f13f8f51..a75bea025 100644 --- a/src/mailman/version.py +++ b/src/mailman/version.py @@ -18,8 +18,8 @@ """Mailman version strings.""" # Mailman version. -VERSION = '3.0.0a8+' -CODENAME = "The Twilight Zone" +VERSION = '3.0.0b1+' +CODENAME = "Freeze" # And as a hex number in the manner of PY_VERSION_HEX. ALPHA = 0xa @@ -34,7 +34,7 @@ MINOR_REV = 0 MICRO_REV = 0 REL_LEVEL = BETA # At most 15 beta releases! -REL_SERIAL = 1 +REL_SERIAL = 2 HEX_VERSION = ((MAJOR_REV << 24) | (MINOR_REV << 16) | (MICRO_REV << 8) | (REL_LEVEL << 4) | (REL_SERIAL << 0)) |