diff options
56 files changed, 361 insertions, 1368 deletions
diff --git a/bootstrap.py b/bootstrap.py deleted file mode 100644 index ec3757a96..000000000 --- a/bootstrap.py +++ /dev/null @@ -1,165 +0,0 @@ -############################################################################## -# -# Copyright (c) 2006 Zope Foundation and Contributors. -# All Rights Reserved. -# -# This software is subject to the provisions of the Zope Public License, -# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. -# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED -# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -# FOR A PARTICULAR PURPOSE. -# -############################################################################## -"""Bootstrap a buildout-based project - -Simply run this script in a directory containing a buildout.cfg. -The script accepts buildout command-line options, so you can -use the -c option to specify an alternate configuration file. -""" - -import os, shutil, sys, tempfile -from optparse import OptionParser - -tmpeggs = tempfile.mkdtemp() - -usage = '''\ -[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options] - -Bootstraps a buildout-based project. - -Simply run this script in a directory containing a buildout.cfg, using the -Python that you want bin/buildout to use. - -Note that by using --setup-source and --download-base to point to -local resources, you can keep this script from going over the network. -''' - -parser = OptionParser(usage=usage) -parser.add_option("-v", "--version", help="use a specific zc.buildout version") - -parser.add_option("-t", "--accept-buildout-test-releases", - dest='accept_buildout_test_releases', - action="store_true", default=False, - help=("Normally, if you do not specify a --version, the " - "bootstrap script and buildout gets the newest " - "*final* versions of zc.buildout and its recipes and " - "extensions for you. If you use this flag, " - "bootstrap and buildout will get the newest releases " - "even if they are alphas or betas.")) -parser.add_option("-c", "--config-file", - help=("Specify the path to the buildout configuration " - "file to be used.")) -parser.add_option("-f", "--find-links", - help=("Specify a URL to search for buildout releases")) - - -options, args = parser.parse_args() - -###################################################################### -# load/install distribute - -to_reload = False -try: - import pkg_resources, setuptools - if not hasattr(pkg_resources, '_distribute'): - to_reload = True - raise ImportError -except ImportError: - ez = {} - - try: - from urllib.request import urlopen - except ImportError: - from urllib2 import urlopen - - exec(urlopen('http://python-distribute.org/distribute_setup.py').read(), ez) - setup_args = dict(to_dir=tmpeggs, download_delay=0, no_fake=True) - ez['use_setuptools'](**setup_args) - - if to_reload: - reload(pkg_resources) - import pkg_resources - # This does not (always?) update the default working set. We will - # do it. - for path in sys.path: - if path not in pkg_resources.working_set.entries: - pkg_resources.working_set.add_entry(path) - -###################################################################### -# Install buildout - -ws = pkg_resources.working_set - -cmd = [sys.executable, '-c', - 'from setuptools.command.easy_install import main; main()', - '-mZqNxd', tmpeggs] - -find_links = os.environ.get( - 'bootstrap-testing-find-links', - options.find_links or - ('http://downloads.buildout.org/' - if options.accept_buildout_test_releases else None) - ) -if find_links: - cmd.extend(['-f', find_links]) - -distribute_path = ws.find( - pkg_resources.Requirement.parse('distribute')).location - -requirement = 'zc.buildout' -version = options.version -if version is None and not options.accept_buildout_test_releases: - # Figure out the most recent final version of zc.buildout. - import setuptools.package_index - _final_parts = '*final-', '*final' - def _final_version(parsed_version): - for part in parsed_version: - if (part[:1] == '*') and (part not in _final_parts): - return False - return True - index = setuptools.package_index.PackageIndex( - search_path=[distribute_path]) - if find_links: - index.add_find_links((find_links,)) - req = pkg_resources.Requirement.parse(requirement) - if index.obtain(req) is not None: - best = [] - bestv = None - for dist in index[req.project_name]: - distv = dist.parsed_version - if _final_version(distv): - if bestv is None or distv > bestv: - best = [dist] - bestv = distv - elif distv == bestv: - best.append(dist) - if best: - best.sort() - version = best[-1].version -if version: - requirement = '=='.join((requirement, version)) -cmd.append(requirement) - -import subprocess -if subprocess.call(cmd, env=dict(os.environ, PYTHONPATH=distribute_path)) != 0: - raise Exception( - "Failed to execute command:\n%s", - repr(cmd)[1:-1]) - -###################################################################### -# Import and run buildout - -ws.add_entry(tmpeggs) -ws.require(requirement) -import zc.buildout.buildout - -if not [a for a in args if '=' not in a]: - args.append('bootstrap') - -# if -c was provided, we push it back into args for buildout' main function -if options.config_file is not None: - args[0:0] = ['-c', options.config_file] - -zc.buildout.buildout.main(args) -shutil.rmtree(tmpeggs) diff --git a/buildout.cfg b/buildout.cfg deleted file mode 100644 index faf597281..000000000 --- a/buildout.cfg +++ /dev/null @@ -1,21 +0,0 @@ -[buildout] -parts = - interpreter - test -unzip = true -develop = . - -[interpreter] -recipe = zc.recipe.egg -interpreter = py -eggs = - mailman - -[test] -recipe = zc.recipe.testrunner -eggs = - mailman -defaults = '--tests-pattern ^tests --exit-with-status'.split() -# Hack in extra arguments to zope.testrunner. -initialization = from mailman.testing import initialize; - initialize('${buildout:directory}') diff --git a/data/coverage.py b/data/coverage.py deleted file mode 100644 index 66e55e0c4..000000000 --- a/data/coverage.py +++ /dev/null @@ -1,952 +0,0 @@ -#!/usr/bin/python -# -# Perforce Defect Tracking Integration Project -# <http://www.ravenbrook.com/project/p4dti/> -# -# COVERAGE.PY -- COVERAGE TESTING -# -# Gareth Rees, Ravenbrook Limited, 2001-12-04 -# Ned Batchelder, 2004-12-12 -# http://nedbatchelder.com/code/modules/coverage.html -# -# -# 1. INTRODUCTION -# -# This module provides coverage testing for Python code. -# -# The intended readership is all Python developers. -# -# This document is not confidential. -# -# See [GDR 2001-12-04a] for the command-line interface, programmatic -# interface and limitations. See [GDR 2001-12-04b] for requirements and -# design. - -r"""Usage: - -coverage.py -x [-p] MODULE.py [ARG1 ARG2 ...] - Execute module, passing the given command-line arguments, collecting - coverage data. With the -p option, write to a temporary file containing - the machine name and process ID. - -coverage.py -e - Erase collected coverage data. - -coverage.py -c - Collect data from multiple coverage files (as created by -p option above) - and store it into a single file representing the union of the coverage. - -coverage.py -r [-m] [-o dir1,dir2,...] FILE1 FILE2 ... - Report on the statement coverage for the given files. With the -m - option, show line numbers of the statements that weren't executed. - -coverage.py -a [-d dir] [-o dir1,dir2,...] FILE1 FILE2 ... - Make annotated copies of the given files, marking statements that - are executed with > and statements that are missed with !. With - the -d option, make the copies in that directory. Without the -d - option, make each copy in the same directory as the original. - --o dir,dir2,... - Omit reporting or annotating files when their filename path starts with - a directory listed in the omit list. - e.g. python coverage.py -i -r -o c:\python23,lib\enthought\traits - -Coverage data is saved in the file .coverage by default. Set the -COVERAGE_FILE environment variable to save it somewhere else.""" - -__version__ = "2.6.20060823" # see detailed history at the end of this file. - -import compiler -import compiler.visitor -import os -import re -import string -import sys -import threading -import types -from socket import gethostname - -# 2. IMPLEMENTATION -# -# This uses the "singleton" pattern. -# -# The word "morf" means a module object (from which the source file can -# be deduced by suitable manipulation of the __file__ attribute) or a -# filename. -# -# When we generate a coverage report we have to canonicalize every -# filename in the coverage dictionary just in case it refers to the -# module we are reporting on. It seems a shame to throw away this -# information so the data in the coverage dictionary is transferred to -# the 'cexecuted' dictionary under the canonical filenames. -# -# The coverage dictionary is called "c" and the trace function "t". The -# reason for these short names is that Python looks up variables by name -# at runtime and so execution time depends on the length of variables! -# In the bottleneck of this application it's appropriate to abbreviate -# names to increase speed. - -class StatementFindingAstVisitor(compiler.visitor.ASTVisitor): - def __init__(self, statements, excluded, suite_spots): - compiler.visitor.ASTVisitor.__init__(self) - self.statements = statements - self.excluded = excluded - self.suite_spots = suite_spots - self.excluding_suite = 0 - - def doRecursive(self, node): - self.recordNodeLine(node) - for n in node.getChildNodes(): - self.dispatch(n) - - visitStmt = visitModule = doRecursive - - def doCode(self, node): - if hasattr(node, 'decorators') and node.decorators: - self.dispatch(node.decorators) - self.recordAndDispatch(node.code) - else: - self.doSuite(node, node.code) - - visitFunction = visitClass = doCode - - def getFirstLine(self, node): - # Find the first line in the tree node. - lineno = node.lineno - for n in node.getChildNodes(): - f = self.getFirstLine(n) - if lineno and f: - lineno = min(lineno, f) - else: - lineno = lineno or f - return lineno - - def getLastLine(self, node): - # Find the first line in the tree node. - lineno = node.lineno - for n in node.getChildNodes(): - lineno = max(lineno, self.getLastLine(n)) - return lineno - - def doStatement(self, node): - self.recordLine(self.getFirstLine(node)) - - visitAssert = visitAssign = visitAssTuple = visitDiscard = visitPrint = \ - visitPrintnl = visitRaise = visitSubscript = visitDecorators = \ - doStatement - - def recordNodeLine(self, node): - return self.recordLine(node.lineno) - - def recordLine(self, lineno): - # Returns a bool, whether the line is included or excluded. - if lineno: - # Multi-line tests introducing suites have to get charged to their - # keyword. - if lineno in self.suite_spots: - lineno = self.suite_spots[lineno][0] - # If we're inside an exluded suite, record that this line was - # excluded. - if self.excluding_suite: - self.excluded[lineno] = 1 - return 0 - # If this line is excluded, or suite_spots maps this line to - # another line that is exlcuded, then we're excluded. - elif self.excluded.has_key(lineno) or \ - self.suite_spots.has_key(lineno) and \ - self.excluded.has_key(self.suite_spots[lineno][1]): - return 0 - # Otherwise, this is an executable line. - else: - self.statements[lineno] = 1 - return 1 - return 0 - - default = recordNodeLine - - def recordAndDispatch(self, node): - self.recordNodeLine(node) - self.dispatch(node) - - def doSuite(self, intro, body, exclude=0): - exsuite = self.excluding_suite - if exclude or (intro and not self.recordNodeLine(intro)): - self.excluding_suite = 1 - self.recordAndDispatch(body) - self.excluding_suite = exsuite - - def doPlainWordSuite(self, prevsuite, suite): - # Finding the exclude lines for else's is tricky, because they aren't - # present in the compiler parse tree. Look at the previous suite, - # and find its last line. If any line between there and the else's - # first line are excluded, then we exclude the else. - lastprev = self.getLastLine(prevsuite) - firstelse = self.getFirstLine(suite) - for l in range(lastprev+1, firstelse): - if self.suite_spots.has_key(l): - self.doSuite(None, suite, exclude=self.excluded.has_key(l)) - break - else: - self.doSuite(None, suite) - - def doElse(self, prevsuite, node): - if node.else_: - self.doPlainWordSuite(prevsuite, node.else_) - - def visitFor(self, node): - self.doSuite(node, node.body) - self.doElse(node.body, node) - - def visitIf(self, node): - # The first test has to be handled separately from the rest. - # The first test is credited to the line with the "if", but the others - # are credited to the line with the test for the elif. - self.doSuite(node, node.tests[0][1]) - for t, n in node.tests[1:]: - self.doSuite(t, n) - self.doElse(node.tests[-1][1], node) - - def visitWhile(self, node): - self.doSuite(node, node.body) - self.doElse(node.body, node) - - def visitTryExcept(self, node): - self.doSuite(node, node.body) - for i in range(len(node.handlers)): - a, b, h = node.handlers[i] - if not a: - # It's a plain "except:". Find the previous suite. - if i > 0: - prev = node.handlers[i-1][2] - else: - prev = node.body - self.doPlainWordSuite(prev, h) - else: - self.doSuite(a, h) - self.doElse(node.handlers[-1][2], node) - - def visitTryFinally(self, node): - self.doSuite(node, node.body) - self.doPlainWordSuite(node.body, node.final) - - def visitGlobal(self, node): - # "global" statements don't execute like others (they don't call the - # trace function), so don't record their line numbers. - pass - -the_coverage = None - -class CoverageException(Exception): pass - -class coverage: - # Name of the cache file (unless environment variable is set). - cache_default = ".coverage" - - # Environment variable naming the cache file. - cache_env = "COVERAGE_FILE" - - # A dictionary with an entry for (Python source file name, line number - # in that file) if that line has been executed. - c = {} - - # A map from canonical Python source file name to a dictionary in - # which there's an entry for each line number that has been - # executed. - cexecuted = {} - - # Cache of results of calling the analysis2() method, so that you can - # specify both -r and -a without doing double work. - analysis_cache = {} - - # Cache of results of calling the canonical_filename() method, to - # avoid duplicating work. - canonical_filename_cache = {} - - def __init__(self): - global the_coverage - if the_coverage: - raise CoverageException, "Only one coverage object allowed." - self.usecache = 1 - self.cache = None - self.exclude_re = '' - self.nesting = 0 - self.cstack = [] - self.xstack = [] - self.relative_dir = os.path.normcase(os.path.abspath(os.curdir)+os.path.sep) - - # t(f, x, y). This method is passed to sys.settrace as a trace function. - # See [van Rossum 2001-07-20b, 9.2] for an explanation of sys.settrace and - # the arguments and return value of the trace function. - # See [van Rossum 2001-07-20a, 3.2] for a description of frame and code - # objects. - - def t(self, f, w, a): #pragma: no cover - if w == 'line': - self.c[(f.f_code.co_filename, f.f_lineno)] = 1 - for c in self.cstack: - c[(f.f_code.co_filename, f.f_lineno)] = 1 - return self.t - - def help(self, error=None): - if error: - print error - print - print __doc__ - sys.exit(1) - - def command_line(self, argv, help=None): - import getopt - help = help or self.help - settings = {} - optmap = { - '-a': 'annotate', - '-c': 'collect', - '-d:': 'directory=', - '-e': 'erase', - '-h': 'help', - '-i': 'ignore-errors', - '-m': 'show-missing', - '-p': 'parallel-mode', - '-r': 'report', - '-x': 'execute', - '-o:': 'omit=', - } - short_opts = string.join(map(lambda o: o[1:], optmap.keys()), '') - long_opts = optmap.values() - options, args = getopt.getopt(argv, short_opts, long_opts) - for o, a in options: - if optmap.has_key(o): - settings[optmap[o]] = 1 - elif optmap.has_key(o + ':'): - settings[optmap[o + ':']] = a - elif o[2:] in long_opts: - settings[o[2:]] = 1 - elif o[2:] + '=' in long_opts: - settings[o[2:]+'='] = a - else: #pragma: no cover - pass # Can't get here, because getopt won't return anything unknown. - - if settings.get('help'): - help() - - for i in ['erase', 'execute']: - for j in ['annotate', 'report', 'collect']: - if settings.get(i) and settings.get(j): - help("You can't specify the '%s' and '%s' " - "options at the same time." % (i, j)) - - args_needed = (settings.get('execute') - or settings.get('annotate') - or settings.get('report')) - action = (settings.get('erase') - or settings.get('collect') - or args_needed) - if not action: - help("You must specify at least one of -e, -x, -c, -r, or -a.") - if not args_needed and args: - help("Unexpected arguments: %s" % " ".join(args)) - - self.get_ready(settings.get('parallel-mode')) - self.exclude('#pragma[: ]+[nN][oO] [cC][oO][vV][eE][rR]') - - if settings.get('erase'): - self.erase() - if settings.get('execute'): - if not args: - help("Nothing to do.") - sys.argv = args - self.start() - import __main__ - sys.path[0] = os.path.dirname(sys.argv[0]) - execfile(sys.argv[0], __main__.__dict__) - if settings.get('collect'): - self.collect() - if not args: - args = self.cexecuted.keys() - - ignore_errors = settings.get('ignore-errors') - show_missing = settings.get('show-missing') - directory = settings.get('directory=') - - omit = settings.get('omit=') - if omit is not None: - omit = omit.split(',') - else: - omit = [] - - if settings.get('report'): - self.report(args, show_missing, ignore_errors, omit_prefixes=omit) - if settings.get('annotate'): - self.annotate(args, directory, ignore_errors, omit_prefixes=omit) - - def use_cache(self, usecache, cache_file=None): - self.usecache = usecache - if cache_file and not self.cache: - self.cache_default = cache_file - - def get_ready(self, parallel_mode=False): - if self.usecache and not self.cache: - self.cache = os.environ.get(self.cache_env, self.cache_default) - if parallel_mode: - self.cache += "." + gethostname() + "." + str(os.getpid()) - self.restore() - self.analysis_cache = {} - - def start(self, parallel_mode=False): - self.get_ready(parallel_mode) - if self.nesting == 0: #pragma: no cover - sys.settrace(self.t) - if hasattr(threading, 'settrace'): - threading.settrace(self.t) - self.nesting += 1 - - def stop(self): - self.nesting -= 1 - if self.nesting == 0: #pragma: no cover - sys.settrace(None) - if hasattr(threading, 'settrace'): - threading.settrace(None) - - def erase(self): - self.c = {} - self.analysis_cache = {} - self.cexecuted = {} - if self.cache and os.path.exists(self.cache): - os.remove(self.cache) - self.exclude_re = "" - - def exclude(self, re): - if self.exclude_re: - self.exclude_re += "|" - self.exclude_re += "(" + re + ")" - - def begin_recursive(self): - self.cstack.append(self.c) - self.xstack.append(self.exclude_re) - - def end_recursive(self): - self.c = self.cstack.pop() - self.exclude_re = self.xstack.pop() - - # save(). Save coverage data to the coverage cache. - - def save(self): - if self.usecache and self.cache: - self.canonicalize_filenames() - cache = open(self.cache, 'wb') - import marshal - marshal.dump(self.cexecuted, cache) - cache.close() - - # restore(). Restore coverage data from the coverage cache (if it exists). - - def restore(self): - self.c = {} - self.cexecuted = {} - assert self.usecache - if os.path.exists(self.cache): - self.cexecuted = self.restore_file(self.cache) - - def restore_file(self, file_name): - try: - cache = open(file_name, 'rb') - import marshal - cexecuted = marshal.load(cache) - cache.close() - if isinstance(cexecuted, types.DictType): - return cexecuted - else: - return {} - except: - return {} - - # collect(). Collect data in multiple files produced by parallel mode - - def collect(self): - cache_dir, local = os.path.split(self.cache) - for file in os.listdir(cache_dir): - if not file.startswith(local): - continue - - full_path = os.path.join(cache_dir, file) - cexecuted = self.restore_file(full_path) - self.merge_data(cexecuted) - - def merge_data(self, new_data): - for file_name, file_data in new_data.items(): - if self.cexecuted.has_key(file_name): - self.merge_file_data(self.cexecuted[file_name], file_data) - else: - self.cexecuted[file_name] = file_data - - def merge_file_data(self, cache_data, new_data): - for line_number in new_data.keys(): - if not cache_data.has_key(line_number): - cache_data[line_number] = new_data[line_number] - - # canonical_filename(filename). Return a canonical filename for the - # file (that is, an absolute path with no redundant components and - # normalized case). See [GDR 2001-12-04b, 3.3]. - - def canonical_filename(self, filename): - if not self.canonical_filename_cache.has_key(filename): - f = filename - if os.path.isabs(f) and not os.path.exists(f): - f = os.path.basename(f) - if not os.path.isabs(f): - for path in [os.curdir] + sys.path: - g = os.path.join(path, f) - if os.path.exists(g): - f = g - break - cf = os.path.normcase(os.path.abspath(f)) - self.canonical_filename_cache[filename] = cf - return self.canonical_filename_cache[filename] - - # canonicalize_filenames(). Copy results from "c" to "cexecuted", - # canonicalizing filenames on the way. Clear the "c" map. - - def canonicalize_filenames(self): - for filename, lineno in self.c.keys(): - f = self.canonical_filename(filename) - if not self.cexecuted.has_key(f): - self.cexecuted[f] = {} - self.cexecuted[f][lineno] = 1 - self.c = {} - - # morf_filename(morf). Return the filename for a module or file. - - def morf_filename(self, morf): - if isinstance(morf, types.ModuleType): - if not hasattr(morf, '__file__'): - raise CoverageException, "Module has no __file__ attribute." - file = morf.__file__ - else: - file = morf - return self.canonical_filename(file) - - # analyze_morf(morf). Analyze the module or filename passed as - # the argument. If the source code can't be found, raise an error. - # Otherwise, return a tuple of (1) the canonical filename of the - # source code for the module, (2) a list of lines of statements - # in the source code, and (3) a list of lines of excluded statements. - - def analyze_morf(self, morf): - if self.analysis_cache.has_key(morf): - return self.analysis_cache[morf] - filename = self.morf_filename(morf) - ext = os.path.splitext(filename)[1] - if ext == '.pyc': - if not os.path.exists(filename[0:-1]): - raise CoverageException, ("No source for compiled code '%s'." - % filename) - filename = filename[0:-1] - elif ext != '.py': - raise CoverageException, "File '%s' not Python source." % filename - source = open(filename, 'r') - lines, excluded_lines = self.find_executable_statements( - source.read(), exclude=self.exclude_re - ) - source.close() - result = filename, lines, excluded_lines - self.analysis_cache[morf] = result - return result - - def get_suite_spots(self, tree, spots): - import symbol, token - for i in range(1, len(tree)): - if type(tree[i]) == type(()): - if tree[i][0] == symbol.suite: - # Found a suite, look back for the colon and keyword. - lineno_colon = lineno_word = None - for j in range(i-1, 0, -1): - if tree[j][0] == token.COLON: - lineno_colon = tree[j][2] - elif tree[j][0] == token.NAME: - if tree[j][1] == 'elif': - # Find the line number of the first non-terminal - # after the keyword. - t = tree[j+1] - while t and token.ISNONTERMINAL(t[0]): - t = t[1] - if t: - lineno_word = t[2] - else: - lineno_word = tree[j][2] - break - elif tree[j][0] == symbol.except_clause: - # "except" clauses look like: - # ('except_clause', ('NAME', 'except', lineno), ...) - if tree[j][1][0] == token.NAME: - lineno_word = tree[j][1][2] - break - if lineno_colon and lineno_word: - # Found colon and keyword, mark all the lines - # between the two with the two line numbers. - for l in range(lineno_word, lineno_colon+1): - spots[l] = (lineno_word, lineno_colon) - self.get_suite_spots(tree[i], spots) - - def find_executable_statements(self, text, exclude=None): - # Find lines which match an exclusion pattern. - excluded = {} - suite_spots = {} - if exclude: - reExclude = re.compile(exclude) - lines = text.split('\n') - for i in range(len(lines)): - if reExclude.search(lines[i]): - excluded[i+1] = 1 - - import parser - tree = parser.suite(text+'\n\n').totuple(1) - self.get_suite_spots(tree, suite_spots) - - # Use the compiler module to parse the text and find the executable - # statements. We add newlines to be impervious to final partial lines. - statements = {} - ast = compiler.parse(text+'\n\n') - visitor = StatementFindingAstVisitor(statements, excluded, suite_spots) - compiler.walk(ast, visitor, walker=visitor) - - lines = statements.keys() - lines.sort() - excluded_lines = excluded.keys() - excluded_lines.sort() - return lines, excluded_lines - - # format_lines(statements, lines). Format a list of line numbers - # for printing by coalescing groups of lines as long as the lines - # represent consecutive statements. This will coalesce even if - # there are gaps between statements, so if statements = - # [1,2,3,4,5,10,11,12,13,14] and lines = [1,2,5,10,11,13,14] then - # format_lines will return "1-2, 5-11, 13-14". - - def format_lines(self, statements, lines): - pairs = [] - i = 0 - j = 0 - start = None - pairs = [] - while i < len(statements) and j < len(lines): - if statements[i] == lines[j]: - if start == None: - start = lines[j] - end = lines[j] - j = j + 1 - elif start: - pairs.append((start, end)) - start = None - i = i + 1 - if start: - pairs.append((start, end)) - def stringify(pair): - start, end = pair - if start == end: - return "%d" % start - else: - return "%d-%d" % (start, end) - return string.join(map(stringify, pairs), ", ") - - # Backward compatibility with version 1. - def analysis(self, morf): - f, s, _, m, mf = self.analysis2(morf) - return f, s, m, mf - - def analysis2(self, morf): - filename, statements, excluded = self.analyze_morf(morf) - self.canonicalize_filenames() - if not self.cexecuted.has_key(filename): - self.cexecuted[filename] = {} - missing = [] - for line in statements: - if not self.cexecuted[filename].has_key(line): - missing.append(line) - return (filename, statements, excluded, missing, - self.format_lines(statements, missing)) - - def relative_filename(self, filename): - """ Convert filename to relative filename from self.relative_dir. - """ - return filename.replace(self.relative_dir, "") - - def morf_name(self, morf): - """ Return the name of morf as used in report. - """ - if isinstance(morf, types.ModuleType): - return morf.__name__ - else: - return self.relative_filename(os.path.splitext(morf)[0]) - - def filter_by_prefix(self, morfs, omit_prefixes): - """ Return list of morfs where the morf name does not begin - with any one of the omit_prefixes. - """ - filtered_morfs = [] - for morf in morfs: - for prefix in omit_prefixes: - if self.morf_name(morf).startswith(prefix): - break - else: - filtered_morfs.append(morf) - - return filtered_morfs - - def morf_name_compare(self, x, y): - return cmp(self.morf_name(x), self.morf_name(y)) - - def report(self, morfs, show_missing=1, ignore_errors=0, file=None, omit_prefixes=[]): - if not isinstance(morfs, types.ListType): - morfs = [morfs] - morfs = self.filter_by_prefix(morfs, omit_prefixes) - morfs.sort(self.morf_name_compare) - - max_name = max([5,] + map(len, map(self.morf_name, morfs))) - fmt_name = "%%- %ds " % max_name - fmt_err = fmt_name + "%s: %s" - header = fmt_name % "Name" + " Stmts Exec Cover" - fmt_coverage = fmt_name + "% 6d % 6d % 5d%%" - if show_missing: - header = header + " Missing" - fmt_coverage = fmt_coverage + " %s" - if not file: - file = sys.stdout - print >>file, header - print >>file, "-" * len(header) - total_statements = 0 - total_executed = 0 - for morf in morfs: - name = self.morf_name(morf) - try: - _, statements, _, missing, readable = self.analysis2(morf) - n = len(statements) - m = n - len(missing) - if n > 0: - pc = 100.0 * m / n - else: - pc = 100.0 - args = (name, n, m, pc) - if show_missing: - args = args + (readable,) - print >>file, fmt_coverage % args - total_statements = total_statements + n - total_executed = total_executed + m - except KeyboardInterrupt: #pragma: no cover - raise - except: - if not ignore_errors: - type, msg = sys.exc_info()[0:2] - print >>file, fmt_err % (name, type, msg) - if len(morfs) > 1: - print >>file, "-" * len(header) - if total_statements > 0: - pc = 100.0 * total_executed / total_statements - else: - pc = 100.0 - args = ("TOTAL", total_statements, total_executed, pc) - if show_missing: - args = args + ("",) - print >>file, fmt_coverage % args - - # annotate(morfs, ignore_errors). - - blank_re = re.compile(r"\s*(#|$)") - else_re = re.compile(r"\s*else\s*:\s*(#|$)") - - def annotate(self, morfs, directory=None, ignore_errors=0, omit_prefixes=[]): - morfs = self.filter_by_prefix(morfs, omit_prefixes) - for morf in morfs: - try: - filename, statements, excluded, missing, _ = self.analysis2(morf) - self.annotate_file(filename, statements, excluded, missing, directory) - except KeyboardInterrupt: - raise - except: - if not ignore_errors: - raise - - def annotate_file(self, filename, statements, excluded, missing, directory=None): - source = open(filename, 'r') - if directory: - dest_file = os.path.join(directory, - os.path.basename(filename) - + ',cover') - else: - dest_file = filename + ',cover' - dest = open(dest_file, 'w') - lineno = 0 - i = 0 - j = 0 - covered = 1 - while 1: - line = source.readline() - if line == '': - break - lineno = lineno + 1 - while i < len(statements) and statements[i] < lineno: - i = i + 1 - while j < len(missing) and missing[j] < lineno: - j = j + 1 - if i < len(statements) and statements[i] == lineno: - covered = j >= len(missing) or missing[j] > lineno - if self.blank_re.match(line): - dest.write(' ') - elif self.else_re.match(line): - # Special logic for lines containing only 'else:'. - # See [GDR 2001-12-04b, 3.2]. - if i >= len(statements) and j >= len(missing): - dest.write('! ') - elif i >= len(statements) or j >= len(missing): - dest.write('> ') - elif statements[i] == missing[j]: - dest.write('! ') - else: - dest.write('> ') - elif lineno in excluded: - dest.write('- ') - elif covered: - dest.write('> ') - else: - dest.write('! ') - dest.write(line) - source.close() - dest.close() - -# Singleton object. -the_coverage = coverage() - -# Module functions call methods in the singleton object. -def use_cache(*args, **kw): return the_coverage.use_cache(*args, **kw) -def start(*args, **kw): return the_coverage.start(*args, **kw) -def stop(*args, **kw): return the_coverage.stop(*args, **kw) -def erase(*args, **kw): return the_coverage.erase(*args, **kw) -def begin_recursive(*args, **kw): return the_coverage.begin_recursive(*args, **kw) -def end_recursive(*args, **kw): return the_coverage.end_recursive(*args, **kw) -def exclude(*args, **kw): return the_coverage.exclude(*args, **kw) -def analysis(*args, **kw): return the_coverage.analysis(*args, **kw) -def analysis2(*args, **kw): return the_coverage.analysis2(*args, **kw) -def report(*args, **kw): return the_coverage.report(*args, **kw) -def annotate(*args, **kw): return the_coverage.annotate(*args, **kw) -def annotate_file(*args, **kw): return the_coverage.annotate_file(*args, **kw) - -# Save coverage data when Python exits. (The atexit module wasn't -# introduced until Python 2.0, so use sys.exitfunc when it's not -# available.) -try: - import atexit - atexit.register(the_coverage.save) -except ImportError: - sys.exitfunc = the_coverage.save - -# Command-line interface. -if __name__ == '__main__': - the_coverage.command_line(sys.argv[1:]) - - -# A. REFERENCES -# -# [GDR 2001-12-04a] "Statement coverage for Python"; Gareth Rees; -# Ravenbrook Limited; 2001-12-04; -# <http://www.nedbatchelder.com/code/modules/rees-coverage.html>. -# -# [GDR 2001-12-04b] "Statement coverage for Python: design and -# analysis"; Gareth Rees; Ravenbrook Limited; 2001-12-04; -# <http://www.nedbatchelder.com/code/modules/rees-design.html>. -# -# [van Rossum 2001-07-20a] "Python Reference Manual (releae 2.1.1)"; -# Guide van Rossum; 2001-07-20; -# <http://www.python.org/doc/2.1.1/ref/ref.html>. -# -# [van Rossum 2001-07-20b] "Python Library Reference"; Guido van Rossum; -# 2001-07-20; <http://www.python.org/doc/2.1.1/lib/lib.html>. -# -# -# B. DOCUMENT HISTORY -# -# 2001-12-04 GDR Created. -# -# 2001-12-06 GDR Added command-line interface and source code -# annotation. -# -# 2001-12-09 GDR Moved design and interface to separate documents. -# -# 2001-12-10 GDR Open cache file as binary on Windows. Allow -# simultaneous -e and -x, or -a and -r. -# -# 2001-12-12 GDR Added command-line help. Cache analysis so that it -# only needs to be done once when you specify -a and -r. -# -# 2001-12-13 GDR Improved speed while recording. Portable between -# Python 1.5.2 and 2.1.1. -# -# 2002-01-03 GDR Module-level functions work correctly. -# -# 2002-01-07 GDR Update sys.path when running a file with the -x option, -# so that it matches the value the program would get if it were run on -# its own. -# -# 2004-12-12 NMB Significant code changes. -# - Finding executable statements has been rewritten so that docstrings and -# other quirks of Python execution aren't mistakenly identified as missing -# lines. -# - Lines can be excluded from consideration, even entire suites of lines. -# - The filesystem cache of covered lines can be disabled programmatically. -# - Modernized the code. -# -# 2004-12-14 NMB Minor tweaks. Return 'analysis' to its original behavior -# and add 'analysis2'. Add a global for 'annotate', and factor it, adding -# 'annotate_file'. -# -# 2004-12-31 NMB Allow for keyword arguments in the module global functions. -# Thanks, Allen. -# -# 2005-12-02 NMB Call threading.settrace so that all threads are measured. -# Thanks Martin Fuzzey. Add a file argument to report so that reports can be -# captured to a different destination. -# -# 2005-12-03 NMB coverage.py can now measure itself. -# -# 2005-12-04 NMB Adapted Greg Rogers' patch for using relative filenames, -# and sorting and omitting files to report on. -# -# 2006-07-23 NMB Applied Joseph Tate's patch for function decorators. -# -# 2006-08-21 NMB Applied Sigve Tjora and Mark van der Wal's fixes for argument -# handling. -# -# 2006-08-22 NMB Applied Geoff Bache's parallel mode patch. -# -# 2006-08-23 NMB Refactorings to improve testability. Fixes to command-line -# logic for parallel mode and collect. - -# C. COPYRIGHT AND LICENCE -# -# Copyright 2001 Gareth Rees. All rights reserved. -# Copyright 2004-2006 Ned Batchelder. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the -# distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS -# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR -# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH -# DAMAGE. -# -# $Id: coverage.py 47 2006-08-24 01:08:48Z Ned $ @@ -93,22 +93,22 @@ case second `m'. Any other spelling is incorrect.""", 'console_scripts' : list(scripts), }, install_requires = [ + 'enum34', 'flufl.bounce', - 'flufl.enum', 'flufl.i18n', 'flufl.lock', 'httplib2', 'lazr.config', 'lazr.smtptest', 'mock', + 'nose2', 'passlib', 'restish', 'storm', - 'zc.buildout', 'zope.component', 'zope.configuration', 'zope.event', 'zope.interface', - 'zope.testing<4', ], + test_suite = 'nose2.collector.collector', ) diff --git a/src/mailman/__init__.py b/src/mailman/__init__.py index 8690e77bd..87a2a2100 100644 --- a/src/mailman/__init__.py +++ b/src/mailman/__init__.py @@ -44,7 +44,7 @@ except ImportError: # # Do *not* do this if we're building the documentation. if 'build_sphinx' not in sys.argv: - if sys.argv[0].split(os.sep)[-1] == 'test': + if any('nose2' in arg for arg in sys.argv): from mailman.testing.i18n import initialize else: from mailman.core.i18n import initialize diff --git a/src/mailman/app/docs/hooks.rst b/src/mailman/app/docs/hooks.rst index 7e214f13f..a29c7ee10 100644 --- a/src/mailman/app/docs/hooks.rst +++ b/src/mailman/app/docs/hooks.rst @@ -52,8 +52,9 @@ script that will produce no output to force the hooks to run. >>> import subprocess >>> from mailman.testing.layers import ConfigLayer >>> def call(): + ... exe = os.path.join(os.path.dirname(sys.executable), 'mailman') ... proc = subprocess.Popen( - ... 'bin/mailman lists --domain ignore -q'.split(), + ... [exe, 'lists', '--domain', 'ignore', '-q'], ... cwd=ConfigLayer.root_directory, ... env=dict(MAILMAN_CONFIG_FILE=config_path, ... PYTHONPATH=config_directory), diff --git a/src/mailman/app/moderator.py b/src/mailman/app/moderator.py index d8603366a..59fcb0976 100644 --- a/src/mailman/app/moderator.py +++ b/src/mailman/app/moderator.py @@ -246,7 +246,7 @@ def handle_subscription(mlist, id, action, comment=None): lang=getUtility(ILanguageManager)[data['language']]) elif action is Action.accept: key, data = requestdb.get_request(id) - delivery_mode = DeliveryMode(data['delivery_mode']) + delivery_mode = DeliveryMode[data['delivery_mode']] address = data['address'] display_name = data['display_name'] language = getUtility(ILanguageManager)[data['language']] diff --git a/src/mailman/app/subscriptions.py b/src/mailman/app/subscriptions.py index 23e09ccc7..0995202b6 100644 --- a/src/mailman/app/subscriptions.py +++ b/src/mailman/app/subscriptions.py @@ -54,7 +54,7 @@ def _membership_sort_key(member): The members are sorted first by unique list id, then by subscribed email address, then by role. """ - return (member.list_id, member.address.email, int(member.role)) + return (member.list_id, member.address.email, member.role.value) diff --git a/src/mailman/archiving/docs/__init__.py b/src/mailman/archiving/docs/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/mailman/archiving/docs/__init__.py diff --git a/src/mailman/bin/master.py b/src/mailman/bin/master.py index 9e1bbd6c5..65737ad75 100644 --- a/src/mailman/bin/master.py +++ b/src/mailman/bin/master.py @@ -34,7 +34,7 @@ import socket import logging from datetime import timedelta -from flufl.enum import Enum +from enum import Enum from flufl.lock import Lock, NotLockedError, TimeOutError from lazr.config import as_boolean diff --git a/src/mailman/commands/cli_conf.py b/src/mailman/commands/cli_conf.py index 1f8095d92..dfa741e3a 100644 --- a/src/mailman/commands/cli_conf.py +++ b/src/mailman/commands/cli_conf.py @@ -65,10 +65,20 @@ class Conf: key-values pair from any section matching the given key will be displayed. """)) + command_parser.add_argument( + '-t', '--sort', + default=False, action='store_true', + help=_('Sort the output by sections and keys.')) def _get_value(self, section, key): return getattr(getattr(config, section), key) + def _sections(self, sort_p): + sections = config.schema._section_schemas + if sort_p: + sections = sorted(sections) + return sections + def _print_full_syntax(self, section, key, value, output): print('[{}] {}: {}'.format(section, key, value), file=output) @@ -78,8 +88,10 @@ class Conf: def _show_section_error(self, section): self.parser.error('No such section: {}'.format(section)) - def _print_values_for_section(self, section, output): + def _print_values_for_section(self, section, output, sort_p): current_section = getattr(config, section) + if sort_p: + current_section = sorted(current_section) for key in current_section: self._print_full_syntax(section, key, self._get_value(section, key), output) @@ -94,6 +106,7 @@ class Conf: # Process the command, ignoring the closing of the output file. section = args.section key = args.key + sort_p = args.sort # Case 1: Both section and key are given, so we can directly look up # the value. if section is not None and key is not None: @@ -106,12 +119,12 @@ class Conf: # Case 2: Section is given, key is not given. elif section is not None and key is None: if self._section_exists(section): - self._print_values_for_section(section, output) + self._print_values_for_section(section, output, sort_p) else: self._show_section_error(section) # Case 3: Section is not given, key is given. elif section is None and key is not None: - for current_section in config.schema._section_schemas: + for current_section in self._sections(sort_p): # We have to ensure that the current section actually exists # and that it contains the given key. if (self._section_exists(current_section) and @@ -124,12 +137,13 @@ class Conf: # Case 4: Neither section nor key are given, just display all the # sections and their corresponding key/value pairs. elif section is None and key is None: - for current_section in config.schema._section_schemas: + for current_section in self._sections(sort_p): # However, we have to make sure that the current sections and # key which are being looked up actually exist before trying # to print them. if self._section_exists(current_section): - self._print_values_for_section(current_section, output) + self._print_values_for_section( + current_section, output, sort_p) def process(self, args): """See `ICLISubCommand`.""" diff --git a/src/mailman/commands/cli_members.py b/src/mailman/commands/cli_members.py index 7e4b5ec88..37df55cd4 100644 --- a/src/mailman/commands/cli_members.py +++ b/src/mailman/commands/cli_members.py @@ -130,7 +130,7 @@ class Members: DeliveryMode.mime_digests, DeliveryMode.summary_digests] elif args.digest is not None: - digest_types = [DeliveryMode(args.digest + '_digests')] + digest_types = [DeliveryMode[args.digest + '_digests']] else: # Don't filter on digest type. pass @@ -140,7 +140,7 @@ class Members: elif args.nomail == 'byadmin': status_types = [DeliveryStatus.by_moderator] elif args.nomail.startswith('by'): - status_types = [DeliveryStatus('by_' + args.nomail[2:])] + status_types = [DeliveryStatus['by_' + args.nomail[2:]]] elif args.nomail == 'enabled': status_types = [DeliveryStatus.enabled] elif args.nomail == 'unknown': diff --git a/src/mailman/commands/cli_status.py b/src/mailman/commands/cli_status.py index 1f4cc552a..8a5cf9bde 100644 --- a/src/mailman/commands/cli_status.py +++ b/src/mailman/commands/cli_status.py @@ -64,4 +64,4 @@ class Status: message = _('GNU Mailman is in an unexpected state ' '($hostname != $fqdn_name)') print(message) - return int(status) + return status.value diff --git a/src/mailman/commands/docs/__init__.py b/src/mailman/commands/docs/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/mailman/commands/docs/__init__.py diff --git a/src/mailman/commands/docs/conf.rst b/src/mailman/commands/docs/conf.rst index 6e458fb54..7b8529ac3 100644 --- a/src/mailman/commands/docs/conf.rst +++ b/src/mailman/commands/docs/conf.rst @@ -14,6 +14,7 @@ a specific key-value pair, or several key-value pairs. ... key = None ... section = None ... output = None + ... sort = False >>> from mailman.commands.cli_conf import Conf >>> command = Conf() @@ -64,5 +65,16 @@ If you specify both a section and a key, you will get the corresponding value. >>> command.process(FakeArgs) noreply@example.com +You can also sort the output. The output is first sorted by section, then by +key. + + >>> FakeArgs.key = None + >>> FakeArgs.section = 'shell' + >>> FakeArgs.sort = True + >>> command.process(FakeArgs) + [shell] banner: Welcome to the GNU Mailman shell + [shell] prompt: >>> + [shell] use_ipython: no + .. _`Postfix command postconf(1)`: http://www.postfix.org/postconf.1.html diff --git a/src/mailman/commands/tests/test_conf.py b/src/mailman/commands/tests/test_conf.py index bca7fe72f..307151c74 100644 --- a/src/mailman/commands/tests/test_conf.py +++ b/src/mailman/commands/tests/test_conf.py @@ -31,6 +31,7 @@ import mock import tempfile import unittest +from StringIO import StringIO from mailman.commands.cli_conf import Conf from mailman.testing.layers import ConfigLayer @@ -40,6 +41,7 @@ class FakeArgs: section = None key = None output = None + sort = False class FakeParser: @@ -99,3 +101,13 @@ class TestConf(unittest.TestCase): finally: os.remove(filename) self.assertEqual(contents, 'no\n') + + def test_sort_by_section(self): + self.args.output = '-' + self.args.sort = True + output = StringIO() + with mock.patch('sys.stdout', output): + self.command.process(self.args) + last_line = '' + for line in output.getvalue().splitlines(): + self.assertTrue(line > last_line) diff --git a/src/mailman/database/schema/mm_20120407000000.py b/src/mailman/database/schema/mm_20120407000000.py index 463635fd9..c1045fa91 100644 --- a/src/mailman/database/schema/mm_20120407000000.py +++ b/src/mailman/database/schema/mm_20120407000000.py @@ -67,11 +67,11 @@ def upgrade(database, store, version, module_path): def archive_policy(archive, archive_private): """Convert archive and archive_private to archive_policy.""" if archive == 0: - return int(ArchivePolicy.never) + return ArchivePolicy.never.value elif archive_private == 1: - return int(ArchivePolicy.private) + return ArchivePolicy.private.value else: - return int(ArchivePolicy.public) + return ArchivePolicy.public.value diff --git a/src/mailman/database/types.py b/src/mailman/database/types.py index 42f64a640..86d504ca3 100644 --- a/src/mailman/database/types.py +++ b/src/mailman/database/types.py @@ -32,7 +32,7 @@ from storm.variables import Variable class _EnumVariable(Variable): - """Storm variable for supporting flufl.enum.Enum types. + """Storm variable for supporting enum types. To use this, make the database column a INTEGER. """ @@ -46,18 +46,18 @@ class _EnumVariable(Variable): return None if not from_db: return value - return self._enum[value] + return self._enum(value) def parse_get(self, value, to_db): if value is None: return None if not to_db: return value - return int(value) + return value.value class Enum(SimpleProperty): - """Custom Enum type for Storm supporting flufl.enum.Enums.""" + """Custom type for Storm supporting enums.""" variable_class = _EnumVariable diff --git a/src/mailman/docs/8-miles-high.rst b/src/mailman/docs/8-miles-high.rst index d068ebd73..85b186fc5 100644 --- a/src/mailman/docs/8-miles-high.rst +++ b/src/mailman/docs/8-miles-high.rst @@ -37,10 +37,12 @@ processing queues. node [shape=box, color=lightblue, style=filled]; msg [shape=ellipse, color=black, fillcolor=white]; lmtpd [label="LMTP\nSERVER"]; + rts [label="Return\nto Sender"]; msg -> MTA [label="SMTP"]; MTA -> lmtpd [label="LMTP"]; lmtpd -> MTA [label="reject"]; lmtpd -> IN -> PIPELINE [label=".pck"]; + IN -> rts; lmtpd -> BOUNCES [label=".pck"]; lmtpd -> COMMAND [label=".pck"]; } @@ -49,7 +51,9 @@ The `in` queue is processed by *filter chains* (explained below) to determine whether the post (or administrative request) will be processed. If not allowed, the message pickle is discarded, rejected (returned to sender), or held (saved for moderator approval -- not shown). Otherwise the message is -added to the `pipeline` (i.e. posting) queue. +added to the `pipeline` (i.e. posting) queue. (Note that rejecting at this +stage is *not* equivalent to rejecting during LMTP processing. This issue is +currently unresolved.) Each of the `command`, `bounce`, and `pipeline` queues is processed by a *pipeline of handlers* as in Mailman 2's pipeline. (Some functions such as @@ -60,7 +64,8 @@ Handlers may copy messages to other queues (*e.g.*, `archive`), and eventually posted messages for distribution to the list membership end up in the `out` queue for injection into the MTA. -The `virgin` queue is a special queue for messages created by Mailman. +The `virgin` queue (not depicted above) is a special queue for messages created +by Mailman. .. graphviz:: @@ -97,12 +102,13 @@ The default set of rules looks something like this: subgraph rules { rankdir=TB; node [shape=record]; - approved [label="<in> approved | { <no> | <yes> }"]; - emergency [label="<in> emergency | { <no> | <yes> }"]; - loop [label="<in> loop | { <no> | <yes> }"]; - modmember [label="<in> member\nmoderated | { <no> | <yes> }"]; - administrivia [group="0", label="<in> administrivia | <always> "]; - maxsize [label="<in> max\ size | {<in> no | <yes>}"]; + approved [label="<in> approved | { <no> no | <yes> }"]; + emergency [label="<in> emergency | { <no> no | <yes> }"]; + loop [label="<in> loop | { <no> no | <yes> }"]; + modmember [label="<in> member\nmoderated | { <no> no | <yes> }"]; + administrivia [group="0", + label="<in> administrivia | { <no> no | <yes> }"]; + maxsize [label="<in> max\ size | {<no> no | <yes>}"]; any [label="<in> any | {<no> | <yes>}"]; truth [label="<in> truth | <always>"]; @@ -114,6 +120,7 @@ The default set of rules looks something like this: DISCARD [shape=invhouse, color=black, style=solid]; MODERATION [color=wheat]; HOLD [color=wheat]; + action [color=wheat]; } { PIPELINE [shape=box, style=filled, color=cyan]; } @@ -130,7 +137,8 @@ The default set of rules looks something like this: modmember:no -> administrivia:in; modmember:yes -> MODERATION; - administrivia:always -> maxsize:in; + administrivia:no -> maxsize:in; + administrivia:yes -> action; maxsize:no -> any:in; maxsize:yes -> MODERATION; @@ -145,7 +153,7 @@ The default set of rules looks something like this: Configuration ============= -Uses `lazr.config`_, essentially an "ini"-style configuration format. +Mailman 3 uses `lazr.config`_, essentially an "ini"-style configuration format. Each Runner's configuration object knows whether it should be started when the Mailman daemon starts, and what queue the Runner manages. diff --git a/src/mailman/docs/INTRODUCTION.rst b/src/mailman/docs/INTRODUCTION.rst index 1ad71cf16..e7128e756 100644 --- a/src/mailman/docs/INTRODUCTION.rst +++ b/src/mailman/docs/INTRODUCTION.rst @@ -82,7 +82,7 @@ lists and archives, etc., are available at: Requirements ============ -Mailman 3.0 requires `Python 2.7`_ or newer. +Mailman 3.0 requires `Python 2.7`_. .. _`GNU Mailman`: http://www.list.org diff --git a/src/mailman/docs/MTA.rst b/src/mailman/docs/MTA.rst index aa53981f8..7165c30b9 100644 --- a/src/mailman/docs/MTA.rst +++ b/src/mailman/docs/MTA.rst @@ -2,32 +2,73 @@ Hooking up your mail server =========================== -Mailman needs to be hooked up to your mail server (a.k.a. *mail transport -agent* or *MTA*) both to accept incoming mail and to deliver outgoing mail. -Mailman itself never delivers messages to the end user; it lets its immediate -upstream mail server do that. +Mailman needs to communicate with your *MTA* (*mail transport agent* +or *mail server*, the software which handles sending mail across the +Internet), both to accept incoming mail and to deliver outgoing mail. +Mailman itself never delivers messages to the end user. It sends them +to its immediate upstream MTA, which delivers them. In the same way, +Mailman never receives mail directly. Mail from outside always comes +via the MTA. -The preferred way to allow Mailman to accept incoming messages from your mail -server is to use the `Local Mail Transfer Protocol`_ (LMTP_) interface. Most -open source mail server support LMTP for local delivery, and this is much more -efficient than spawning a process just to do the delivery. +Mailman accepts incoming messages from the MTA using the `Local Mail +Transfer Protocol`_ (LMTP_) interface. Mailman can use other incoming +transports, but LMTP is much more efficient than spawning a process +just to do the delivery. Most open source MTAs support LMTP for local +delivery. If yours doesn't, and you need to use a different +interface, please ask on the `mailing list or on IRC`_. -Your mail server should also accept `Simple Mail Transfer Protocol`_ (SMTP_) -connections from Mailman, for all outgoing messages. +Mailman passes all outgoing messages to the MTA using the `Simple Mail +Transfer Protocol`_ (SMTP_). -The specific instructions for hooking your mail server up to Mailman differs -depending on which mail server you're using. The following are instructions -for the popular open source mail servers. +Cooperation between Mailman and the MTA requires some configuration of +both. MTA configuration differs for each of the available MTAs, and +there is a section for each one. Instructions for Postfix are given +below. We would really appreciate contributions of configurations for +Exim and Sendmail, and welcome information about other popular open +source mail servers. -Note that Mailman provides lots of configuration variables that you can use to -tweak performance for your operating environment. See the -``src/mailman/config/schema.cfg`` file for details. +Configuring Mailman to communicate with the MTA is straightforward, +and basically the same for all MTAs. In your ``mailman.cfg`` file, +add (or edit) a section like the following:: + + [mta] + incoming: mailman.mta.postfix.LMTP + outgoing: mailman.mta.deliver.deliver + lmtp_host: 127.0.0.1 + lmtp_port: 8024 + smtp_host: localhost + smtp_port: 25 +This configuration is for a system where Mailman and the MTA are on +the same host. -Exim -==== +The ``incoming`` and ``outgoing`` parameters identify the Python +objects used to communicate with the MTA. The ``deliver`` module used +in ``outgoing`` is pretty standard across all MTAs. The ``postfix`` +module in ``incoming`` is specific to Postfix. See the section for +your MTA below for details on these parameters. -Contributions are welcome! +``lmtp_host`` and ``lmtp_port`` are parameters which are used by +Mailman, but also will be passed to the MTA to identify the Mailman +host. The "same host" case is special; some MTAs (including Postfix) +do not recognize "localhost", and need the numerical IP address. If +they are on different hosts, ``lmtp_host`` should be set to the domain +name or IP address of the Mailman host. ``lmtp_port`` is fairly +arbitrary (there is no standard port for LMTP). Use any port +convenient for your site. "8024" is as good as any, unless another +service is using it. + +``smtp_host`` and ``smtp_port`` are parameters used to identify the +MTA to Mailman. If the MTA and Mailman are on separate hosts, +``smtp_host`` should be set to the domain name or IP address of the +MTA host. ``smtp_port`` will almost always be 25, which is the +standard port for SMTP. (Some special site configurations set it to a +different port. If you need this, you probably already know that, +know why, and what to do, too!) + +Mailman also provides many other configuration variables that you can +use to tweak performance for your operating environment. See the +``src/mailman/config/schema.cfg`` file for details. Postfix @@ -131,12 +172,19 @@ the Postfix documentation at: .. _`mydestination`: http://www.postfix.org/postconf.5.html#mydestination +Exim +==== + +Contributions are welcome! + + Sendmail ======== Contributions are welcome! +.. _`mailing list or on IRC`: START.html#contact-us .. _`Local Mail Transfer Protocol`: http://en.wikipedia.org/wiki/Local_Mail_Transfer_Protocol .. _LMTP: http://www.faqs.org/rfcs/rfc2033.html diff --git a/src/mailman/docs/NEWS.rst b/src/mailman/docs/NEWS.rst index 8faeb3074..6572c9a7b 100644 --- a/src/mailman/docs/NEWS.rst +++ b/src/mailman/docs/NEWS.rst @@ -12,12 +12,27 @@ Here is a history of user visible changes to Mailman. =============================== (2013-XX-XX) +Development +----------- + * Mailman 3 no longer uses ``zc.buildout`` and tests are now run by the + ``nose2`` test runner. See ``src/mailman/docs/START.rst`` for details on + how to build Mailman and run the test suite. + * Use the ``enum34`` package instead of ``flufl.enum``. + REST ---- * Add ``reply_to_address`` and ``first_strip_reply_to`` as writable attributes of a mailing list's configuration. (LP: #1157881) * Support pagination of some large collections (lists, users, members). Given by Florian Fuchs. (LP: #1156529) + * Expose ``hide_address`` to the ``.../preferences`` REST API. Contributed + by Sneha Priscilla. + +Commands +-------- + * `mailman conf` now has a `-t/--sort` flag which sorts the output by section + and then key. Contributed by Karl-Aksel Puulmann and David Soto + (LP: 1162492) Configuration ------------- diff --git a/src/mailman/docs/START.rst b/src/mailman/docs/START.rst index 85b869c2d..a50deabf9 100644 --- a/src/mailman/docs/START.rst +++ b/src/mailman/docs/START.rst @@ -25,58 +25,64 @@ search, and retrieval of archived messages to a separate application (a simple implementation is provided). The web interface (known as `Postorius`_) and archiver (known as `Hyperkitty`_) are in separate development. -Contributions are welcome. Please submit bug reports on the Mailman bug -tracker at https://bugs.launchpad.net/mailman though you will currently need -to have a login on Launchpad to do so. You can also send email to the -mailman-developers@python.org mailing list. + +Contact Us +========== + +Contributions of code, problem reports, and feature requests are welcome. +Please submit bug reports on the Mailman bug tracker at +https://bugs.launchpad.net/mailman (you need to have a login on Launchpad to +do so). You can also send email to the mailman-developers@python.org mailing +list, or ask on IRC channel ``#mailman`` on Freenode. Requirements ============ Python 2.7 is required. It can either be the default 'python' on your -``$PATH`` or it can be accessible via the ``python2.7`` binary. -If your operating system does not include Python, see http://www.python.org -downloading and installing it from source. Python 3 is not yet supported. - -In this documentation, a bare ``python`` refers to the Python executable used -to invoke ``bootstrap.py``. +``$PATH`` or it can be accessible via the ``python2.7`` binary. If +your operating system does not include Python, see http://www.python.org +for information about downloading installers (where available) and +installing it from source (when necessary or preferred). Python 3 is +not yet supported. -Mailman 3 is now based on the `zc.buildout`_ infrastructure, which greatly -simplifies testing Mailman. Buildout is not required for installation. - -During the beta program, you may need some additional dependencies, such as a -C compiler and the Python development headers and libraries. You will need an -internet connection. +You may need some additional dependencies, which are either available from +your OS vendor, or can be downloaded automatically from the `Python +Cheeseshop`_. Building Mailman 3 ================== -We provide several recipes for building Mailman. All should generally work, -but some may provide a better experience for developing Mailman versus -deploying Mailman. - +To build Mailman for development purposes, you will create a virtual +environment. You need to have the `virtualenv`_ program installed. Building for development ------------------------ -The best way to build Mailman for development is to use the `zc.buildout`_ -tools. This will download all Mailman dependencies from the `Python -Cheeseshop`_. The dependencies will get installed locally, but isolated from -your system Python. Here are the commands to build Mailman for development:: +First, create a virtual environment. By default ``virtualenv`` uses the +``python`` executable it finds first on your ``$PATH``. Make sure this is +Python 2.7. The directory you install the virtualenv into is up to you, but +for purposes of this document, we'll install it into ``/tmp/py27``:: + + % virtualenv --system-site-packages /tmp/py27 - % python bootstrap.py - % bin/buildout +Now, activate the virtual environment and set it up for development:: + + % source /tmp/py27/bin/activate + % python setup.py develop Sit back and have some Kombucha while you wait for everything to download and install. Now you can run the test suite via:: - % bin/test -vv + % nose2 -v + +You should see no failures. You can also run a subset of the full test suite +by filter tests on the module or test name using the ``-P`` option:: -You should see no failures. + % nose2 -v -P user Build the online docs by running:: @@ -91,21 +97,6 @@ doctests by looking in all the 'doc' directories under the 'mailman' package. Doctests are documentation first, so they should give you a pretty good idea how various components of Mailman 3 work. - -Building for deployment using virtualenv ----------------------------------------- - -`virtualenv`_ is a way to create isolated Python environments. You can use -virtualenv as a way to deploy Mailman without installing it into your system -Python. There are lots of ways to use virtualenv, but as described here, it -will be default use any dependencies which are already installed in your -system, downloading from the Cheeseshop only those which are missing. Here -are the steps to install Mailman using virtualenv:: - - $ virtualenv --system-site-packages /path/to/your/installation - $ source /path/to/your/installation/bin/activate - $ python setup.py install - Once everything is downloaded and installed, you can initialize Mailman and get a display of the basic configuration settings by running:: @@ -149,8 +140,9 @@ Try ``bin/mailman --help`` for more details. You can use the commands ``bin/mailman start`` to start the runner subprocess daemons, and of course ``bin/mailman stop`` to stop them. -Postorius is being developed as a separate, Django-based project. For now, -all configuration happens via the command line and REST API. +Postorius, a web UI for administration and subscriber settings, is being +developed as a separate, Django-based project. For now, the most flexible +means of configuration is via the command line and REST API. Mailman Web UI @@ -167,16 +159,16 @@ Postorius was prototyped at the `Pycon 2012 sprint`_, so it is "very alpha" as of Mailman 3 beta 1, and comes in several components. In particular, it requires a `Django`_ installation, and Bazaar checkouts of the `REST client module`_ and `Postorius`_ itself. Building it is fairly straightforward, -however, given Florian Fuchs' `Five Minute Guide` from his `blog post`_ on the +based on Florian Fuchs' `Five Minute Guide` from his `blog post`_ on the Mailman wiki. (Check the `blog post`_ for the most recent version!) The Archiver ------------ -In Mailman 3, the archivers are decoupled from the core engine. It is useful -to provide a simple, standard interface for third-party archiving tools and -services. For this reason, Mailman 3 defines a formal interface to insert +In Mailman 3, the archivers are decoupled from the core engine. Instead, +Mailman 3 provides a simple, standard interface for third-party archiving tools +and services. For this reason, Mailman 3 defines a formal interface to insert messages into any of a number of configured archivers, using whatever protocol is appropriate for that archiver. Summary, search, and retrieval of archived posts are handled by a separate application. @@ -190,7 +182,6 @@ email application that speaks LMTP or SMTP will be able to use Hyperkitty. A `five minute guide to Hyperkitty`_ is based on Toshio Kuratomi's README. -.. _`zc.buildout`: http://pypi.python.org/pypi/zc.buildout .. _`Postorius`: https://launchpad.net/postorius .. _`Hyperkitty`: https://launchpad.net/hyperkitty .. _`Django`: http://djangoproject.org/ diff --git a/src/mailman/handlers/docs/__init__.py b/src/mailman/handlers/docs/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/mailman/handlers/docs/__init__.py diff --git a/src/mailman/interfaces/action.py b/src/mailman/interfaces/action.py index 287d253f9..22a65d5e1 100644 --- a/src/mailman/interfaces/action.py +++ b/src/mailman/interfaces/action.py @@ -24,7 +24,7 @@ __all__ = [ ] -from flufl.enum import Enum +from enum import Enum diff --git a/src/mailman/interfaces/archiver.py b/src/mailman/interfaces/archiver.py index 2182798ad..5f074503e 100644 --- a/src/mailman/interfaces/archiver.py +++ b/src/mailman/interfaces/archiver.py @@ -27,7 +27,7 @@ __all__ = [ ] -from flufl.enum import Enum +from enum import Enum from zope.interface import Interface, Attribute diff --git a/src/mailman/interfaces/autorespond.py b/src/mailman/interfaces/autorespond.py index 7c66d984f..acad717f1 100644 --- a/src/mailman/interfaces/autorespond.py +++ b/src/mailman/interfaces/autorespond.py @@ -30,7 +30,7 @@ __all__ = [ from datetime import timedelta -from flufl.enum import Enum +from enum import Enum from zope.interface import Interface, Attribute ALWAYS_REPLY = timedelta() diff --git a/src/mailman/interfaces/bounce.py b/src/mailman/interfaces/bounce.py index 1aa1e22b6..ff7a47732 100644 --- a/src/mailman/interfaces/bounce.py +++ b/src/mailman/interfaces/bounce.py @@ -28,7 +28,7 @@ __all__ = [ ] -from flufl.enum import Enum +from enum import Enum from zope.interface import Attribute, Interface diff --git a/src/mailman/interfaces/chain.py b/src/mailman/interfaces/chain.py index b4fd98bb2..9fffc2760 100644 --- a/src/mailman/interfaces/chain.py +++ b/src/mailman/interfaces/chain.py @@ -35,7 +35,7 @@ __all__ = [ ] -from flufl.enum import Enum +from enum import Enum from zope.interface import Interface, Attribute diff --git a/src/mailman/interfaces/command.py b/src/mailman/interfaces/command.py index 42a859f36..18a8024a8 100644 --- a/src/mailman/interfaces/command.py +++ b/src/mailman/interfaces/command.py @@ -28,7 +28,7 @@ __all__ = [ ] -from flufl.enum import Enum +from enum import Enum from zope.interface import Interface, Attribute diff --git a/src/mailman/interfaces/digests.py b/src/mailman/interfaces/digests.py index c00758f7a..416e18f13 100644 --- a/src/mailman/interfaces/digests.py +++ b/src/mailman/interfaces/digests.py @@ -25,7 +25,7 @@ __all__ = [ ] -from flufl.enum import Enum +from enum import Enum from zope.interface import Interface, Attribute diff --git a/src/mailman/interfaces/mailinglist.py b/src/mailman/interfaces/mailinglist.py index eb2a931f4..7beaf9c46 100644 --- a/src/mailman/interfaces/mailinglist.py +++ b/src/mailman/interfaces/mailinglist.py @@ -29,7 +29,7 @@ __all__ = [ ] -from flufl.enum import Enum +from enum import Enum from zope.interface import Interface, Attribute from mailman.interfaces.member import MemberRole diff --git a/src/mailman/interfaces/member.py b/src/mailman/interfaces/member.py index ee609a0d9..b561a7571 100644 --- a/src/mailman/interfaces/member.py +++ b/src/mailman/interfaces/member.py @@ -36,7 +36,7 @@ __all__ = [ ] -from flufl.enum import Enum +from enum import Enum from zope.interface import Interface, Attribute from mailman.core.errors import MailmanError diff --git a/src/mailman/interfaces/mime.py b/src/mailman/interfaces/mime.py index 629fc776b..549e6c3db 100644 --- a/src/mailman/interfaces/mime.py +++ b/src/mailman/interfaces/mime.py @@ -27,7 +27,7 @@ __all__ = [ ] -from flufl.enum import Enum +from enum import Enum from zope.interface import Interface, Attribute diff --git a/src/mailman/interfaces/nntp.py b/src/mailman/interfaces/nntp.py index 2246fff6e..03be8f11d 100644 --- a/src/mailman/interfaces/nntp.py +++ b/src/mailman/interfaces/nntp.py @@ -25,7 +25,7 @@ __all__ = [ ] -from flufl.enum import Enum +from enum import Enum diff --git a/src/mailman/interfaces/requests.py b/src/mailman/interfaces/requests.py index 9f841a6a3..35de05100 100644 --- a/src/mailman/interfaces/requests.py +++ b/src/mailman/interfaces/requests.py @@ -30,7 +30,7 @@ __all__ = [ ] -from flufl.enum import Enum +from enum import Enum from zope.interface import Interface, Attribute diff --git a/src/mailman/model/docs/autorespond.rst b/src/mailman/model/docs/autorespond.rst index 3a9ad01b2..b2bf03ed9 100644 --- a/src/mailman/model/docs/autorespond.rst +++ b/src/mailman/model/docs/autorespond.rst @@ -90,7 +90,7 @@ You can also use the response set to get the date of the last response sent. >>> response.address <Address: aperson@example.com [not verified] at ...> >>> response.response_type - <EnumValue: Response.hold [int=1]> + <Response.hold: 1> >>> response.date_sent datetime.date(2005, 8, 1) diff --git a/src/mailman/model/docs/membership.rst b/src/mailman/model/docs/membership.rst index f257f25ce..eeba9b332 100644 --- a/src/mailman/model/docs/membership.rst +++ b/src/mailman/model/docs/membership.rst @@ -217,7 +217,7 @@ There is also a roster containing all the subscribers of a mailing list, regardless of their role. >>> def sortkey(member): - ... return (member.address.email, int(member.role)) + ... return (member.address.email, member.role.value) >>> for member in sorted(mlist.subscribers.members, key=sortkey): ... print member.address.email, member.role aperson@example.com MemberRole.member diff --git a/src/mailman/model/docs/users.rst b/src/mailman/model/docs/users.rst index 997f983b2..889fe46d1 100644 --- a/src/mailman/model/docs/users.rst +++ b/src/mailman/model/docs/users.rst @@ -332,8 +332,7 @@ membership role. >>> len(members) 4 >>> def sortkey(member): - ... return (member.address.email, member.mailing_list, - ... int(member.role)) + ... return member.address.email, member.mailing_list, member.role.value >>> for member in sorted(members, key=sortkey): ... print member.address.email, member.mailing_list.list_id, \ ... member.role diff --git a/src/mailman/mta/docs/__init__.py b/src/mailman/mta/docs/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/mailman/mta/docs/__init__.py diff --git a/src/mailman/rest/addresses.py b/src/mailman/rest/addresses.py index 2b0bf5890..2908fad57 100644 --- a/src/mailman/rest/addresses.py +++ b/src/mailman/rest/addresses.py @@ -178,7 +178,7 @@ class UserAddresses(_AddressBase): def membership_key(member): # Sort first by mailing list, then by address, then by role. - return member.list_id, member.address.email, int(member.role) + return member.list_id, member.address.email, member.role.value class AddressMemberships(MemberCollection): diff --git a/src/mailman/rest/docs/preferences.rst b/src/mailman/rest/docs/preferences.rst index e82fd6001..8694364a4 100644 --- a/src/mailman/rest/docs/preferences.rst +++ b/src/mailman/rest/docs/preferences.rst @@ -95,6 +95,7 @@ PUT operation. ... 'acknowledge_posts': True, ... 'delivery_mode': 'plaintext_digests', ... 'delivery_status': 'by_user', + ... 'hide_address': False, ... 'preferred_language': 'ja', ... 'receive_list_copy': True, ... 'receive_own_postings': False, @@ -109,6 +110,7 @@ PUT operation. acknowledge_posts: True delivery_mode: plaintext_digests delivery_status: by_user + hide_address: False http_etag: "..." preferred_language: ja receive_list_copy: True @@ -133,6 +135,7 @@ You can also update just a few of the attributes using PATCH. acknowledge_posts: True delivery_mode: plaintext_digests delivery_status: by_user + hide_address: False http_etag: "..." preferred_language: ja receive_list_copy: False diff --git a/src/mailman/rest/helpers.py b/src/mailman/rest/helpers.py index 3a548cb10..7d911d42b 100644 --- a/src/mailman/rest/helpers.py +++ b/src/mailman/rest/helpers.py @@ -36,7 +36,7 @@ import hashlib from cStringIO import StringIO from datetime import datetime, timedelta -from flufl.enum import Enum +from enum import Enum from lazr.config import as_boolean from restish import http from restish.http import Response @@ -79,7 +79,7 @@ class ExtendedEncoder(json.JSONEncoder): seconds = obj.seconds + obj.microseconds / 1000000.0 return '{0}d{1}s'.format(obj.days, seconds) return '{0}d'.format(obj.days) - elif hasattr(obj, 'enum') and issubclass(obj.enum, Enum): + elif isinstance(obj, Enum): # It's up to the decoding validator to associate this name with # the right Enum class. return obj.name diff --git a/src/mailman/rest/lists.py b/src/mailman/rest/lists.py index 2a3c72bfd..32e22a76b 100644 --- a/src/mailman/rest/lists.py +++ b/src/mailman/rest/lists.py @@ -57,7 +57,7 @@ def member_matcher(request, segments): return None try: role = MemberRole[segments[0]] - except ValueError: + except KeyError: # Not a valid role. return None # No more segments. @@ -76,7 +76,7 @@ def roster_matcher(request, segments): return None try: return (), dict(role=MemberRole[segments[1]]), () - except ValueError: + except KeyError: # Not a valid role. return None diff --git a/src/mailman/rest/moderation.py b/src/mailman/rest/moderation.py index c0dbe93b8..491807f38 100644 --- a/src/mailman/rest/moderation.py +++ b/src/mailman/rest/moderation.py @@ -60,7 +60,7 @@ class _ModerationBase: resource.update(data) # Check for a matching request type, and insert the type name into the # resource. - request_type = RequestType(resource.pop('_request_type')) + request_type = RequestType[resource.pop('_request_type')] if request_type not in expected_request_types: return None resource['type'] = request_type.name @@ -205,7 +205,7 @@ class MembershipChangeRequest(resource.Resource, _ModerationBase): return http.not_found() key, data = results try: - request_type = RequestType(data['_request_type']) + request_type = RequestType[data['_request_type']] except ValueError: return http.bad_request() if request_type is RequestType.subscription: diff --git a/src/mailman/rest/preferences.py b/src/mailman/rest/preferences.py index d392c0299..49db6c632 100644 --- a/src/mailman/rest/preferences.py +++ b/src/mailman/rest/preferences.py @@ -84,6 +84,7 @@ class Preferences(ReadOnlyPreferences): return http.not_found() kws = dict( acknowledge_posts=GetterSetter(as_boolean), + hide_address = GetterSetter(as_boolean), delivery_mode=GetterSetter(enum_validator(DeliveryMode)), delivery_status=GetterSetter(enum_validator(DeliveryStatus)), preferred_language=GetterSetter(language_validator), diff --git a/src/mailman/rest/validator.py b/src/mailman/rest/validator.py index c59bc53d8..2d178226f 100644 --- a/src/mailman/rest/validator.py +++ b/src/mailman/rest/validator.py @@ -48,9 +48,13 @@ class enum_validator: self._enum_class = enum_class def __call__(self, enum_value): - # This will raise a ValueError if the enum value is unknown. Let that - # percolate up. - return self._enum_class[enum_value] + # This will raise a KeyError if the enum value is unknown. The + # Validator API requires turning this into a ValueError. + try: + return self._enum_class[enum_value] + except KeyError as exception: + # Retain the error message. + raise ValueError(exception.message) def subscriber_validator(subscriber): diff --git a/src/mailman/rules/docs/__init__.py b/src/mailman/rules/docs/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/mailman/rules/docs/__init__.py diff --git a/src/mailman/rules/docs/moderation.rst b/src/mailman/rules/docs/moderation.rst index eacc1cff3..3000f23a2 100644 --- a/src/mailman/rules/docs/moderation.rst +++ b/src/mailman/rules/docs/moderation.rst @@ -124,7 +124,7 @@ cperson is neither a member, nor a nonmember of the mailing list. :: >>> def memberkey(member): - ... return member.mailing_list, member.address.email, int(member.role) + ... return member.mailing_list, member.address.email, member.role.value >>> dump_list(mlist.members.members, key=memberkey) <Member: Anne Person <aperson@example.com> diff --git a/src/mailman/runners/docs/__init__.py b/src/mailman/runners/docs/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/mailman/runners/docs/__init__.py diff --git a/src/mailman/testing/__init__.py b/src/mailman/testing/__init__.py index e6a5047b6..e69de29bb 100644 --- a/src/mailman/testing/__init__.py +++ b/src/mailman/testing/__init__.py @@ -1,39 +0,0 @@ -# Copyright (C) 2011-2013 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Set up testing. - -This is used as an interface to buildout.cfg's [test] section. -zope.testrunner supports an initialization variable. It is set to import and -run the following test initialization method. -""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'initialize', - ] - - - -def initialize(root_directory): - """Initialize the test infrastructure.""" - from mailman.testing import layers - layers.MockAndMonkeyLayer.testing_mode = True - layers.ConfigLayer.enable_stderr(); - layers.ConfigLayer.set_root_directory(root_directory) diff --git a/src/mailman/tests/test_documentation.py b/src/mailman/testing/documentation.py index 37eb760ab..b1cc36f91 100644 --- a/src/mailman/tests/test_documentation.py +++ b/src/mailman/testing/documentation.py @@ -25,23 +25,16 @@ from __future__ import absolute_import, unicode_literals __metaclass__ = type __all__ = [ - 'test_suite', + 'setup', + 'teardown' ] -import os -import sys -import doctest -import unittest - from inspect import isfunction, ismethod -import mailman - from mailman.app.lifecycle import create_list from mailman.config import config -from mailman.testing.helpers import ( - call_api, chdir, specialized_message_from_string) +from mailman.testing.helpers import call_api, specialized_message_from_string from mailman.testing.layers import SMTPLayer @@ -181,53 +174,3 @@ def teardown(testobj): cleanup() else: cleanup[0](*cleanup[1:]) - - - -def test_suite(): - """Create test suites for all .rst documentation tests. - - .txt files are also tested, but .rst is highly preferred. - """ - suite = unittest.TestSuite() - topdir = os.path.dirname(mailman.__file__) - packages = [] - for dirpath, dirnames, filenames in os.walk(topdir): - if 'docs' in dirnames: - docsdir = os.path.join(dirpath, 'docs')[len(topdir)+1:] - packages.append(docsdir) - # Under higher verbosity settings, report all doctest errors, not just the - # first one. - flags = (doctest.ELLIPSIS | - doctest.NORMALIZE_WHITESPACE | - doctest.REPORT_NDIFF) - # Add all the doctests in all subpackages. - doctest_files = {} - with chdir(topdir): - for docsdir in packages: - # Look to see if the package defines a test layer, otherwise use - # SMTPLayer. - package_path = 'mailman.' + DOT.join(docsdir.split(os.sep)) - try: - __import__(package_path) - except ImportError: - layer = SMTPLayer - else: - layer = getattr(sys.modules[package_path], 'layer', SMTPLayer) - for filename in os.listdir(docsdir): - base, extension = os.path.splitext(filename) - if os.path.splitext(filename)[1] in ('.txt', '.rst'): - module_path = package_path + '.' + base - doctest_files[module_path] = ( - os.path.join(docsdir, filename), layer) - for module_path in sorted(doctest_files): - path, layer = doctest_files[module_path] - test = doctest.DocFileSuite( - path, - package='mailman', - optionflags=flags, - setUp=setup, - tearDown=teardown) - test.layer = layer - suite.addTest(test) - return suite diff --git a/src/mailman/testing/layers.py b/src/mailman/testing/layers.py index e47d5c9e0..6d150815f 100644 --- a/src/mailman/testing/layers.py +++ b/src/mailman/testing/layers.py @@ -154,7 +154,7 @@ class ConfigLayer(MockAndMonkeyLayer): continue logger_name = 'mailman.' + sub_name log = logging.getLogger(logger_name) - log.propagate = True + #log.propagate = True # Reopen the file to a new path that tests can get at. Instead of # using the configuration file path though, use a path that's # specific to the logger so that tests can find expected output diff --git a/src/mailman/testing/nose.py b/src/mailman/testing/nose.py new file mode 100644 index 000000000..86a3e6a01 --- /dev/null +++ b/src/mailman/testing/nose.py @@ -0,0 +1,107 @@ +# Copyright (C) 2013 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""nose2 test infrastructure.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'NosePlugin', + ] + + +import os +import re +import doctest +import mailman +import importlib + +from mailman.testing.documentation import setup, teardown +from mailman.testing.layers import ConfigLayer, MockAndMonkeyLayer, SMTPLayer +from nose2.events import Plugin + +DOT = '.' +FLAGS = doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF +TOPDIR = os.path.dirname(mailman.__file__) + + + +class NosePlugin(Plugin): + configSection = 'mailman' + + def __init__(self): + super(NosePlugin, self).__init__() + self.patterns = [] + self.addArgument(self.patterns, 'P', 'pattern', + 'Add a test matching pattern') + + def startTestRun(self, event): + MockAndMonkeyLayer.testing_mode = True + ConfigLayer.enable_stderr() + + def getTestCaseNames(self, event): + if len(self.patterns) == 0: + # No filter patterns, so everything should be tested. + return + # Does the pattern match the fully qualified class name? + for pattern in self.patterns: + full_name = '{}.{}'.format( + event.testCase.__module__, event.testCase.__name__) + if re.search(pattern, full_name): + # Don't suppress this test class. + return + names = filter(event.isTestMethod, dir(event.testCase)) + for name in names: + for pattern in self.patterns: + if re.search(pattern, name): + break + else: + event.excludedNames.append(name) + + def handleFile(self, event): + path = event.path[len(TOPDIR)+1:] + if len(self.patterns) > 0: + for pattern in self.patterns: + if re.search(pattern, path): + break + else: + # Skip this doctest. + return + base, ext = os.path.splitext(path) + if ext != '.rst': + return + # Look to see if the package defines a test layer, otherwise use the + # default layer. First turn the file system path into a dotted Python + # module path. + parent = os.path.dirname(path) + dotted = 'mailman.' + DOT.join(parent.split(os.path.sep)) + try: + module = importlib.import_module(dotted) + except ImportError: + layer = SMTPLayer + else: + layer = getattr(module, 'layer', SMTPLayer) + test = doctest.DocFileTest( + path, package='mailman', + optionflags=FLAGS, + setUp=setup, + tearDown=teardown) + test.layer = layer + # Suppress the extra "Doctest: ..." line. + test.shortDescription = lambda: None + event.extraTests.append(test) diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py index 20941bfd6..f5aa8d10a 100644 --- a/src/mailman/utilities/importer.py +++ b/src/mailman/utilities/importer.py @@ -28,6 +28,7 @@ __all__ = [ import sys import datetime +from mailman.interfaces.action import FilterAction from mailman.interfaces.autorespond import ResponseAction from mailman.interfaces.digests import DigestFrequency from mailman.interfaces.mailinglist import Personalization, ReplyToMunging @@ -47,6 +48,7 @@ TYPES = dict( bounce_info_stale_after=seconds_to_delta, bounce_you_are_disabled_warnings_interval=seconds_to_delta, digest_volume_frequency=DigestFrequency, + filter_action=FilterAction, newsgroup_moderation=NewsgroupModeration, personalize=Personalization, reply_goes_to_list=ReplyToMunging, diff --git a/unittest.cfg b/unittest.cfg new file mode 100644 index 000000000..d639a8ae3 --- /dev/null +++ b/unittest.cfg @@ -0,0 +1,10 @@ +[unittest] +verbose = 2 +plugins = mailman.testing.nose + nose2.plugins.layers + +[mailman] +always-on = True + +[log-capture] +always-on = False |