From 38a86adcdb78c1944c26a5ab8deddff619b33bcf Mon Sep 17 00:00:00 2001 From: J08nY Date: Wed, 31 May 2017 02:09:09 +0200 Subject: Add pluggable components. - Adds the notion of a 'plugin'. - A plugin has a package path and a flag specifying whether it's enabled or not. - Adds a find_pluggable_components function similar to the find_components one. This one dynamically searches not only the mailman package but all of plugins. - e.g. find_pluggable_components('rules', IRule) finds all IRule components in mailman.rules but also in example_plugin.rules for plugin names example_plugin. - Uses the find_pluggable_components function in place of find_components when searching for Rules, Handlers, Chains, EmailCommands, and Styles. --- src/mailman/app/commands.py | 4 +- src/mailman/app/docs/hooks.rst | 121 ------------------------------ src/mailman/app/docs/plugins.rst | 121 ++++++++++++++++++++++++++++++ src/mailman/bin/mailman.py | 39 ++++++---- src/mailman/commands/docs/conf.rst | 1 + src/mailman/commands/docs/info.rst | 1 - src/mailman/config/config.py | 7 +- src/mailman/config/schema.cfg | 20 +++-- src/mailman/core/chains.py | 5 +- src/mailman/core/pipelines.py | 6 +- src/mailman/core/rules.py | 4 +- src/mailman/rest/tests/test_systemconf.py | 1 + src/mailman/styles/manager.py | 10 +-- src/mailman/utilities/plugins.py | 76 +++++++++++++++++++ 14 files changed, 253 insertions(+), 163 deletions(-) delete mode 100644 src/mailman/app/docs/hooks.rst create mode 100644 src/mailman/app/docs/plugins.rst create mode 100644 src/mailman/utilities/plugins.py (limited to 'src/mailman') diff --git a/src/mailman/app/commands.py b/src/mailman/app/commands.py index 7cf3bc4c5..ca257ffda 100644 --- a/src/mailman/app/commands.py +++ b/src/mailman/app/commands.py @@ -19,11 +19,11 @@ from mailman.config import config from mailman.interfaces.command import IEmailCommand -from mailman.utilities.modules import add_components +from mailman.utilities.plugins import add_pluggable_components from public import public @public def initialize(): """Initialize the email commands.""" - add_components('mailman.commands', IEmailCommand, config.commands) + add_pluggable_components('commands', IEmailCommand, config.commands) diff --git a/src/mailman/app/docs/hooks.rst b/src/mailman/app/docs/hooks.rst deleted file mode 100644 index 07d0ec33c..000000000 --- a/src/mailman/app/docs/hooks.rst +++ /dev/null @@ -1,121 +0,0 @@ -===== -Hooks -===== - -Mailman defines two initialization hooks, one which is run early in the -initialization process and the other run late in the initialization process. -Hooks name an importable callable so it must be accessible on ``sys.path``. -:: - - >>> import os, sys - >>> from mailman.config import config - >>> config_directory = os.path.dirname(config.filename) - >>> sys.path.insert(0, config_directory) - - >>> hook_path = os.path.join(config_directory, 'hooks.py') - >>> with open(hook_path, 'w') as fp: - ... print("""\ - ... counter = 1 - ... def pre_hook(): - ... global counter - ... print('pre-hook:', counter) - ... counter += 1 - ... - ... def post_hook(): - ... global counter - ... print('post-hook:', counter) - ... counter += 1 - ... """, file=fp) - >>> fp.close() - - -Pre-hook -======== - -We can set the pre-hook in the configuration file. - - >>> config_path = os.path.join(config_directory, 'hooks.cfg') - >>> with open(config_path, 'w') as fp: - ... print("""\ - ... [meta] - ... extends: test.cfg - ... - ... [mailman] - ... pre_hook: hooks.pre_hook - ... """, file=fp) - -The hooks are run in the second and third steps of initialization. However, -we can't run those initialization steps in process, so call a command line -script that will produce no output to force the hooks to run. -:: - - >>> import subprocess - >>> from mailman.testing.layers import ConfigLayer - >>> def call(): - ... exe = os.path.join(os.path.dirname(sys.executable), 'mailman') - ... env = os.environ.copy() - ... env.update( - ... MAILMAN_CONFIG_FILE=config_path, - ... PYTHONPATH=config_directory, - ... ) - ... test_cfg = os.environ.get('MAILMAN_EXTRA_TESTING_CFG') - ... if test_cfg is not None: - ... env['MAILMAN_EXTRA_TESTING_CFG'] = test_cfg - ... proc = subprocess.Popen( - ... [exe, 'lists', '--domain', 'ignore', '-q'], - ... cwd=ConfigLayer.root_directory, env=env, - ... universal_newlines=True, - ... stdout=subprocess.PIPE, stderr=subprocess.PIPE) - ... stdout, stderr = proc.communicate() - ... assert proc.returncode == 0, stderr - ... print(stdout) - - >>> call() - pre-hook: 1 - - - >>> os.remove(config_path) - - -Post-hook -========= - -We can set the post-hook in the configuration file. -:: - - >>> with open(config_path, 'w') as fp: - ... print("""\ - ... [meta] - ... extends: test.cfg - ... - ... [mailman] - ... post_hook: hooks.post_hook - ... """, file=fp) - - >>> call() - post-hook: 1 - - - >>> os.remove(config_path) - - -Running both hooks -================== - -We can set the pre- and post-hooks in the configuration file. -:: - - >>> with open(config_path, 'w') as fp: - ... print("""\ - ... [meta] - ... extends: test.cfg - ... - ... [mailman] - ... pre_hook: hooks.pre_hook - ... post_hook: hooks.post_hook - ... """, file=fp) - - >>> call() - pre-hook: 1 - post-hook: 2 - diff --git a/src/mailman/app/docs/plugins.rst b/src/mailman/app/docs/plugins.rst new file mode 100644 index 000000000..07d0ec33c --- /dev/null +++ b/src/mailman/app/docs/plugins.rst @@ -0,0 +1,121 @@ +===== +Hooks +===== + +Mailman defines two initialization hooks, one which is run early in the +initialization process and the other run late in the initialization process. +Hooks name an importable callable so it must be accessible on ``sys.path``. +:: + + >>> import os, sys + >>> from mailman.config import config + >>> config_directory = os.path.dirname(config.filename) + >>> sys.path.insert(0, config_directory) + + >>> hook_path = os.path.join(config_directory, 'hooks.py') + >>> with open(hook_path, 'w') as fp: + ... print("""\ + ... counter = 1 + ... def pre_hook(): + ... global counter + ... print('pre-hook:', counter) + ... counter += 1 + ... + ... def post_hook(): + ... global counter + ... print('post-hook:', counter) + ... counter += 1 + ... """, file=fp) + >>> fp.close() + + +Pre-hook +======== + +We can set the pre-hook in the configuration file. + + >>> config_path = os.path.join(config_directory, 'hooks.cfg') + >>> with open(config_path, 'w') as fp: + ... print("""\ + ... [meta] + ... extends: test.cfg + ... + ... [mailman] + ... pre_hook: hooks.pre_hook + ... """, file=fp) + +The hooks are run in the second and third steps of initialization. However, +we can't run those initialization steps in process, so call a command line +script that will produce no output to force the hooks to run. +:: + + >>> import subprocess + >>> from mailman.testing.layers import ConfigLayer + >>> def call(): + ... exe = os.path.join(os.path.dirname(sys.executable), 'mailman') + ... env = os.environ.copy() + ... env.update( + ... MAILMAN_CONFIG_FILE=config_path, + ... PYTHONPATH=config_directory, + ... ) + ... test_cfg = os.environ.get('MAILMAN_EXTRA_TESTING_CFG') + ... if test_cfg is not None: + ... env['MAILMAN_EXTRA_TESTING_CFG'] = test_cfg + ... proc = subprocess.Popen( + ... [exe, 'lists', '--domain', 'ignore', '-q'], + ... cwd=ConfigLayer.root_directory, env=env, + ... universal_newlines=True, + ... stdout=subprocess.PIPE, stderr=subprocess.PIPE) + ... stdout, stderr = proc.communicate() + ... assert proc.returncode == 0, stderr + ... print(stdout) + + >>> call() + pre-hook: 1 + + + >>> os.remove(config_path) + + +Post-hook +========= + +We can set the post-hook in the configuration file. +:: + + >>> with open(config_path, 'w') as fp: + ... print("""\ + ... [meta] + ... extends: test.cfg + ... + ... [mailman] + ... post_hook: hooks.post_hook + ... """, file=fp) + + >>> call() + post-hook: 1 + + + >>> os.remove(config_path) + + +Running both hooks +================== + +We can set the pre- and post-hooks in the configuration file. +:: + + >>> with open(config_path, 'w') as fp: + ... print("""\ + ... [meta] + ... extends: test.cfg + ... + ... [mailman] + ... pre_hook: hooks.pre_hook + ... post_hook: hooks.post_hook + ... """, file=fp) + + >>> call() + pre-hook: 1 + post-hook: 2 + diff --git a/src/mailman/bin/mailman.py b/src/mailman/bin/mailman.py index e012237f3..ece223180 100644 --- a/src/mailman/bin/mailman.py +++ b/src/mailman/bin/mailman.py @@ -16,7 +16,6 @@ # GNU Mailman. If not, see . """The 'mailman' command dispatcher.""" - import click from contextlib import ExitStack @@ -25,7 +24,7 @@ from mailman.core.i18n import _ from mailman.core.initialize import initialize from mailman.database.transaction import transaction from mailman.interfaces.command import ICLISubCommand -from mailman.utilities.modules import add_components +from mailman.utilities.plugins import add_pluggable_components from mailman.version import MAILMAN_VERSION_FULL from public import public @@ -35,17 +34,22 @@ class Subcommands(click.MultiCommand): def __init__(self, *args, **kws): super().__init__(*args, **kws) self._commands = {} - # Look at all modules in the mailman.bin package and if they are - # prepared to add a subcommand, let them do so. I'm still undecided as - # to whether this should be pluggable or not. If so, then we'll - # probably have to partially parse the arguments now, then initialize - # the system, then find the plugins. Punt on this for now. - add_components('mailman.commands', ICLISubCommand, self._commands) + self._loaded = False + + def _load(self): + # Load commands lazily as pluggable commands need a parsed config to + # find plugins. + if not self._loaded: + add_pluggable_components('commands', ICLISubCommand, + self._commands) + self._loaded = True def list_commands(self, ctx): + self._load() return sorted(self._commands) # pragma: nocover def get_command(self, ctx, name): + self._load() try: return self._commands[name].command except KeyError as error: @@ -83,10 +87,12 @@ class Subcommands(click.MultiCommand): formatter.write_dl(opts) -@click.group( - cls=Subcommands, - context_settings=dict(help_option_names=['-h', '--help'])) -@click.pass_context +def initialize_config(ctx, param, value): + if ctx.resilient_parsing: + return + initialize(value) + + @click.option( '-C', '--config', 'config_file', envvar='MAILMAN_CONFIG_FILE', @@ -94,7 +100,12 @@ class Subcommands(click.MultiCommand): help=_("""\ Configuration file to use. If not given, the environment variable MAILMAN_CONFIG_FILE is consulted and used if set. If neither are given, a - default configuration file is loaded.""")) + default configuration file is loaded."""), + is_eager=True, callback=initialize_config) +@click.group( + cls=Subcommands, + context_settings=dict(help_option_names=['-h', '--help'])) +@click.pass_context @click.version_option(MAILMAN_VERSION_FULL, message='%(version)s') @public def main(ctx, config_file): @@ -104,6 +115,4 @@ def main(ctx, config_file): Copyright 1998-2017 by the Free Software Foundation, Inc. http://www.list.org """ - # Initialize the system. Honor the -C flag if given. - initialize(config_file) # click handles dispatching to the subcommand via the Subcommands class. diff --git a/src/mailman/commands/docs/conf.rst b/src/mailman/commands/docs/conf.rst index 1a6a4679d..6c7309773 100644 --- a/src/mailman/commands/docs/conf.rst +++ b/src/mailman/commands/docs/conf.rst @@ -50,6 +50,7 @@ key, along with the names of the corresponding sections. [logging.smtp] path: smtp.log [logging.subscribe] path: mailman.log [logging.vette] path: mailman.log + [plugin.master] path: If you specify both a section and a key, you will get the corresponding value. diff --git a/src/mailman/commands/docs/info.rst b/src/mailman/commands/docs/info.rst index 219143cab..40784a98b 100644 --- a/src/mailman/commands/docs/info.rst +++ b/src/mailman/commands/docs/info.rst @@ -59,7 +59,6 @@ definition. CFG_FILE = .../test.cfg DATA_DIR = /var/lib/mailman/data ETC_DIR = /etc - EXT_DIR = /etc/mailman.d LIST_DATA_DIR = /var/lib/mailman/lists LOCK_DIR = /var/lock/mailman LOCK_FILE = /var/lock/mailman/master.lck diff --git a/src/mailman/config/config.py b/src/mailman/config/config.py index 61c9fe6ed..2ba657905 100644 --- a/src/mailman/config/config.py +++ b/src/mailman/config/config.py @@ -158,7 +158,7 @@ class Configuration: else category.template_dir), ) # Directories. - for name in ('archive', 'bin', 'cache', 'data', 'etc', 'ext', + for name in ('archive', 'bin', 'cache', 'data', 'etc', 'list_data', 'lock', 'log', 'messages', 'queue'): key = '{}_dir'.format(name) substitutions[key] = getattr(category, key) @@ -247,6 +247,11 @@ class Configuration: archiver.is_enabled = as_boolean(section.enable) yield archiver + @property + def plugin_configs(self): + """Iterate over all the plugin configuration sections.""" + return self._config.getByCategory('plugin', []) + @property def language_configs(self): """Iterate over all the language configuration sections.""" diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg index 9568694be..b708c6ceb 100644 --- a/src/mailman/config/schema.cfg +++ b/src/mailman/config/schema.cfg @@ -82,6 +82,19 @@ html_to_plain_text_command: /usr/bin/lynx -dump $filename # unpredictable. listname_chars: [-_.0-9a-z] +[plugin.master] +# Plugin package +# It's sub packages will be searched for components. +# - commands for IEmailCommand +# - chains for IChain +# - rules for IRule +# - pipelines for IPipeline +# - handlers for IHandler +# - styles for IStyle +path: + +# Whether to enable this plugin or not. +enable: no [shell] # `mailman shell` (also `withlist`) gives you an interactive prompt that you @@ -138,8 +151,6 @@ data_dir: $var_dir/data cache_dir: $var_dir/cache # Directory for configuration files and such. etc_dir: $var_dir/etc -# Directory containing Mailman plugins. -ext_dir: $var_dir/ext # Directory where the default IMessageStore puts its messages. messages_dir: $var_dir/messages # Directory for archive backends to store their messages in. Archivers should @@ -797,11 +808,6 @@ class: mailman.archiving.prototype.Prototype [styles] -# Python import paths inside which components are searched for which implement -# the IStyle interface. Use one path per line. -paths: - mailman.styles - # The default style to apply if nothing else was requested. The value is the # name of an existing style. If no such style exists, no style will be # applied. diff --git a/src/mailman/core/chains.py b/src/mailman/core/chains.py index c251d38bc..48ed82d63 100644 --- a/src/mailman/core/chains.py +++ b/src/mailman/core/chains.py @@ -19,7 +19,7 @@ from mailman.config import config from mailman.interfaces.chain import IChain, LinkAction -from mailman.utilities.modules import add_components +from mailman.utilities.plugins import add_pluggable_components from public import public @@ -89,5 +89,4 @@ def process(mlist, msg, msgdata, start_chain='default-posting-chain'): @public def initialize(): """Set up chains, both built-in and from the database.""" - add_components('mailman.chains', IChain, config.chains) - # XXX Read chains from the database and initialize them. + add_pluggable_components('chains', IChain, config.chains) diff --git a/src/mailman/core/pipelines.py b/src/mailman/core/pipelines.py index 4265d24d1..6e2ad4a27 100644 --- a/src/mailman/core/pipelines.py +++ b/src/mailman/core/pipelines.py @@ -24,7 +24,7 @@ from mailman.config import config from mailman.interfaces.handler import IHandler from mailman.interfaces.pipeline import ( DiscardMessage, IPipeline, RejectMessage) -from mailman.utilities.modules import add_components +from mailman.utilities.plugins import add_pluggable_components from public import public @@ -63,6 +63,6 @@ def process(mlist, msg, msgdata, pipeline_name='built-in'): def initialize(): """Initialize the pipelines.""" # Find all handlers in the registered plugins. - add_components('mailman.handlers', IHandler, config.handlers) + add_pluggable_components('handlers', IHandler, config.handlers) # Set up some pipelines. - add_components('mailman.pipelines', IPipeline, config.pipelines) + add_pluggable_components('pipelines', IPipeline, config.pipelines) diff --git a/src/mailman/core/rules.py b/src/mailman/core/rules.py index 8e0d9197c..74934eb99 100644 --- a/src/mailman/core/rules.py +++ b/src/mailman/core/rules.py @@ -19,7 +19,7 @@ from mailman.config import config from mailman.interfaces.rules import IRule -from mailman.utilities.modules import add_components +from mailman.utilities.plugins import add_pluggable_components from public import public @@ -27,4 +27,4 @@ from public import public def initialize(): """Find and register all rules in all plugins.""" # Find rules in plugins. - add_components('mailman.rules', IRule, config.rules) + add_pluggable_components('rules', IRule, config.rules) diff --git a/src/mailman/rest/tests/test_systemconf.py b/src/mailman/rest/tests/test_systemconf.py index f9fe9caa0..5acc38adb 100644 --- a/src/mailman/rest/tests/test_systemconf.py +++ b/src/mailman/rest/tests/test_systemconf.py @@ -182,6 +182,7 @@ class TestSystemConfiguration(unittest.TestCase): 'paths.here', 'paths.local', 'paths.testing', + 'plugin.master', 'runner.archive', 'runner.bad', 'runner.bounces', diff --git a/src/mailman/styles/manager.py b/src/mailman/styles/manager.py index 1ec83b2a9..1969d4484 100644 --- a/src/mailman/styles/manager.py +++ b/src/mailman/styles/manager.py @@ -20,7 +20,7 @@ from mailman.interfaces.configuration import ConfigurationUpdatedEvent from mailman.interfaces.styles import ( DuplicateStyleError, IStyle, IStyleManager) -from mailman.utilities.modules import add_components +from mailman.utilities.plugins import add_pluggable_components from public import public from zope.component import getUtility from zope.interface import implementer @@ -38,13 +38,7 @@ class StyleManager: def populate(self): self._styles.clear() - # Avoid circular imports. - from mailman.config import config - # Calculate the Python import paths to search. - paths = filter(None, (path.strip() - for path in config.styles.paths.splitlines())) - for path in paths: - add_components(path, IStyle, self._styles) + add_pluggable_components('styles', IStyle, self._styles) def get(self, name): """See `IStyleManager`.""" diff --git a/src/mailman/utilities/plugins.py b/src/mailman/utilities/plugins.py new file mode 100644 index 000000000..a8449af31 --- /dev/null +++ b/src/mailman/utilities/plugins.py @@ -0,0 +1,76 @@ +# Copyright (C) 2009-2017 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Plugin utilities.""" + +from lazr.config import as_boolean +from mailman.config import config +from mailman.utilities.modules import find_components +from pkg_resources import resource_isdir +from public import public + + +@public +def find_pluggable_components(subpackage, interface): + """Find components which conform to a given interface, in subpackage. + + :param subpackage: Mailman subpackage to search. The search is also + done inside the corresponding plugins subpackages. + :type subpackage: string + :param interface: The interface that returned objects must conform to. + :type interface: `Interface` + :return: The sequence of matching components. + :rtype: objects implementing `interface` + """ + yield from find_components('mailman.' + subpackage, interface) + package_roots = [plugin.path + for plugin in config.plugin_configs + if as_boolean(plugin.enable) and plugin.path] + for package_root in package_roots: + package = package_root + '.' + subpackage + if resource_isdir(package_root, subpackage): + yield from find_components(package, interface) + + +@public +def add_pluggable_components(subpackage, interface, mapping): + """Add components to a given mapping. + + Similarly to `find_pluggable_components()` this inspects all modules in a + given mailman and plugin's subpackage looking for objects that conform to + a given interface. All such found objects (unless decorated with + `@abstract_component`) are added to the given mapping, keyed by the + object's `.name` attribute, which is required. It is a fatal error if + that key already exists in the mapping. + + :param package: The subpackage path to search. + :type package: string + :param interface: The interface that returned objects must conform to. + Objects found must have a `.name` attribute containing a unique + string. + :type interface: `Interface` + :param mapping: The mapping to add the found components to. + :type mapping: A dict-like mapping. This only needs to support + containment tests (e.g. `in` and `not in`) and `__setitem__()`. + :raises RuntimeError: when a duplicate key is found. + """ + for component in find_pluggable_components(subpackage, interface): + if component.name in mapping: + raise RuntimeError( + 'Duplicate key "{}" found in {}; previously {}'.format( + component.name, component, mapping[component.name])) + mapping[component.name] = component -- cgit v1.2.3-70-g09d2 From 0d4f53b51892866b5cc85ace229b23f4b9bac896 Mon Sep 17 00:00:00 2001 From: J08nY Date: Thu, 1 Jun 2017 15:46:48 +0200 Subject: Add per-plugin hooks, add docs about plugins. - Removes pre_hook, post_hook and ext_dir. With the latter being unused. Warns on startup if the hooks are present in config. - Adds IPlugin interface with pre_hook and post_hook methods. - Adds 'class' config parameter to plugins, which can be set to a class implementing the IPlugin interface, it will be initialized once on Mailman's startup, then before the DB setup the pre_hook will be run, after DB and other components the post_hook will be run. Plugin instances are stored in the config.plugins dict, with keys being their configuration section names. --- src/mailman/app/docs/plugins.rst | 174 +++++++++++++++++++----------- src/mailman/commands/docs/conf.rst | 1 - src/mailman/config/config.py | 1 + src/mailman/config/schema.cfg | 12 +-- src/mailman/core/initialize.py | 29 +++-- src/mailman/core/plugins.py | 42 ++++++++ src/mailman/interfaces/plugin.py | 32 ++++++ src/mailman/rest/docs/systemconf.rst | 2 - src/mailman/rest/tests/test_systemconf.py | 4 +- src/mailman/testing/plugin.py | 36 +++++++ src/mailman/testing/testing.cfg | 4 + 11 files changed, 250 insertions(+), 87 deletions(-) create mode 100644 src/mailman/core/plugins.py create mode 100644 src/mailman/interfaces/plugin.py create mode 100644 src/mailman/testing/plugin.py (limited to 'src/mailman') diff --git a/src/mailman/app/docs/plugins.rst b/src/mailman/app/docs/plugins.rst index 07d0ec33c..42038b654 100644 --- a/src/mailman/app/docs/plugins.rst +++ b/src/mailman/app/docs/plugins.rst @@ -1,10 +1,10 @@ -===== -Hooks -===== +======= +Plugins +======= -Mailman defines two initialization hooks, one which is run early in the -initialization process and the other run late in the initialization process. -Hooks name an importable callable so it must be accessible on ``sys.path``. +Mailman defines a plugin as a package accessible on ``sys.path`` that provides +components Mailman will use. Such add handlers, rules, chains, etc... +First we create an example plugin that provides one additional rule: :: >>> import os, sys @@ -12,27 +12,115 @@ Hooks name an importable callable so it must be accessible on ``sys.path``. >>> config_directory = os.path.dirname(config.filename) >>> sys.path.insert(0, config_directory) + >>> example_plugin_path = os.path.join(config_directory, 'example_plugin') + >>> rules_path = os.path.join(example_plugin_path, 'rules') + >>> os.makedirs(rules_path) + >>> open(os.path.join(example_plugin_path, '__init__.py'), 'a').close() + >>> open(os.path.join(rules_path, '__init__.py'), 'a').close() + >>> rule_path = os.path.join(rules_path, 'example_rule.py') + >>> with open(rule_path, 'w') as fp: + ... print("""\ + ... from mailman.interfaces.rules import IRule + ... from public import public + ... from zope.interface import implementer + ... + ... @public + ... @implementer(IRule) + ... class ExampleRule: + ... + ... name = 'example' + ... description = 'Example rule to show pluggable components.' + ... record = True + ... + ... def check(self, mlist, msg, msgdata): + ... return msg.original_size > 1024 + ... """, file=fp) + >>> fp.close() + +Then enable the example plugin in config. +:: + + >>> example_config = """\ + ... [plugin.example] + ... path: example_plugin + ... enable: yes + ... """ + >>> config.push('example_cfg', example_config) + +Now the `example` rule can be seen as an ``IRule`` component and will be used +when any chain uses a link called `example`. +:: + + >>> from mailman.interfaces.rules import IRule + >>> from mailman.utilities.plugins import find_pluggable_components + >>> rules = sorted([rule.name + ... for rule in find_pluggable_components('rules', IRule)]) + >>> for rule in rules: + ... print(rule) + administrivia + any + approved + banned-address + dmarc-mitigation + emergency + example + implicit-dest + loop + max-recipients + max-size + member-moderation + news-moderation + no-senders + no-subject + nonmember-moderation + suspicious-header + truth + + + >>> config.pop('example_cfg') + >>> from shutil import rmtree + >>> rmtree(example_plugin_path) + + +Hooks +===== + +Plugins can also add initialization hooks, which will be run during the +initialization process. By creating a class implementing the IPlugin interface. +One which is run early in the process and the other run late in the +initialization process. +:: + >>> hook_path = os.path.join(config_directory, 'hooks.py') >>> with open(hook_path, 'w') as fp: ... print("""\ + ... from mailman.interfaces.plugin import IPlugin + ... from public import public + ... from zope.interface import implementer + ... ... counter = 1 - ... def pre_hook(): - ... global counter - ... print('pre-hook:', counter) - ... counter += 1 ... - ... def post_hook(): - ... global counter - ... print('post-hook:', counter) - ... counter += 1 + ... @public + ... @implementer(IPlugin) + ... class ExamplePlugin: + ... + ... def pre_hook(self): + ... global counter + ... print('pre-hook:', counter) + ... counter += 1 + ... + ... def post_hook(self): + ... global counter + ... print('post-hook:', counter) + ... counter += 1 ... """, file=fp) >>> fp.close() +Running the hooks +----------------- -Pre-hook -======== - -We can set the pre-hook in the configuration file. +We can set the plugin class in the config file. +:: >>> config_path = os.path.join(config_directory, 'hooks.cfg') >>> with open(config_path, 'w') as fp: @@ -40,8 +128,9 @@ We can set the pre-hook in the configuration file. ... [meta] ... extends: test.cfg ... - ... [mailman] - ... pre_hook: hooks.pre_hook + ... [plugin.hook_example] + ... class: hooks.ExamplePlugin + ... enable: yes ... """, file=fp) The hooks are run in the second and third steps of initialization. However, @@ -72,50 +161,7 @@ script that will produce no output to force the hooks to run. >>> call() pre-hook: 1 + post-hook: 2 >>> os.remove(config_path) - - -Post-hook -========= - -We can set the post-hook in the configuration file. -:: - - >>> with open(config_path, 'w') as fp: - ... print("""\ - ... [meta] - ... extends: test.cfg - ... - ... [mailman] - ... post_hook: hooks.post_hook - ... """, file=fp) - - >>> call() - post-hook: 1 - - - >>> os.remove(config_path) - - -Running both hooks -================== - -We can set the pre- and post-hooks in the configuration file. -:: - - >>> with open(config_path, 'w') as fp: - ... print("""\ - ... [meta] - ... extends: test.cfg - ... - ... [mailman] - ... pre_hook: hooks.pre_hook - ... post_hook: hooks.post_hook - ... """, file=fp) - - >>> call() - pre-hook: 1 - post-hook: 2 - diff --git a/src/mailman/commands/docs/conf.rst b/src/mailman/commands/docs/conf.rst index 6c7309773..1a6a4679d 100644 --- a/src/mailman/commands/docs/conf.rst +++ b/src/mailman/commands/docs/conf.rst @@ -50,7 +50,6 @@ key, along with the names of the corresponding sections. [logging.smtp] path: smtp.log [logging.subscribe] path: mailman.log [logging.vette] path: mailman.log - [plugin.master] path: If you specify both a section and a key, you will get the corresponding value. diff --git a/src/mailman/config/config.py b/src/mailman/config/config.py index 2ba657905..405b689ef 100644 --- a/src/mailman/config/config.py +++ b/src/mailman/config/config.py @@ -76,6 +76,7 @@ class Configuration: self.handlers = {} self.pipelines = {} self.commands = {} + self.plugins = {} self.password_context = None self.db = None diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg index b708c6ceb..0e7936943 100644 --- a/src/mailman/config/schema.cfg +++ b/src/mailman/config/schema.cfg @@ -54,14 +54,6 @@ pending_request_life: 3d # How long should files be saved before they are evicted from the cache? cache_life: 7d -# A callable to run with no arguments early in the initialization process. -# This runs before database initialization. -pre_hook: - -# A callable to run with no arguments late in the initialization process. -# This runs after adapters are initialized. -post_hook: - # Which paths.* file system layout to use. layout: here @@ -93,9 +85,13 @@ listname_chars: [-_.0-9a-z] # - styles for IStyle path: +# The full import path for the class implementing IPlugin. +class: + # Whether to enable this plugin or not. enable: no + [shell] # `mailman shell` (also `withlist`) gives you an interactive prompt that you # can use to interact with an initialized and configured Mailman system. Use diff --git a/src/mailman/core/initialize.py b/src/mailman/core/initialize.py index dc67e9a68..18e874552 100644 --- a/src/mailman/core/initialize.py +++ b/src/mailman/core/initialize.py @@ -26,11 +26,11 @@ by the command line arguments. import os import sys +import warnings import mailman.config.config import mailman.core.logging from mailman.interfaces.database import IDatabaseFactory -from mailman.utilities.modules import call_name from pkg_resources import resource_string as resource_bytes from public import public from zope.component import getUtility @@ -133,7 +133,7 @@ def initialize_2(debug=False, propagate_logs=None, testing=False): * Database * Logging - * Pre-hook + * Plugin's pre_hook * Rules * Chains * Pipelines @@ -146,10 +146,17 @@ def initialize_2(debug=False, propagate_logs=None, testing=False): """ # Create the queue and log directories if they don't already exist. mailman.core.logging.initialize(propagate_logs) - # Run the pre-hook if there is one. + # Initialize plugins + from mailman.core.plugins import initialize as initialize_plugins + initialize_plugins() + # Run the plugin pre_hooks config = mailman.config.config - if config.mailman.pre_hook: - call_name(config.mailman.pre_hook) + if "pre_hook" in config.mailman: # pragma: no cover + warnings.warn( + "The pre_hook configuration value has been replaced by the " + "plugins infrastructure.", DeprecationWarning) + for plugin in config.plugins.values(): + plugin.pre_hook() # Instantiate the database class, ensure that it's of the right type, and # initialize it. Then stash the object on our configuration object. utility_name = ('testing' if testing else 'production') @@ -171,12 +178,16 @@ def initialize_2(debug=False, propagate_logs=None, testing=False): def initialize_3(): """Third initialization step. - * Post-hook + * Plugin's post_hook """ - # Run the post-hook if there is one. + # Run the plugin post_hooks config = mailman.config.config - if config.mailman.post_hook: - call_name(config.mailman.post_hook) + if "post_hook" in config.mailman: # pragma: no cover + warnings.warn( + "The post_hook configuration value has been replaced by the " + "plugins infrastructure.", DeprecationWarning) + for plugin in config.plugins.values(): + plugin.post_hook() @public diff --git a/src/mailman/core/plugins.py b/src/mailman/core/plugins.py new file mode 100644 index 000000000..565c590b5 --- /dev/null +++ b/src/mailman/core/plugins.py @@ -0,0 +1,42 @@ +# Copyright (C) 2008-2017 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Various plugin helpers.""" + +from lazr.config import as_boolean +from mailman.config import config +from mailman.interfaces.plugin import IPlugin +from mailman.utilities.modules import find_name +from public import public +from zope.interface.verify import verifyObject + + +@public +def initialize(): + """Initialize all enabled plugins.""" + for plugin_config in config.plugin_configs: + plugin_class_path = plugin_config['class'] + if as_boolean(plugin_config.enable) and plugin_class_path: + plugin_class = find_name(plugin_class_path) + plugin = plugin_class() + verifyObject(IPlugin, plugin) + name = plugin_config.name.split('.')[-1] + plugin.name = name + assert plugin.name not in config.plugins, ( + 'Duplicate plugin "{}" found in {}'.format( + plugin.name, plugin_class)) + config.plugins[plugin.name] = plugin diff --git a/src/mailman/interfaces/plugin.py b/src/mailman/interfaces/plugin.py new file mode 100644 index 000000000..9e226824f --- /dev/null +++ b/src/mailman/interfaces/plugin.py @@ -0,0 +1,32 @@ +# Copyright (C) 2007-2017 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Interfaces for plugins.""" + +from public import public +from zope.interface import Interface + + +@public +class IPlugin(Interface): + """A plugin providing components and hooks.""" + + def pre_hook(): + """A plugin hook called in the first initialization step. Before DB.""" + + def post_hook(): + """A plugin hook called in the second initialization step. After DB.""" diff --git a/src/mailman/rest/docs/systemconf.rst b/src/mailman/rest/docs/systemconf.rst index a65b81ab8..f5c9b691a 100644 --- a/src/mailman/rest/docs/systemconf.rst +++ b/src/mailman/rest/docs/systemconf.rst @@ -24,8 +24,6 @@ You can also get all the values for a particular section, such as the listname_chars: [-_.0-9a-z] noreply_address: noreply pending_request_life: 3d - post_hook: - pre_hook: self_link: http://localhost:9001/3.0/system/configuration/mailman sender_headers: from from_ reply-to sender site_owner: noreply@example.com diff --git a/src/mailman/rest/tests/test_systemconf.py b/src/mailman/rest/tests/test_systemconf.py index 5acc38adb..42802068c 100644 --- a/src/mailman/rest/tests/test_systemconf.py +++ b/src/mailman/rest/tests/test_systemconf.py @@ -46,8 +46,6 @@ class TestSystemConfiguration(unittest.TestCase): listname_chars='[-_.0-9a-z]', noreply_address='noreply', pending_request_life='3d', - post_hook='', - pre_hook='', self_link='http://localhost:9001/3.0/system/configuration/mailman', sender_headers='from from_ reply-to sender', site_owner='noreply@example.com', @@ -182,7 +180,7 @@ class TestSystemConfiguration(unittest.TestCase): 'paths.here', 'paths.local', 'paths.testing', - 'plugin.master', + 'plugin.example', 'runner.archive', 'runner.bad', 'runner.bounces', diff --git a/src/mailman/testing/plugin.py b/src/mailman/testing/plugin.py new file mode 100644 index 000000000..75667844a --- /dev/null +++ b/src/mailman/testing/plugin.py @@ -0,0 +1,36 @@ +# Copyright (C) 2009-2017 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Example plugin with hooks for testing purposes.""" + +from mailman.interfaces.plugin import IPlugin +from public import public +from zope.interface import implementer + + +@public +@implementer(IPlugin) +class ExamplePlugin: + + def __init__(self): + pass + + def pre_hook(self): + pass + + def post_hook(self): + pass diff --git a/src/mailman/testing/testing.cfg b/src/mailman/testing/testing.cfg index 94b4c07b9..9e17015ef 100644 --- a/src/mailman/testing/testing.cfg +++ b/src/mailman/testing/testing.cfg @@ -28,6 +28,10 @@ incoming: mailman.testing.mta.FakeMTA [passwords] configuration: python:mailman.testing.passlib +[plugin.example] +enable: yes +class: mailman.testing.plugin.ExamplePlugin + [webservice] port: 9001 -- cgit v1.2.3-70-g09d2 From 1f1a35e7ccde1cfe239286a9a6333ebe8882d8f3 Mon Sep 17 00:00:00 2001 From: J08nY Date: Sat, 3 Jun 2017 23:00:48 +0200 Subject: Pretty print dicts in dump_json. - Makes dump_json documentation helper recursively print dicts with indentation. So for example {'name': {'a':1, 'b':2}, 'other': test} becomes: name: a: 1 b: 2 other: test --- src/mailman/testing/documentation.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'src/mailman') diff --git a/src/mailman/testing/documentation.py b/src/mailman/testing/documentation.py index d460be89f..421a272f4 100644 --- a/src/mailman/testing/documentation.py +++ b/src/mailman/testing/documentation.py @@ -109,6 +109,13 @@ def call_http(url, data=None, method=None, username=None, password=None): return content +def _print_dict(data, depth=0): + for item, value in sorted(data.items()): + if isinstance(value, dict): + _print_dict(value, depth+1) + print(' '*depth + '{}: {}'.format(item, value)) + + def dump_json(url, data=None, method=None, username=None, password=None): """Print the JSON dictionary read from a URL. @@ -140,6 +147,9 @@ def dump_json(url, data=None, method=None, username=None, password=None): printable_value = COMMASPACE.join( "'{}'".format(s) for s in sorted(value)) print('{}: [{}]'.format(key, printable_value)) + elif isinstance(value, dict): + print('{}:'.format(key)) + _print_dict(value, 1) else: print('{}: {}'.format(key, value)) -- cgit v1.2.3-70-g09d2 From a665dccf9404d6f95d8a4587f05d748b504e1f9d Mon Sep 17 00:00:00 2001 From: J08nY Date: Thu, 1 Jun 2017 20:53:53 +0200 Subject: Add REST-ability to plugins. - Adds the rest_object method to the IPlugin interface. This method is called by the REST api to route requests to the plugin's REST api. All attributes are proxied, so the object becomes a true REST object in the ObjectRouter hierarchy. For example see the RESTExample and ExamplePlugin classes in mailman.testing.plugin. The plugin might return None from this method, in which case all calls to it's REST api route will return 404 NotFound. - Adds new routes to the REST api. For version >= 3.1: - /3.1/plugins Returns a dictionary with plugin names as keys, and their configs as values. - /3.1/plugins/ Proxies everything to plugin's rest_object, if plugin with such name exists and provides a non-null rest_object, else NotFound. --- src/mailman/app/docs/plugins.rst | 3 + src/mailman/interfaces/plugin.py | 6 ++ src/mailman/rest/docs/plugins.rst | 59 +++++++++++++++++++ src/mailman/rest/plugins.py | 68 +++++++++++++++++++++ src/mailman/rest/root.py | 15 +++++ src/mailman/rest/tests/test_plugins.py | 104 +++++++++++++++++++++++++++++++++ src/mailman/testing/plugin.py | 69 +++++++++++++++++++++- 7 files changed, 322 insertions(+), 2 deletions(-) create mode 100644 src/mailman/rest/docs/plugins.rst create mode 100644 src/mailman/rest/plugins.py create mode 100644 src/mailman/rest/tests/test_plugins.py (limited to 'src/mailman') diff --git a/src/mailman/app/docs/plugins.rst b/src/mailman/app/docs/plugins.rst index 42038b654..a66e4413d 100644 --- a/src/mailman/app/docs/plugins.rst +++ b/src/mailman/app/docs/plugins.rst @@ -113,6 +113,9 @@ initialization process. ... global counter ... print('post-hook:', counter) ... counter += 1 + ... + ... def rest_object(self): + ... pass ... """, file=fp) >>> fp.close() diff --git a/src/mailman/interfaces/plugin.py b/src/mailman/interfaces/plugin.py index 9e226824f..f1b7212f0 100644 --- a/src/mailman/interfaces/plugin.py +++ b/src/mailman/interfaces/plugin.py @@ -30,3 +30,9 @@ class IPlugin(Interface): def post_hook(): """A plugin hook called in the second initialization step. After DB.""" + + def rest_object(): + """Return the REST object which will be hooked up to the REST api. + + This object will be served at the //plugins// root. + """ diff --git a/src/mailman/rest/docs/plugins.rst b/src/mailman/rest/docs/plugins.rst new file mode 100644 index 000000000..787726e55 --- /dev/null +++ b/src/mailman/rest/docs/plugins.rst @@ -0,0 +1,59 @@ +========= + Plugins +========= + +Plugins can supply REST routes. +:: + + >>> dump_json('http://localhost:9001/3.1/plugins') + entry 0: + class: mailman.testing.plugin.ExamplePlugin + enable: True + http_etag: "..." + name: example + path: + http_etag: "..." + start: 0 + total_size: 1 + + >>> dump_json('http://localhost:9001/3.1/plugins/example') + example-plugin-reply: yes + http_etag: "..." + + >>> dump_json('http://localhost:9001/3.1/plugins/example/good') + good: True + http_etag: "..." + + >>> dump_json('http://localhost:9001/3.1/plugins/example/bad') + Traceback (most recent call last): + ... + urllib.error.HTTPError: HTTP Error 400: ... + +For example this route that presents a counter. +:: + + >>> dump_json('http://localhost:9001/3.1/plugins/example/count') + count: 0 + http_etag: "..." + + >>> call_http('http://localhost:9001/3.1/plugins/example/count', + ... {'count':3}, method='POST') + content-length: 0 + date: ... + server: ... + status: 204 + + >>> dump_json('http://localhost:9001/3.1/plugins/example/count') + count: 3 + http_etag: "..." + + >>> call_http('http://localhost:9001/3.1/plugins/example/count', + ... method='DELETE') + content-length: 0 + date: ... + server: ... + status: 204 + + >>> dump_json('http://localhost:9001/3.1/plugins/example/count') + count: 0 + http_etag: "..." diff --git a/src/mailman/rest/plugins.py b/src/mailman/rest/plugins.py new file mode 100644 index 000000000..ac8fc0ace --- /dev/null +++ b/src/mailman/rest/plugins.py @@ -0,0 +1,68 @@ +# Copyright (C) 2010-2017 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""REST for plugins, dynamically proxies requests to plugin's rest_object.""" + +from lazr.config import as_boolean +from mailman.config import config +from mailman.rest.helpers import CollectionMixin, NotFound, etag, okay +from public import public + + +@public +class AllPlugins(CollectionMixin): + """Read-only list of all plugin configs.""" + + def _resource_as_dict(self, plugin_config): + """See `CollectionMixin`.""" + resource = { + 'name': plugin_config.name.split('.')[-1], + 'class': plugin_config['class'], + 'enable': as_boolean(plugin_config['enable']), + 'path': plugin_config['path'] + } + return resource + + def _get_collection(self, request): + """See `CollectionMixin`.""" + return config.plugin_configs + + def on_get(self, request, response): + """/plugins""" + resource = self._make_collection(request) + okay(response, etag(resource)) + + +@public +class APlugin: + """REST proxy to the plugins rest_object.""" + + def __init__(self, plugin_name): + self.__rest_object = None + if plugin_name in config.plugins.keys(): + plugin = config.plugins[plugin_name] + self.__rest_object = plugin.rest_object() + # If the plugin doesn't exist or doesn't provide a rest_object, + # just proxy to NotFound + if self.__rest_object is None: + self.__rest_object = NotFound() + + def __getattr__(self, attrib): + return getattr(self.__rest_object, attrib) + + def __dir__(self): + return dir(self.__rest_object) diff --git a/src/mailman/rest/root.py b/src/mailman/rest/root.py index 76289078b..46116aba6 100644 --- a/src/mailman/rest/root.py +++ b/src/mailman/rest/root.py @@ -30,6 +30,7 @@ from mailman.rest.helpers import ( BadRequest, NotFound, child, etag, no_content, not_found, okay) from mailman.rest.lists import AList, AllLists, Styles from mailman.rest.members import AMember, AllMembers, FindMembers +from mailman.rest.plugins import APlugin, AllPlugins from mailman.rest.preferences import ReadOnlyPreferences from mailman.rest.queues import AQueue, AQueueFile, AllQueues from mailman.rest.templates import TemplateFinder @@ -307,6 +308,20 @@ class TopLevel: else: return BadRequest(), [] + @child() + def plugins(self, context, segments): + """//plugins + //plugins/ + //plugins//... + """ + if self.api.version_info < (3, 1): + return NotFound(), [] + if len(segments) == 0: + return AllPlugins(), [] + else: + plugin_name = segments.pop(0) + return APlugin(plugin_name), segments + @child() def bans(self, context, segments): """//bans diff --git a/src/mailman/rest/tests/test_plugins.py b/src/mailman/rest/tests/test_plugins.py new file mode 100644 index 000000000..f6d7afb76 --- /dev/null +++ b/src/mailman/rest/tests/test_plugins.py @@ -0,0 +1,104 @@ +# Copyright (C) 2010-2017 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Test RESTability of plugins.""" + +import unittest + +from mailman.testing.helpers import call_api +from mailman.testing.layers import RESTLayer +from urllib.error import HTTPError + + +class TestAPI31Plugins(unittest.TestCase): + layer = RESTLayer + + def test_list_plugins(self): + json, response = call_api('http://localhost:9001/3.1/plugins') + self.assertEqual(response.status_code, 200) + json.pop('http_etag') + self.assertEqual(len(json.keys()), 3) + self.assertIn('entries', json.keys()) + entries = json['entries'] + self.assertEqual(len(entries), 1) + plugin_conf = entries[0] + self.assertEqual(plugin_conf['name'], 'example') + self.assertEqual(plugin_conf['enable'], True) + self.assertEqual(plugin_conf['class'], + 'mailman.testing.plugin.ExamplePlugin') + self.assertEqual(plugin_conf['path'], '') + + def test_non_existent_plugin(self): + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.1/plugins/does-not-exist') + self.assertEqual(cm.exception.code, 404) + + def test_example_plugin(self): + json, response = call_api('http://localhost:9001/3.1/plugins/example') + self.assertEqual(response.status_code, 200) + json.pop('http_etag') + self.assertEqual(len(json.keys()), 1) + self.assertEqual(json['example-plugin-reply'], 'yes') + + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.1/plugins/example', + method='POST') + self.assertEqual(cm.exception.code, 400) + + def test_example_plugin_child(self): + json, response = \ + call_api('http://localhost:9001/3.1/plugins/example/good') + self.assertEqual(response.status_code, 200) + json.pop('http_etag') + self.assertEqual(len(json.keys()), 1) + self.assertEqual(json['good'], True) + + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.1/plugins/example/bad') + self.assertEqual(cm.exception.code, 400) + + def test_example_plugin_counter(self): + json, response = \ + call_api('http://localhost:9001/3.1/plugins/example/count') + self.assertEqual(response.status_code, 200) + json.pop('http_etag') + self.assertEqual(len(json.keys()), 1) + self.assertEqual(json['count'], 0) + + json, response = \ + call_api('http://localhost:9001/3.1/plugins/example/count', + method='POST', data={'count': 5}) + self.assertEqual(response.status_code, 204) + + json, response = \ + call_api('http://localhost:9001/3.1/plugins/example/count') + self.assertEqual(response.status_code, 200) + json.pop('http_etag') + self.assertEqual(len(json.keys()), 1) + self.assertEqual(json['count'], 5) + + json, response = \ + call_api('http://localhost:9001/3.1/plugins/example/count', + method='DELETE') + self.assertEqual(response.status_code, 204) + + json, response = \ + call_api('http://localhost:9001/3.1/plugins/example/count') + self.assertEqual(response.status_code, 200) + json.pop('http_etag') + self.assertEqual(len(json.keys()), 1) + self.assertEqual(json['count'], 0) diff --git a/src/mailman/testing/plugin.py b/src/mailman/testing/plugin.py index 75667844a..a8b001406 100644 --- a/src/mailman/testing/plugin.py +++ b/src/mailman/testing/plugin.py @@ -15,22 +15,87 @@ # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see . -"""Example plugin with hooks for testing purposes.""" +"""Example plugin with hooks and REST for testing purposes.""" +from mailman.config import config from mailman.interfaces.plugin import IPlugin +from mailman.rest.helpers import bad_request, child, etag, no_content, okay +from mailman.rest.validator import Validator from public import public from zope.interface import implementer +@public +class GoodREST: + + def on_get(self, request, response): + okay(response, etag({'good': True})) + + +@public +class BadREST: + + def on_get(self, request, response): + bad_request(response, etag({'good': False})) + + +@public +class CountingREST: + + def __init__(self): + self._plugin = config.plugins['example'] + + def on_get(self, request, response): + okay(response, etag({'count': self._plugin.counter})) + + def on_post(self, request, response): + try: + counter = Validator(count=int)(request) + self._plugin.counter = counter['count'] + except ValueError as error: + bad_request(response, str(error)) + else: + no_content(response) + + def on_delete(self, request, response): + self._plugin.counter = 0 + no_content(response) + + +@public +class RESTExample: + + def on_get(self, request, response): + okay(response, etag({'example-plugin-reply': 'yes'})) + + def on_post(self, request, response): + bad_request(response) + + @child() + def good(self, context, segments): + return GoodREST(), [] + + @child() + def bad(self, context, segments): + return BadREST(), [] + + @child() + def count(self, context, segments): + return CountingREST(), [] + + @public @implementer(IPlugin) class ExamplePlugin: def __init__(self): - pass + self.counter = 0 def pre_hook(self): pass def post_hook(self): pass + + def rest_object(self): + return RESTExample() -- cgit v1.2.3-70-g09d2 From 260733a13a0ad891e1b10836399dd02af7335fe0 Mon Sep 17 00:00:00 2001 From: J08nY Date: Tue, 6 Jun 2017 01:05:08 +0200 Subject: Add optional external configuration to plugins. - Adds an optional 'conriguration' option similar to the one in [mta] which plugins can use to load their specific configs from. --- src/mailman/config/schema.cfg | 8 ++++++++ src/mailman/rest/docs/plugins.rst | 1 + src/mailman/rest/plugins.py | 3 ++- src/mailman/rest/tests/test_plugins.py | 1 + 4 files changed, 12 insertions(+), 1 deletion(-) (limited to 'src/mailman') diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg index 0e7936943..cae95a07f 100644 --- a/src/mailman/config/schema.cfg +++ b/src/mailman/config/schema.cfg @@ -91,6 +91,14 @@ class: # Whether to enable this plugin or not. enable: no +# Additional config for this plugin. The path can be either a file system path +# or a Python import path. If the value starts with python: then it is +# a Python import path, otherwise it is a file system path. File system paths +# must be absolute since no guarantees are made about the current working +# directory. Python paths should not include the trailing .cfg, which the file +# must end with. +configuration: + [shell] # `mailman shell` (also `withlist`) gives you an interactive prompt that you diff --git a/src/mailman/rest/docs/plugins.rst b/src/mailman/rest/docs/plugins.rst index 787726e55..737a911c0 100644 --- a/src/mailman/rest/docs/plugins.rst +++ b/src/mailman/rest/docs/plugins.rst @@ -8,6 +8,7 @@ Plugins can supply REST routes. >>> dump_json('http://localhost:9001/3.1/plugins') entry 0: class: mailman.testing.plugin.ExamplePlugin + configuration: enable: True http_etag: "..." name: example diff --git a/src/mailman/rest/plugins.py b/src/mailman/rest/plugins.py index ac8fc0ace..2209e634e 100644 --- a/src/mailman/rest/plugins.py +++ b/src/mailman/rest/plugins.py @@ -33,7 +33,8 @@ class AllPlugins(CollectionMixin): 'name': plugin_config.name.split('.')[-1], 'class': plugin_config['class'], 'enable': as_boolean(plugin_config['enable']), - 'path': plugin_config['path'] + 'path': plugin_config['path'], + 'configuration': plugin_config['configuration'] } return resource diff --git a/src/mailman/rest/tests/test_plugins.py b/src/mailman/rest/tests/test_plugins.py index f6d7afb76..9c12e1160 100644 --- a/src/mailman/rest/tests/test_plugins.py +++ b/src/mailman/rest/tests/test_plugins.py @@ -41,6 +41,7 @@ class TestAPI31Plugins(unittest.TestCase): self.assertEqual(plugin_conf['class'], 'mailman.testing.plugin.ExamplePlugin') self.assertEqual(plugin_conf['path'], '') + self.assertEqual(plugin_conf['configuration'], '') def test_non_existent_plugin(self): with self.assertRaises(HTTPError) as cm: -- cgit v1.2.3-70-g09d2 From a2668307017e8b3a0c705e5cff49a8fa7dbeac7d Mon Sep 17 00:00:00 2001 From: J08nY Date: Tue, 6 Jun 2017 15:51:10 +0200 Subject: Make config.plugin_configs yield a dict with plugin names. - Allows to better loop over pluging configs and their names. --- src/mailman/config/config.py | 6 ++++-- src/mailman/core/plugins.py | 3 +-- src/mailman/rest/plugins.py | 13 +++++++------ src/mailman/utilities/plugins.py | 2 +- 4 files changed, 13 insertions(+), 11 deletions(-) (limited to 'src/mailman') diff --git a/src/mailman/config/config.py b/src/mailman/config/config.py index 405b689ef..a5e3ed3a7 100644 --- a/src/mailman/config/config.py +++ b/src/mailman/config/config.py @@ -250,8 +250,10 @@ class Configuration: @property def plugin_configs(self): - """Iterate over all the plugin configuration sections.""" - return self._config.getByCategory('plugin', []) + """Return all the plugin configuration sections.""" + plugin_sections = self._config.getByCategory('plugin', []) + for section in plugin_sections: + yield section.category_and_section_names[1], section @property def language_configs(self): diff --git a/src/mailman/core/plugins.py b/src/mailman/core/plugins.py index 565c590b5..e891f380d 100644 --- a/src/mailman/core/plugins.py +++ b/src/mailman/core/plugins.py @@ -28,13 +28,12 @@ from zope.interface.verify import verifyObject @public def initialize(): """Initialize all enabled plugins.""" - for plugin_config in config.plugin_configs: + for name, plugin_config in config.plugin_configs: plugin_class_path = plugin_config['class'] if as_boolean(plugin_config.enable) and plugin_class_path: plugin_class = find_name(plugin_class_path) plugin = plugin_class() verifyObject(IPlugin, plugin) - name = plugin_config.name.split('.')[-1] plugin.name = name assert plugin.name not in config.plugins, ( 'Duplicate plugin "{}" found in {}'.format( diff --git a/src/mailman/rest/plugins.py b/src/mailman/rest/plugins.py index 2209e634e..fa2605f24 100644 --- a/src/mailman/rest/plugins.py +++ b/src/mailman/rest/plugins.py @@ -29,18 +29,19 @@ class AllPlugins(CollectionMixin): def _resource_as_dict(self, plugin_config): """See `CollectionMixin`.""" + name, plugin_section = plugin_config resource = { - 'name': plugin_config.name.split('.')[-1], - 'class': plugin_config['class'], - 'enable': as_boolean(plugin_config['enable']), - 'path': plugin_config['path'], - 'configuration': plugin_config['configuration'] + 'name': name, + 'class': plugin_section['class'], + 'enable': as_boolean(plugin_section['enable']), + 'path': plugin_section['path'], + 'configuration': plugin_section['configuration'] } return resource def _get_collection(self, request): """See `CollectionMixin`.""" - return config.plugin_configs + return sorted(config.plugin_configs) def on_get(self, request, response): """/plugins""" diff --git a/src/mailman/utilities/plugins.py b/src/mailman/utilities/plugins.py index a8449af31..98a34fbca 100644 --- a/src/mailman/utilities/plugins.py +++ b/src/mailman/utilities/plugins.py @@ -38,7 +38,7 @@ def find_pluggable_components(subpackage, interface): """ yield from find_components('mailman.' + subpackage, interface) package_roots = [plugin.path - for plugin in config.plugin_configs + for name, plugin in config.plugin_configs if as_boolean(plugin.enable) and plugin.path] for package_root in package_roots: package = package_root + '.' + subpackage -- cgit v1.2.3-70-g09d2 From 0923455ba337fa734f89b3be72bb7fde061dcc91 Mon Sep 17 00:00:00 2001 From: J08nY Date: Mon, 19 Jun 2017 15:04:12 +0200 Subject: Don't fail when plugins pre_hook or post_hook fails. - Catches all exceptions raised when running a plugins (pre|post) _hooks. If its pre_hook raises an exception, the plugin is disabled and removed, so its post_hook will not be run and its components will not be loaded. If its post_hook raises an exception we just continue and hope for the best. --- src/mailman/core/initialize.py | 43 ++++++++++++++++++++++++++++++---------- src/mailman/utilities/plugins.py | 6 +++--- 2 files changed, 36 insertions(+), 13 deletions(-) (limited to 'src/mailman') diff --git a/src/mailman/core/initialize.py b/src/mailman/core/initialize.py index 18e874552..a1314b51e 100644 --- a/src/mailman/core/initialize.py +++ b/src/mailman/core/initialize.py @@ -27,6 +27,7 @@ by the command line arguments. import os import sys import warnings +import traceback import mailman.config.config import mailman.core.logging @@ -149,14 +150,30 @@ def initialize_2(debug=False, propagate_logs=None, testing=False): # Initialize plugins from mailman.core.plugins import initialize as initialize_plugins initialize_plugins() - # Run the plugin pre_hooks + # Check for deprecated features in config. config = mailman.config.config - if "pre_hook" in config.mailman: # pragma: no cover + if 'pre_hook' in config.mailman: # pragma: no cover warnings.warn( - "The pre_hook configuration value has been replaced by the " - "plugins infrastructure.", DeprecationWarning) - for plugin in config.plugins.values(): - plugin.pre_hook() + 'The pre_hook configuration value has been replaced by the ' + 'plugins infrastructure.', DeprecationWarning) + # Run the plugin pre_hooks, if one fails, disable the offending plugin. + for name in list(config.plugins.keys()): + plugin = config.plugins[name] + try: + plugin.pre_hook() + except BaseException: # pragma: no cover + traceback.print_exc() + warnings.warn('Plugin {} failed to run its pre_hook,' + 'it will be disabled and its components' + 'wont be loaded.'.format(name), RuntimeWarning) + # It failed, push disabling overlay to config. This will stop + # components from being loaded. + config.push(name + '_error', """\ +[plugin.{}] +enable: no + """.format(name)) + # And forget about it. This will stop running its pot_hook. + del config.plugins[name] # Instantiate the database class, ensure that it's of the right type, and # initialize it. Then stash the object on our configuration object. utility_name = ('testing' if testing else 'production') @@ -182,12 +199,18 @@ def initialize_3(): """ # Run the plugin post_hooks config = mailman.config.config - if "post_hook" in config.mailman: # pragma: no cover + if 'post_hook' in config.mailman: # pragma: no cover warnings.warn( - "The post_hook configuration value has been replaced by the " - "plugins infrastructure.", DeprecationWarning) + 'The post_hook configuration value has been replaced by the ' + 'plugins infrastructure.', DeprecationWarning) for plugin in config.plugins.values(): - plugin.post_hook() + try: + plugin.post_hook() + except: # pragma: no cover + # A post_hook may fail, here we just hope for the best that the + # plugin can work even if it post_hook failed as it's components + # are already loaded. + pass @public diff --git a/src/mailman/utilities/plugins.py b/src/mailman/utilities/plugins.py index 98a34fbca..3fb5e49bc 100644 --- a/src/mailman/utilities/plugins.py +++ b/src/mailman/utilities/plugins.py @@ -57,8 +57,8 @@ def add_pluggable_components(subpackage, interface, mapping): object's `.name` attribute, which is required. It is a fatal error if that key already exists in the mapping. - :param package: The subpackage path to search. - :type package: string + :param subpackage: The subpackage path to search. + :type subpackage: string :param interface: The interface that returned objects must conform to. Objects found must have a `.name` attribute containing a unique string. @@ -70,7 +70,7 @@ def add_pluggable_components(subpackage, interface, mapping): """ for component in find_pluggable_components(subpackage, interface): if component.name in mapping: - raise RuntimeError( + raise RuntimeError( # pragma: no cover 'Duplicate key "{}" found in {}; previously {}'.format( component.name, component, mapping[component.name])) mapping[component.name] = component -- cgit v1.2.3-70-g09d2 From 7afec8975cb2c8fa1c6725176f52e100d396d1c4 Mon Sep 17 00:00:00 2001 From: J08nY Date: Tue, 27 Jun 2017 16:42:33 +0200 Subject: Add new logger which plugins can use. --- src/mailman/commands/docs/conf.rst | 1 + src/mailman/config/schema.cfg | 4 ++++ src/mailman/rest/tests/test_systemconf.py | 1 + 3 files changed, 6 insertions(+) (limited to 'src/mailman') diff --git a/src/mailman/commands/docs/conf.rst b/src/mailman/commands/docs/conf.rst index 1a6a4679d..9981b71ed 100644 --- a/src/mailman/commands/docs/conf.rst +++ b/src/mailman/commands/docs/conf.rst @@ -45,6 +45,7 @@ key, along with the names of the corresponding sections. [logging.http] path: mailman.log [logging.locks] path: mailman.log [logging.mischief] path: mailman.log + [logging.plugin] path: plugin.log [logging.root] path: mailman.log [logging.runner] path: mailman.log [logging.smtp] path: smtp.log diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg index cae95a07f..a55c37ff4 100644 --- a/src/mailman/config/schema.cfg +++ b/src/mailman/config/schema.cfg @@ -276,6 +276,7 @@ debug: no # - http -- Internal wsgi-based web interface # - locks -- Lock state changes # - mischief -- Various types of hostile activity +# - plugin -- Plugin logs # - runner -- Runner process start/stops # - smtp -- Successful SMTP activity # - smtp-failure -- Unsuccessful SMTP activity @@ -313,6 +314,9 @@ level: info [logging.mischief] +[logging.plugin] +path: plugin.log + [logging.runner] [logging.smtp] diff --git a/src/mailman/rest/tests/test_systemconf.py b/src/mailman/rest/tests/test_systemconf.py index 42802068c..fe5d951a8 100644 --- a/src/mailman/rest/tests/test_systemconf.py +++ b/src/mailman/rest/tests/test_systemconf.py @@ -166,6 +166,7 @@ class TestSystemConfiguration(unittest.TestCase): 'logging.http', 'logging.locks', 'logging.mischief', + 'logging.plugin', 'logging.root', 'logging.runner', 'logging.smtp', -- cgit v1.2.3-70-g09d2 From cc99242654f2bcc11b3a6f124908de4b175c48d4 Mon Sep 17 00:00:00 2001 From: J08nY Date: Wed, 28 Jun 2017 14:28:21 +0200 Subject: Ignore plugin started subprocesses in master main loop. --- src/mailman/bin/master.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src/mailman') diff --git a/src/mailman/bin/master.py b/src/mailman/bin/master.py index 0b273d332..a23fb8e0b 100644 --- a/src/mailman/bin/master.py +++ b/src/mailman/bin/master.py @@ -188,6 +188,9 @@ class PIDWatcher: def __init__(self): self._pids = {} + def __contains__(self, pid): + return pid in self._pids.keys() + def __iter__(self): # Safely iterate over all the keys in the dictionary. Because # asynchronous signals are involved, the dictionary's size could @@ -402,6 +405,10 @@ class Loop: except InterruptedError: # pragma: nocover # If the system call got interrupted, just restart it. continue + if pid not in self._kids: + # Not a runner subprocess, maybe a plugin started one + # ignore it + continue # Find out why the subprocess exited by getting the signal # received or exit status. if os.WIFSIGNALED(status): -- cgit v1.2.3-70-g09d2 From 8addebbf9802e911c06f6a27b7ffff1e0f1d2e57 Mon Sep 17 00:00:00 2001 From: J08nY Date: Tue, 25 Jul 2017 21:44:54 +0200 Subject: Fix coverage, deprecate, but run non-plugin (post|pre)_hooks. --- src/mailman/app/docs/plugins.rst | 50 +++++++++++++++++++++++++++---- src/mailman/bin/mailman.py | 6 ++-- src/mailman/bin/master.py | 2 +- src/mailman/bin/tests/test_mailman.py | 18 +++++++++-- src/mailman/config/schema.cfg | 9 +++++- src/mailman/core/initialize.py | 28 +++++++++-------- src/mailman/rest/docs/systemconf.rst | 2 ++ src/mailman/rest/tests/test_systemconf.py | 2 ++ src/mailman/utilities/plugins.py | 2 +- 9 files changed, 94 insertions(+), 25 deletions(-) (limited to 'src/mailman') diff --git a/src/mailman/app/docs/plugins.rst b/src/mailman/app/docs/plugins.rst index a66e4413d..4430179ae 100644 --- a/src/mailman/app/docs/plugins.rst +++ b/src/mailman/app/docs/plugins.rst @@ -117,7 +117,6 @@ initialization process. ... def rest_object(self): ... pass ... """, file=fp) - >>> fp.close() Running the hooks ----------------- @@ -143,12 +142,12 @@ script that will produce no output to force the hooks to run. >>> import subprocess >>> from mailman.testing.layers import ConfigLayer - >>> def call(): + >>> def call(cfg_path, python_path): ... exe = os.path.join(os.path.dirname(sys.executable), 'mailman') ... env = os.environ.copy() ... env.update( - ... MAILMAN_CONFIG_FILE=config_path, - ... PYTHONPATH=config_directory, + ... MAILMAN_CONFIG_FILE=cfg_path, + ... PYTHONPATH=python_path, ... ) ... test_cfg = os.environ.get('MAILMAN_EXTRA_TESTING_CFG') ... if test_cfg is not None: @@ -161,10 +160,51 @@ script that will produce no output to force the hooks to run. ... stdout, stderr = proc.communicate() ... assert proc.returncode == 0, stderr ... print(stdout) + ... print(stderr) - >>> call() + >>> call(config_path, config_directory) pre-hook: 1 post-hook: 2 + >>> os.remove(config_path) + +Deprecated hooks +---------------- + +The old-style `pre_hook` and `post_hook` callables are deprecated and are no +longer called upon startup. +:: + + >>> deprecated_hook_path = os.path.join(config_directory, 'deprecated_hooks.py') + >>> with open(deprecated_hook_path, 'w') as fp: + ... print("""\ + ... def do_something(): + ... print("does something") + ... + ... def do_something_else(): + ... print("does something else") + ... + ... """, file=fp) + + >>> deprecated_config_path = os.path.join(config_directory, 'deprecated.cfg') + >>> with open(deprecated_config_path, 'w') as fp: + ... print("""\ + ... [meta] + ... extends: test.cfg + ... + ... [mailman] + ... pre_hook: deprecated_hooks.do_something + ... post_hook: deprecated_hooks.do_something_else + ... """, file=fp) + + >>> call(deprecated_config_path, config_directory) + does something + does something else + + ... UserWarning: The pre_hook configuration value has been replaced by the plugins infrastructure. ... + ... UserWarning: The post_hook configuration value has been replaced by the plugins infrastructure. ... + + + >>> os.remove(deprecated_config_path) diff --git a/src/mailman/bin/mailman.py b/src/mailman/bin/mailman.py index ece223180..f8006218d 100644 --- a/src/mailman/bin/mailman.py +++ b/src/mailman/bin/mailman.py @@ -44,9 +44,9 @@ class Subcommands(click.MultiCommand): self._commands) self._loaded = True - def list_commands(self, ctx): + def list_commands(self, ctx): # pragma: nocover self._load() - return sorted(self._commands) # pragma: nocover + return sorted(self._commands) def get_command(self, ctx, name): self._load() @@ -88,7 +88,7 @@ class Subcommands(click.MultiCommand): def initialize_config(ctx, param, value): - if ctx.resilient_parsing: + if ctx.resilient_parsing: # pragma: nocover return initialize(value) diff --git a/src/mailman/bin/master.py b/src/mailman/bin/master.py index a23fb8e0b..ccaa16398 100644 --- a/src/mailman/bin/master.py +++ b/src/mailman/bin/master.py @@ -405,7 +405,7 @@ class Loop: except InterruptedError: # pragma: nocover # If the system call got interrupted, just restart it. continue - if pid not in self._kids: + if pid not in self._kids: # pragma: nocover # Not a runner subprocess, maybe a plugin started one # ignore it continue diff --git a/src/mailman/bin/tests/test_mailman.py b/src/mailman/bin/tests/test_mailman.py index 54ee54bce..196437291 100644 --- a/src/mailman/bin/tests/test_mailman.py +++ b/src/mailman/bin/tests/test_mailman.py @@ -27,6 +27,7 @@ from mailman.config import config from mailman.database.transaction import transaction from mailman.testing.layers import ConfigLayer from mailman.utilities.datetime import now +from pkg_resources import resource_filename from unittest.mock import patch @@ -36,7 +37,19 @@ class TestMailmanCommand(unittest.TestCase): def setUp(self): self._command = CliRunner() - def test_mailman_command_without_subcommand_prints_help(self): + def test_mailman_command_config(self): + config_path = resource_filename('mailman.testing', 'testing.cfg') + with patch('mailman.bin.mailman.initialize') as init: + self._command.invoke(main, ('-C', config_path, 'info')) + init.assert_called_once_with(config_path) + + def test_mailman_command_no_config(self): + with patch('mailman.bin.mailman.initialize') as init: + self._command.invoke(main, ('info',)) + init.assert_called_once_with(None) + + @patch('mailman.bin.mailman.initialize') + def test_mailman_command_without_subcommand_prints_help(self, mock): # Issue #137: Running `mailman` without a subcommand raises an # AttributeError. result = self._command.invoke(main) @@ -46,7 +59,8 @@ class TestMailmanCommand(unittest.TestCase): # command line. self.assertEqual(lines[0], 'Usage: main [OPTIONS] COMMAND [ARGS]...') - def test_mailman_command_with_bad_subcommand_prints_help(self): + @patch('mailman.bin.mailman.initialize') + def test_mailman_command_with_bad_subcommand_prints_help(self, mock): # Issue #137: Running `mailman` without a subcommand raises an # AttributeError. result = self._command.invoke(main, ('not-a-subcommand',)) diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg index a55c37ff4..742bceae1 100644 --- a/src/mailman/config/schema.cfg +++ b/src/mailman/config/schema.cfg @@ -74,10 +74,17 @@ html_to_plain_text_command: /usr/bin/lynx -dump $filename # unpredictable. listname_chars: [-_.0-9a-z] +# Deprecated, callable run before DB initialization. +pre_hook: + +# Deprecated, callable run after DB initialization. +post_hook: + + [plugin.master] # Plugin package # It's sub packages will be searched for components. -# - commands for IEmailCommand +# - commands for IEmailCommand and ICliSubCommand # - chains for IChain # - rules for IRule # - pipelines for IPipeline diff --git a/src/mailman/core/initialize.py b/src/mailman/core/initialize.py index a1314b51e..830bd9b84 100644 --- a/src/mailman/core/initialize.py +++ b/src/mailman/core/initialize.py @@ -31,7 +31,9 @@ import traceback import mailman.config.config import mailman.core.logging +from mailman.core.i18n import _ from mailman.interfaces.database import IDatabaseFactory +from mailman.utilities.modules import call_name from pkg_resources import resource_string as resource_bytes from public import public from zope.component import getUtility @@ -152,27 +154,28 @@ def initialize_2(debug=False, propagate_logs=None, testing=False): initialize_plugins() # Check for deprecated features in config. config = mailman.config.config - if 'pre_hook' in config.mailman: # pragma: no cover + if config.mailman.pre_hook: # pragma: nocover warnings.warn( - 'The pre_hook configuration value has been replaced by the ' - 'plugins infrastructure.', DeprecationWarning) + _('The pre_hook configuration value has been replaced by the ' + 'plugins infrastructure.'), UserWarning) + call_name(config.mailman.pre_hook) # Run the plugin pre_hooks, if one fails, disable the offending plugin. for name in list(config.plugins.keys()): plugin = config.plugins[name] try: plugin.pre_hook() - except BaseException: # pragma: no cover + except: # pragma: nocover traceback.print_exc() - warnings.warn('Plugin {} failed to run its pre_hook,' - 'it will be disabled and its components' - 'wont be loaded.'.format(name), RuntimeWarning) + warnings.warn(_('Plugin $name failed to run its pre_hook,' + 'it will be disabled and its components' + 'wont be loaded.'), RuntimeWarning) # It failed, push disabling overlay to config. This will stop # components from being loaded. config.push(name + '_error', """\ [plugin.{}] enable: no """.format(name)) - # And forget about it. This will stop running its pot_hook. + # And forget about it. This will stop running its post_hook. del config.plugins[name] # Instantiate the database class, ensure that it's of the right type, and # initialize it. Then stash the object on our configuration object. @@ -199,14 +202,15 @@ def initialize_3(): """ # Run the plugin post_hooks config = mailman.config.config - if 'post_hook' in config.mailman: # pragma: no cover + if config.mailman.post_hook: # pragma: nocover warnings.warn( - 'The post_hook configuration value has been replaced by the ' - 'plugins infrastructure.', DeprecationWarning) + _('The post_hook configuration value has been replaced by the ' + 'plugins infrastructure.'), UserWarning) + call_name(config.mailman.post_hook) for plugin in config.plugins.values(): try: plugin.post_hook() - except: # pragma: no cover + except: # pragma: nocover # A post_hook may fail, here we just hope for the best that the # plugin can work even if it post_hook failed as it's components # are already loaded. diff --git a/src/mailman/rest/docs/systemconf.rst b/src/mailman/rest/docs/systemconf.rst index f5c9b691a..a65b81ab8 100644 --- a/src/mailman/rest/docs/systemconf.rst +++ b/src/mailman/rest/docs/systemconf.rst @@ -24,6 +24,8 @@ You can also get all the values for a particular section, such as the listname_chars: [-_.0-9a-z] noreply_address: noreply pending_request_life: 3d + post_hook: + pre_hook: self_link: http://localhost:9001/3.0/system/configuration/mailman sender_headers: from from_ reply-to sender site_owner: noreply@example.com diff --git a/src/mailman/rest/tests/test_systemconf.py b/src/mailman/rest/tests/test_systemconf.py index fe5d951a8..59f0619a3 100644 --- a/src/mailman/rest/tests/test_systemconf.py +++ b/src/mailman/rest/tests/test_systemconf.py @@ -46,6 +46,8 @@ class TestSystemConfiguration(unittest.TestCase): listname_chars='[-_.0-9a-z]', noreply_address='noreply', pending_request_life='3d', + post_hook='', + pre_hook='', self_link='http://localhost:9001/3.0/system/configuration/mailman', sender_headers='from from_ reply-to sender', site_owner='noreply@example.com', diff --git a/src/mailman/utilities/plugins.py b/src/mailman/utilities/plugins.py index 3fb5e49bc..aad909056 100644 --- a/src/mailman/utilities/plugins.py +++ b/src/mailman/utilities/plugins.py @@ -70,7 +70,7 @@ def add_pluggable_components(subpackage, interface, mapping): """ for component in find_pluggable_components(subpackage, interface): if component.name in mapping: - raise RuntimeError( # pragma: no cover + raise RuntimeError( # pragma: nocover 'Duplicate key "{}" found in {}; previously {}'.format( component.name, component, mapping[component.name])) mapping[component.name] = component -- cgit v1.2.3-70-g09d2