20 files changed, 635 insertions, 0 deletions

diff --git a/src/mailman/plugins/__init__.py b/src/mailman/plugins/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/src/mailman/plugins/__init__.py
diff --git a/src/mailman/plugins/docs/__init__.py b/src/mailman/plugins/docs/__init__.py
new file mode 100644
index 000000000..0b3957648
--- /dev/null
+++ b/src/mailman/plugins/docs/__init__.py
@@ -0,0 +1,25 @@
+# Copyright (C) 2017 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""REST layer for plugins."""
+
+
+from mailman.plugins.testing.layer import PluginRESTLayer
+from public import public
+
+# For flufl.testing.
+public(layer=PluginRESTLayer)
diff --git a/src/mailman/plugins/docs/intro.rst b/src/mailman/plugins/docs/intro.rst
new file mode 100644
index 000000000..be6fb365b
--- /dev/null
+++ b/src/mailman/plugins/docs/intro.rst
@@ -0,0 +1,207 @@
+=========
+ Plugins
+=========
+
+Mailman defines a plugin as a Python package on ``sys.path`` that provides
+components matching the ``IPlugin`` interface.  ``IPlugin`` implementations
+can define a *pre-hook*, a *post-hook*, and a *REST resource*.  Plugins are
+enabled by adding a section to your ``mailman.cfg`` file, such as:
+
+.. literalinclude:: ../testing/hooks.cfg
+
+.. note::
+   Because of a `design limitation`_ in the underlying configuration library,
+   you cannot name a plugin "master".  Specifically you cannot define a
+   section in your ``mailman.cfg`` file named ``[plugin.master]``.
+
+We have such a configuration file handy.
+
+    >>> from pkg_resources import resource_filename
+    >>> config_file = resource_filename('mailman.plugins.testing', 'hooks.cfg')
+
+The section must at least define the class implementing the ``IPlugin``
+interface, using a Python dotted-name import path.  For the import to work,
+you must include the top-level directory on ``sys.path``.
+
+    >>> import os
+    >>> from pkg_resources import resource_filename
+    >>> plugin_path = os.path.join(os.path.dirname(
+    ...     resource_filename('mailman.plugins', '__init__.py')),
+    ...     'testing')
+
+
+Hooks
+=====
+
+Plugins can add initialization hooks, which will be run at two stages in the
+initialization process - one before the database is initialized and one after.
+These correspond to methods the plugin defines, a ``pre_hook()`` method and a
+``post_hook()`` method.  Each of these methods are optional.
+
+Here is a plugin that defines these hooks:
+
+.. literalinclude:: ../testing/example/hooks.py
+
+To illustrate how the hooks work, we'll invoke a simple Mailman command to be
+run in a subprocess.  The plugin itself supports debugging hooking invocation
+when an environment variable is set.
+
+    >>> proc = run(['-C', config_file, 'info'],
+    ...            DEBUG_HOOKS='1',
+    ...            PYTHONPATH=plugin_path)
+    >>> print(proc.stdout)
+    I'm in my pre-hook
+    I'm in my post-hook
+    ...
+
+
+Components
+==========
+
+Plugins can also add components such as rules, chains, list styles, etc.  By
+default, components are searched for in the package matching the plugin's
+name.  So in the case above, the plugin is named ``example`` (because the
+section is called ``[plugin.example]``, and there is a subpackage called
+``rules`` under the ``example`` package.  The file system layout looks like
+this::
+
+    example/
+        __init__.py
+        hooks.py
+        rules/
+            __init__.py
+            rules.py
+
+And the contents of ``rules.py`` looks like:
+
+.. literalinclude:: ../testing/example/rules/rules.py
+
+To see that the plugin's rule get added, we invoke Mailman as an external
+process, running a script that prints out all the defined rule names,
+including our plugin's ``example-rule``.
+
+    >>> proc = run(['-C', config_file, 'withlist', '-r', 'showrules'],
+    ...            PYTHONPATH=plugin_path)
+    >>> print(proc.stdout)
+    administrivia
+    ...
+    example-rule
+    ...
+
+Component directories can live under any importable path, not just one named
+after the plugin.  By adding a ``component_package`` section to your plugin's
+configuration, you can name an alternative location to search for components.
+
+.. literalinclude:: ../testing/alternate.cfg
+
+We use this configuration file and the following file system layout::
+
+    example/
+        __init__.py
+        hooks.py
+    alternate/
+        rules/
+            __init__.py
+            rules.py
+
+Here, ``rules.py`` likes like:
+
+.. literalinclude:: ../testing/alternate/rules/rules.py
+
+You can see that this rule has a different name.  If we use the
+``alternate.cfg`` configuration file from above::
+
+    >>> config_file = resource_filename(
+    ...     'mailman.plugins.testing', 'alternate.cfg')
+
+we'll pick up the alternate rule when we print them out.
+
+    >>> proc = run(['-C', config_file, 'withlist', '-r', 'showrules'],
+    ...            PYTHONPATH=plugin_path)
+    >>> print(proc.stdout)
+    administrivia
+    alternate-rule
+    ...
+
+
+REST
+====
+
+Plugins can also supply REST routes.  Let's say we have a plugin defined like
+so:
+
+.. literalinclude:: ../testing/example/rest.py
+
+which we can enable with the following configuration file:
+
+.. literalinclude:: ../testing/rest.cfg
+
+The plugin defines a ``resource`` attribute that exposes the root of the
+plugin's resource tree.  The plugin will show up when we navigate to the
+``plugin`` resource.
+::
+
+    >>> dump_json('http://localhost:9001/3.1/plugins')
+    entry 0:
+        class: example.rest.ExamplePlugin
+        enabled: True
+        http_etag: "..."
+        name: example
+    http_etag: "..."
+    start: 0
+    total_size: 1
+
+The plugin may provide a ``GET`` on the resource itself.
+::
+
+    >>> dump_json('http://localhost:9001/3.1/plugins/example')
+    http_etag: "..."
+    my-child-resources: yes, no, echo
+    my-name: example-plugin
+
+And it may provide child resources.
+::
+
+    >>> dump_json('http://localhost:9001/3.1/plugins/example/yes')
+    http_etag: "..."
+    yes: True
+
+Plugins and their child resources can support any HTTP method, such as
+``GET``...
+::
+
+    >>> dump_json('http://localhost:9001/3.1/plugins/example/echo')
+    http_etag: "..."
+    number: 0
+
+... or ``POST`` ...
+::
+
+    >>> dump_json('http://localhost:9001/3.1/plugins/example/echo',
+    ...           dict(number=7))
+    content-length: 0
+    date: ...
+    server: ...
+    status: 204
+
+    >>> dump_json('http://localhost:9001/3.1/plugins/example/echo')
+    http_etag: "..."
+    number: 7
+
+... or ``DELETE``.
+
+    >>> dump_json('http://localhost:9001/3.1/plugins/example/echo',
+    ...           method='DELETE')
+    content-length: 0
+    date: ...
+    server: ...
+    status: 204
+
+    >>> dump_json('http://localhost:9001/3.1/plugins/example/echo')
+    http_etag: "..."
+    number: 0
+
+It's up to the plugin of course.
+
+
+.. _`design limitation`: https://bugs.launchpad.net/lazr.config/+bug/310619
diff --git a/src/mailman/plugins/initialize.py b/src/mailman/plugins/initialize.py
new file mode 100644
index 000000000..431f69a6f
--- /dev/null
+++ b/src/mailman/plugins/initialize.py
@@ -0,0 +1,54 @@
+# Copyright (C) 2017 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Initialize the plugins."""
+
+import logging
+
+from lazr.config import as_boolean
+from mailman.config import config
+from mailman.interfaces.plugin import IPlugin
+from mailman.utilities.modules import call_name
+from public import public
+from zope.interface.exceptions import DoesNotImplement
+from zope.interface.verify import verifyObject
+
+
+log = logging.getLogger('mailman.plugins')
+
+
+@public
+def initialize():
+    """Initialize all enabled plugins."""
+    for name, plugin_config in config.plugin_configs:
+        class_path = plugin_config['class'].strip()
+        if not as_boolean(plugin_config['enabled']) or len(class_path) == 0:
+            log.info('Plugin not enabled, or empty class path: {}'.format(
+                name))
+            continue
+        if name in config.plugins:
+            log.error('Duplicate plugin name: {}'.format(name))
+            continue
+        plugin = call_name(class_path)
+        try:
+            verifyObject(IPlugin, plugin)
+        except DoesNotImplement:
+            log.error('Plugin class does not implement IPlugin: {}'.format(
+                class_path))
+            continue
+        plugin.name = name
+        config.plugins[name] = plugin
diff --git a/src/mailman/plugins/testing/__init__.py b/src/mailman/plugins/testing/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/src/mailman/plugins/testing/__init__.py
diff --git a/src/mailman/plugins/testing/alternate.cfg b/src/mailman/plugins/testing/alternate.cfg
new file mode 100644
index 000000000..6cf6ea4f3
--- /dev/null
+++ b/src/mailman/plugins/testing/alternate.cfg
@@ -0,0 +1,7 @@
+[plugin.example]
+class: example.hooks.ExamplePlugin
+enabled: yes
+component_package: alternate
+
+[logging.plugins]
+propagate: yes
diff --git a/src/mailman/plugins/testing/alternate/__init__.py b/src/mailman/plugins/testing/alternate/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/src/mailman/plugins/testing/alternate/__init__.py
diff --git a/src/mailman/plugins/testing/alternate/rules/__init__.py b/src/mailman/plugins/testing/alternate/rules/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/src/mailman/plugins/testing/alternate/rules/__init__.py
diff --git a/src/mailman/plugins/testing/alternate/rules/rules.py b/src/mailman/plugins/testing/alternate/rules/rules.py
new file mode 100644
index 000000000..f8b8d725c
--- /dev/null
+++ b/src/mailman/plugins/testing/alternate/rules/rules.py
@@ -0,0 +1,14 @@
+from mailman.interfaces.rules import IRule
+from public import public
+from zope.interface import implementer
+
+
+@public
+@implementer(IRule)
+class AlternateRule:
+    name = 'alternate-rule'
+    description = 'An alternate rule.'
+    record = True
+
+    def check(self, mlist, msg, msgdata):
+        return 'alternate' in msgdata
diff --git a/src/mailman/plugins/testing/example/__init__.py b/src/mailman/plugins/testing/example/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/src/mailman/plugins/testing/example/__init__.py
diff --git a/src/mailman/plugins/testing/example/hooks.py b/src/mailman/plugins/testing/example/hooks.py
new file mode 100644
index 000000000..55c09c4a5
--- /dev/null
+++ b/src/mailman/plugins/testing/example/hooks.py
@@ -0,0 +1,21 @@
+import os
+
+from mailman.interfaces.plugin import IPlugin
+from public import public
+from zope.interface import implementer
+
+
+@public
+@implementer(IPlugin)
+class ExamplePlugin:
+    def pre_hook(self):
+        if os.environ.get('DEBUG_HOOKS'):
+            print("I'm in my pre-hook")
+
+    def post_hook(self):
+        if os.environ.get('DEBUG_HOOKS'):
+            print("I'm in my post-hook")
+
+    @property
+    def resource(self):
+        return None
diff --git a/src/mailman/plugins/testing/example/rest.py b/src/mailman/plugins/testing/example/rest.py
new file mode 100644
index 000000000..1d83ce2cd
--- /dev/null
+++ b/src/mailman/plugins/testing/example/rest.py
@@ -0,0 +1,79 @@
+from mailman.config import config
+from mailman.interfaces.plugin import IPlugin
+from mailman.rest.helpers import bad_request, child, etag, no_content, okay
+from mailman.rest.validator import Validator
+from public import public
+from zope.interface import implementer
+
+
+@public
+class Yes:
+    def on_get(self, request, response):
+        okay(response, etag(dict(yes=True)))
+
+
+@public
+class No:
+    def on_get(self, request, response):
+        bad_request(response, etag(dict(no=False)))
+
+
+@public
+class NumberEcho:
+    def __init__(self):
+        self._plugin = config.plugins['example']
+
+    def on_get(self, request, response):
+        okay(response, etag(dict(number=self._plugin.number)))
+
+    def on_post(self, request, response):
+        try:
+            resource = Validator(number=int)(request)
+            self._plugin.number = resource['number']
+        except ValueError as error:
+            bad_request(response, str(error))
+        else:
+            no_content(response)
+
+    def on_delete(self, request, response):
+        self._plugin.number = 0
+        no_content(response)
+
+
+@public
+class RESTExample:
+    def on_get(self, request, response):
+        resource = {
+            'my-name': 'example-plugin',
+            'my-child-resources': 'yes, no, echo',
+            }
+        okay(response, etag(resource))
+
+    @child()
+    def yes(self, context, segments):
+        return Yes(), []
+
+    @child()
+    def no(self, context, segments):
+        return No(), []
+
+    @child()
+    def echo(self, context, segments):
+        return NumberEcho(), []
+
+
+@public
+@implementer(IPlugin)
+class ExamplePlugin:
+    def __init__(self):
+        self.number = 0
+
+    def pre_hook(self):
+        pass
+
+    def post_hook(self):
+        pass
+
+    @property
+    def resource(self):
+        return RESTExample()
diff --git a/src/mailman/plugins/testing/example/rules/__init__.py b/src/mailman/plugins/testing/example/rules/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/src/mailman/plugins/testing/example/rules/__init__.py
diff --git a/src/mailman/plugins/testing/example/rules/rules.py b/src/mailman/plugins/testing/example/rules/rules.py
new file mode 100644
index 000000000..7e9ba6ce7
--- /dev/null
+++ b/src/mailman/plugins/testing/example/rules/rules.py
@@ -0,0 +1,14 @@
+from mailman.interfaces.rules import IRule
+from public import public
+from zope.interface import implementer
+
+
+@public
+@implementer(IRule)
+class ExampleRule:
+    name = 'example-rule'
+    description = 'An example rule.'
+    record = True
+
+    def check(self, mlist, msg, msgdata):
+        return 'example' in msgdata
diff --git a/src/mailman/plugins/testing/hooks.cfg b/src/mailman/plugins/testing/hooks.cfg
new file mode 100644
index 000000000..15fb31815
--- /dev/null
+++ b/src/mailman/plugins/testing/hooks.cfg
@@ -0,0 +1,3 @@
+[plugin.example]
+class: example.hooks.ExamplePlugin
+enabled: yes
diff --git a/src/mailman/plugins/testing/layer.py b/src/mailman/plugins/testing/layer.py
new file mode 100644
index 000000000..311a419f4
--- /dev/null
+++ b/src/mailman/plugins/testing/layer.py
@@ -0,0 +1,53 @@
+# Copyright (C) 2017 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""REST layer for plugins."""
+
+import os
+
+from contextlib import ExitStack
+from mailman.testing.helpers import (
+    TestableMaster, hackenv, wait_for_webservice)
+from mailman.testing.layers import SMTPLayer
+from pkg_resources import resource_filename
+from public import public
+
+
+# Don't inherit from RESTLayer since layers get run in bottom up order,
+# meaning RESTLayer will get setUp() before this layer does, and that won't
+# use the configuration file we need it to use.
+@public
+class PluginRESTLayer(SMTPLayer):
+    @classmethod
+    def setUp(cls):
+        cls.resources = ExitStack()
+        plugin_path = os.path.join(
+            os.path.dirname(
+                resource_filename('mailman.plugins', '__init__.py')),
+            'testing')
+        config_file = resource_filename('mailman.plugins.testing', 'rest.cfg')
+        cls.resources.enter_context(
+            hackenv('MAILMAN_CONFIG_FILE', config_file))
+        cls.resources.enter_context(
+            hackenv('PYTHONPATH', plugin_path))
+        cls.server = TestableMaster(wait_for_webservice)
+        cls.server.start('rest')
+        cls.resources.callback(cls.server.stop)
+
+    @classmethod
+    def tearDown(cls):
+        cls.resources.close()
diff --git a/src/mailman/plugins/testing/rest.cfg b/src/mailman/plugins/testing/rest.cfg
new file mode 100644
index 000000000..a9b4b9bd9
--- /dev/null
+++ b/src/mailman/plugins/testing/rest.cfg
@@ -0,0 +1,6 @@
+[plugin.example]
+class: example.rest.ExamplePlugin
+enabled: yes
+
+[webservice]
+port: 9001
diff --git a/src/mailman/plugins/testing/showrules.py b/src/mailman/plugins/testing/showrules.py
new file mode 100644
index 000000000..926ef0f70
--- /dev/null
+++ b/src/mailman/plugins/testing/showrules.py
@@ -0,0 +1,6 @@
+from mailman.config import config
+
+
+def showrules():
+    for name in sorted(config.rules):
+        print(name)
diff --git a/src/mailman/plugins/tests/__init__.py b/src/mailman/plugins/tests/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/src/mailman/plugins/tests/__init__.py
diff --git a/src/mailman/plugins/tests/test_plugins.py b/src/mailman/plugins/tests/test_plugins.py
new file mode 100644
index 000000000..4b432f603
--- /dev/null
+++ b/src/mailman/plugins/tests/test_plugins.py
@@ -0,0 +1,146 @@
+# Copyright (C) 2017 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Test some additional plugin stuff."""
+
+import sys
+import unittest
+
+from contextlib import ExitStack
+from mailman.interfaces.plugin import IPlugin
+from mailman.plugins.initialize import initialize
+from mailman.plugins.testing.layer import PluginRESTLayer
+from mailman.testing.helpers import call_api
+from mailman.testing.layers import ConfigLayer
+from tempfile import TemporaryDirectory
+from types import SimpleNamespace
+from unittest.mock import patch
+from urllib.error import HTTPError
+from zope.interface import implementer
+
+
+class TestRESTPlugin(unittest.TestCase):
+    layer = PluginRESTLayer
+
+    def test_plugin_raises_exception(self):
+        with self.assertRaises(HTTPError) as cm:
+            call_api('http://localhost:9001/3.1/plugins/example/no')
+        self.assertEqual(cm.exception.code, 400)
+
+
+@implementer(IPlugin)
+class TestablePlugin:
+    def pre_hook(self):
+        pass
+
+    def post_hook(self):
+        pass
+
+    resource = None
+
+
+class TestInitializePlugins(unittest.TestCase):
+    layer = ConfigLayer
+
+    def test_duplicate_plugin_name(self):
+        with ExitStack() as resources:
+            system_path = resources.enter_context(TemporaryDirectory())
+            fake_plugin_config = {
+                'path': system_path,
+                'enabled': 'yes',
+                'class': 'ExamplePlugin',
+                }
+            log_mock = resources.enter_context(
+                patch('mailman.plugins.initialize.log'))
+            fake_mailman_config = SimpleNamespace(
+                plugin_configs=[('example', fake_plugin_config)],
+                plugins=['example'],
+                )
+            resources.enter_context(patch(
+                'mailman.plugins.initialize.config', fake_mailman_config))
+            initialize()
+            log_mock.error.assert_called_once_with(
+                'Duplicate plugin name: example')
+
+    def test_does_not_implement(self):
+        with ExitStack() as resources:
+            system_path = resources.enter_context(TemporaryDirectory())
+            fake_plugin_config = {
+                'path': system_path,
+                'enabled': 'yes',
+                'class': 'ExamplePlugin',
+                }
+            log_mock = resources.enter_context(
+                patch('mailman.plugins.initialize.log'))
+            fake_mailman_config = SimpleNamespace(
+                plugin_configs=[('example', fake_plugin_config)],
+                plugins=[],
+                )
+            resources.enter_context(patch(
+                'mailman.plugins.initialize.config', fake_mailman_config))
+            resources.enter_context(patch(
+                'mailman.plugins.initialize.call_name',
+                # object() does not implement IPlugin.
+                return_value=object()))
+            initialize()
+            log_mock.error.assert_called_once_with(
+                'Plugin class does not implement IPlugin: ExamplePlugin')
+            self.assertNotIn(system_path, sys.path)
+
+    def test_adds_plugins_to_config(self):
+        with ExitStack() as resources:
+            system_path = resources.enter_context(TemporaryDirectory())
+            fake_plugin_config = {
+                'path': system_path,
+                'enabled': 'yes',
+                'class': 'ExamplePlugin',
+                }
+            fake_mailman_config = SimpleNamespace(
+                plugin_configs=[('example', fake_plugin_config)],
+                plugins={},
+                )
+            resources.enter_context(patch(
+                'mailman.plugins.initialize.config', fake_mailman_config))
+            testable_plugin = TestablePlugin()
+            resources.enter_context(patch(
+                'mailman.plugins.initialize.call_name',
+                # object() does not implement IPlugin.
+                return_value=testable_plugin))
+            initialize()
+            self.assertIn('example', fake_mailman_config.plugins)
+            self.assertEqual(
+                fake_mailman_config.plugins['example'],
+                testable_plugin)
+
+    def test_not_enabled(self):
+        with ExitStack() as resources:
+            fake_plugin_config = {
+                'path': '/does/not/exist',
+                'enabled': 'no',
+                'class': 'ExamplePlugin',
+                }
+            log_mock = resources.enter_context(
+                patch('mailman.plugins.initialize.log'))
+            fake_mailman_config = SimpleNamespace(
+                plugin_configs=[('example', fake_plugin_config)],
+                plugins={},
+                )
+            resources.enter_context(patch(
+                'mailman.plugins.initialize.config', fake_mailman_config))
+            initialize()
+            log_mock.info.assert_called_once_with(
+                'Plugin not enabled, or empty class path: example')