13 files changed, 132 insertions, 42 deletions

diff --git a/src/mailman/app/commands.py b/src/mailman/app/commands.py
index 7cf3bc4c5..ca257ffda 100644
--- a/src/mailman/app/commands.py
+++ b/src/mailman/app/commands.py
@@ -19,11 +19,11 @@
 
 from mailman.config import config
 from mailman.interfaces.command import IEmailCommand
-from mailman.utilities.modules import add_components
+from mailman.utilities.plugins import add_pluggable_components
 from public import public
 
 
 @public
 def initialize():
     """Initialize the email commands."""
-    add_components('mailman.commands', IEmailCommand, config.commands)
+    add_pluggable_components('commands', IEmailCommand, config.commands)
diff --git a/src/mailman/app/docs/hooks.rst b/src/mailman/app/docs/plugins.rst
index 07d0ec33c..07d0ec33c 100644
--- a/src/mailman/app/docs/hooks.rst
+++ b/src/mailman/app/docs/plugins.rst
diff --git a/src/mailman/bin/mailman.py b/src/mailman/bin/mailman.py
index e012237f3..ece223180 100644
--- a/src/mailman/bin/mailman.py
+++ b/src/mailman/bin/mailman.py
@@ -16,7 +16,6 @@
 # GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
 
 """The 'mailman' command dispatcher."""
-
 import click
 
 from contextlib import ExitStack
@@ -25,7 +24,7 @@ from mailman.core.i18n import _
 from mailman.core.initialize import initialize
 from mailman.database.transaction import transaction
 from mailman.interfaces.command import ICLISubCommand
-from mailman.utilities.modules import add_components
+from mailman.utilities.plugins import add_pluggable_components
 from mailman.version import MAILMAN_VERSION_FULL
 from public import public
 
@@ -35,17 +34,22 @@ class Subcommands(click.MultiCommand):
     def __init__(self, *args, **kws):
         super().__init__(*args, **kws)
         self._commands = {}
-        # Look at all modules in the mailman.bin package and if they are
-        # prepared to add a subcommand, let them do so.  I'm still undecided as
-        # to whether this should be pluggable or not.  If so, then we'll
-        # probably have to partially parse the arguments now, then initialize
-        # the system, then find the plugins.  Punt on this for now.
-        add_components('mailman.commands', ICLISubCommand, self._commands)
+        self._loaded = False
+
+    def _load(self):
+        # Load commands lazily as pluggable commands need a parsed config to
+        # find plugins.
+        if not self._loaded:
+            add_pluggable_components('commands', ICLISubCommand,
+                                     self._commands)
+            self._loaded = True
 
     def list_commands(self, ctx):
+        self._load()
         return sorted(self._commands)               # pragma: nocover
 
     def get_command(self, ctx, name):
+        self._load()
         try:
             return self._commands[name].command
         except KeyError as error:
@@ -83,10 +87,12 @@ class Subcommands(click.MultiCommand):
                 formatter.write_dl(opts)
 
 
-@click.group(
-    cls=Subcommands,
-    context_settings=dict(help_option_names=['-h', '--help']))
-@click.pass_context
+def initialize_config(ctx, param, value):
+    if ctx.resilient_parsing:
+        return
+    initialize(value)
+
+
 @click.option(
     '-C', '--config', 'config_file',
     envvar='MAILMAN_CONFIG_FILE',
@@ -94,7 +100,12 @@ class Subcommands(click.MultiCommand):
     help=_("""\
     Configuration file to use.  If not given, the environment variable
     MAILMAN_CONFIG_FILE is consulted and used if set.  If neither are given, a
-    default configuration file is loaded."""))
+    default configuration file is loaded."""),
+    is_eager=True, callback=initialize_config)
+@click.group(
+    cls=Subcommands,
+    context_settings=dict(help_option_names=['-h', '--help']))
+@click.pass_context
 @click.version_option(MAILMAN_VERSION_FULL, message='%(version)s')
 @public
 def main(ctx, config_file):
@@ -104,6 +115,4 @@ def main(ctx, config_file):
     Copyright 1998-2017 by the Free Software Foundation, Inc.
     http://www.list.org
     """
-    # Initialize the system.  Honor the -C flag if given.
-    initialize(config_file)
     # click handles dispatching to the subcommand via the Subcommands class.
diff --git a/src/mailman/commands/docs/conf.rst b/src/mailman/commands/docs/conf.rst
index 1a6a4679d..6c7309773 100644
--- a/src/mailman/commands/docs/conf.rst
+++ b/src/mailman/commands/docs/conf.rst
@@ -50,6 +50,7 @@ key, along with the names of the corresponding sections.
     [logging.smtp] path: smtp.log
     [logging.subscribe] path: mailman.log
     [logging.vette] path: mailman.log
+    [plugin.master] path:
 
 If you specify both a section and a key, you will get the corresponding value.
 
diff --git a/src/mailman/commands/docs/info.rst b/src/mailman/commands/docs/info.rst
index 219143cab..40784a98b 100644
--- a/src/mailman/commands/docs/info.rst
+++ b/src/mailman/commands/docs/info.rst
@@ -59,7 +59,6 @@ definition.
         CFG_FILE        = .../test.cfg
         DATA_DIR        = /var/lib/mailman/data
         ETC_DIR         = /etc
-        EXT_DIR         = /etc/mailman.d
         LIST_DATA_DIR   = /var/lib/mailman/lists
         LOCK_DIR        = /var/lock/mailman
         LOCK_FILE       = /var/lock/mailman/master.lck
diff --git a/src/mailman/config/config.py b/src/mailman/config/config.py
index 61c9fe6ed..2ba657905 100644
--- a/src/mailman/config/config.py
+++ b/src/mailman/config/config.py
@@ -158,7 +158,7 @@ class Configuration:
                 else category.template_dir),
             )
         # Directories.
-        for name in ('archive', 'bin', 'cache', 'data', 'etc', 'ext',
+        for name in ('archive', 'bin', 'cache', 'data', 'etc',
                      'list_data', 'lock', 'log', 'messages', 'queue'):
             key = '{}_dir'.format(name)
             substitutions[key] = getattr(category, key)
@@ -248,6 +248,11 @@ class Configuration:
             yield archiver
 
     @property
+    def plugin_configs(self):
+        """Iterate over all the plugin configuration sections."""
+        return self._config.getByCategory('plugin', [])
+
+    @property
     def language_configs(self):
         """Iterate over all the language configuration sections."""
         yield from self._config.getByCategory('language', [])
diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg
index 9568694be..b708c6ceb 100644
--- a/src/mailman/config/schema.cfg
+++ b/src/mailman/config/schema.cfg
@@ -82,6 +82,19 @@ html_to_plain_text_command: /usr/bin/lynx -dump $filename
 # unpredictable.
 listname_chars: [-_.0-9a-z]
 
+[plugin.master]
+# Plugin package
+# It's sub packages will be searched for components.
+#   - commands for IEmailCommand
+#   - chains for IChain
+#   - rules for IRule
+#   - pipelines for IPipeline
+#   - handlers for IHandler
+#   - styles for IStyle
+path:
+
+# Whether to enable this plugin or not.
+enable: no
 
 [shell]
 # `mailman shell` (also `withlist`) gives you an interactive prompt that you
@@ -138,8 +151,6 @@ data_dir: $var_dir/data
 cache_dir: $var_dir/cache
 # Directory for configuration files and such.
 etc_dir: $var_dir/etc
-# Directory containing Mailman plugins.
-ext_dir: $var_dir/ext
 # Directory where the default IMessageStore puts its messages.
 messages_dir: $var_dir/messages
 # Directory for archive backends to store their messages in.  Archivers should
@@ -797,11 +808,6 @@ class: mailman.archiving.prototype.Prototype
 
 
 [styles]
-# Python import paths inside which components are searched for which implement
-# the IStyle interface.  Use one path per line.
-paths:
-    mailman.styles
-
 # The default style to apply if nothing else was requested.  The value is the
 # name of an existing style.  If no such style exists, no style will be
 # applied.
diff --git a/src/mailman/core/chains.py b/src/mailman/core/chains.py
index c251d38bc..48ed82d63 100644
--- a/src/mailman/core/chains.py
+++ b/src/mailman/core/chains.py
@@ -19,7 +19,7 @@
 
 from mailman.config import config
 from mailman.interfaces.chain import IChain, LinkAction
-from mailman.utilities.modules import add_components
+from mailman.utilities.plugins import add_pluggable_components
 from public import public
 
 
@@ -89,5 +89,4 @@ def process(mlist, msg, msgdata, start_chain='default-posting-chain'):
 @public
 def initialize():
     """Set up chains, both built-in and from the database."""
-    add_components('mailman.chains', IChain, config.chains)
-    # XXX Read chains from the database and initialize them.
+    add_pluggable_components('chains', IChain, config.chains)
diff --git a/src/mailman/core/pipelines.py b/src/mailman/core/pipelines.py
index 4265d24d1..6e2ad4a27 100644
--- a/src/mailman/core/pipelines.py
+++ b/src/mailman/core/pipelines.py
@@ -24,7 +24,7 @@ from mailman.config import config
 from mailman.interfaces.handler import IHandler
 from mailman.interfaces.pipeline import (
     DiscardMessage, IPipeline, RejectMessage)
-from mailman.utilities.modules import add_components
+from mailman.utilities.plugins import add_pluggable_components
 from public import public
 
 
@@ -63,6 +63,6 @@ def process(mlist, msg, msgdata, pipeline_name='built-in'):
 def initialize():
     """Initialize the pipelines."""
     # Find all handlers in the registered plugins.
-    add_components('mailman.handlers', IHandler, config.handlers)
+    add_pluggable_components('handlers', IHandler, config.handlers)
     # Set up some pipelines.
-    add_components('mailman.pipelines', IPipeline, config.pipelines)
+    add_pluggable_components('pipelines', IPipeline, config.pipelines)
diff --git a/src/mailman/core/rules.py b/src/mailman/core/rules.py
index 8e0d9197c..74934eb99 100644
--- a/src/mailman/core/rules.py
+++ b/src/mailman/core/rules.py
@@ -19,7 +19,7 @@
 
 from mailman.config import config
 from mailman.interfaces.rules import IRule
-from mailman.utilities.modules import add_components
+from mailman.utilities.plugins import add_pluggable_components
 from public import public
 
 
@@ -27,4 +27,4 @@ from public import public
 def initialize():
     """Find and register all rules in all plugins."""
     # Find rules in plugins.
-    add_components('mailman.rules', IRule, config.rules)
+    add_pluggable_components('rules', IRule, config.rules)
diff --git a/src/mailman/rest/tests/test_systemconf.py b/src/mailman/rest/tests/test_systemconf.py
index f9fe9caa0..5acc38adb 100644
--- a/src/mailman/rest/tests/test_systemconf.py
+++ b/src/mailman/rest/tests/test_systemconf.py
@@ -182,6 +182,7 @@ class TestSystemConfiguration(unittest.TestCase):
             'paths.here',
             'paths.local',
             'paths.testing',
+            'plugin.master',
             'runner.archive',
             'runner.bad',
             'runner.bounces',
diff --git a/src/mailman/styles/manager.py b/src/mailman/styles/manager.py
index 1ec83b2a9..1969d4484 100644
--- a/src/mailman/styles/manager.py
+++ b/src/mailman/styles/manager.py
@@ -20,7 +20,7 @@
 from mailman.interfaces.configuration import ConfigurationUpdatedEvent
 from mailman.interfaces.styles import (
     DuplicateStyleError, IStyle, IStyleManager)
-from mailman.utilities.modules import add_components
+from mailman.utilities.plugins import add_pluggable_components
 from public import public
 from zope.component import getUtility
 from zope.interface import implementer
@@ -38,13 +38,7 @@ class StyleManager:
 
     def populate(self):
         self._styles.clear()
-        # Avoid circular imports.
-        from mailman.config import config
-        # Calculate the Python import paths to search.
-        paths = filter(None, (path.strip()
-                              for path in config.styles.paths.splitlines()))
-        for path in paths:
-            add_components(path, IStyle, self._styles)
+        add_pluggable_components('styles', IStyle, self._styles)
 
     def get(self, name):
         """See `IStyleManager`."""
diff --git a/src/mailman/utilities/plugins.py b/src/mailman/utilities/plugins.py
new file mode 100644
index 000000000..a8449af31
--- /dev/null
+++ b/src/mailman/utilities/plugins.py
@@ -0,0 +1,76 @@
+# Copyright (C) 2009-2017 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Plugin utilities."""
+
+from lazr.config import as_boolean
+from mailman.config import config
+from mailman.utilities.modules import find_components
+from pkg_resources import resource_isdir
+from public import public
+
+
+@public
+def find_pluggable_components(subpackage, interface):
+    """Find components which conform to a given interface, in subpackage.
+
+    :param subpackage: Mailman subpackage to search. The search is also
+    done inside the corresponding plugins subpackages.
+    :type subpackage: string
+    :param interface: The interface that returned objects must conform to.
+    :type interface: `Interface`
+    :return: The sequence of matching components.
+    :rtype: objects implementing `interface`
+    """
+    yield from find_components('mailman.' + subpackage, interface)
+    package_roots = [plugin.path
+                     for plugin in config.plugin_configs
+                     if as_boolean(plugin.enable) and plugin.path]
+    for package_root in package_roots:
+        package = package_root + '.' + subpackage
+        if resource_isdir(package_root, subpackage):
+            yield from find_components(package, interface)
+
+
+@public
+def add_pluggable_components(subpackage, interface, mapping):
+    """Add components to a given mapping.
+
+    Similarly to `find_pluggable_components()` this inspects all modules in a
+    given mailman and plugin's subpackage looking for objects that conform to
+    a given interface.  All such found objects (unless decorated with
+    `@abstract_component`) are added to the given mapping, keyed by the
+    object's `.name` attribute, which is required.  It is a fatal error if
+    that key already exists in the mapping.
+
+    :param package: The subpackage path to search.
+    :type package: string
+    :param interface: The interface that returned objects must conform to.
+        Objects found must have a `.name` attribute containing a unique
+        string.
+    :type interface: `Interface`
+    :param mapping: The mapping to add the found components to.
+    :type mapping: A dict-like mapping.  This only needs to support
+        containment tests (e.g. `in` and `not in`) and `__setitem__()`.
+    :raises RuntimeError: when a duplicate key is found.
+    """
+    for component in find_pluggable_components(subpackage, interface):
+        if component.name in mapping:
+            raise RuntimeError(
+                'Duplicate key "{}" found in {}; previously {}'.format(
+                    component.name, component, mapping[component.name]))
+        mapping[component.name] = component