6 files changed, 289 insertions, 269 deletions

diff --git a/src/mailman/interfaces/cache.py b/src/mailman/interfaces/cache.py
new file mode 100644
index 000000000..74fb2084a
--- /dev/null
+++ b/src/mailman/interfaces/cache.py
@@ -0,0 +1,66 @@
+# Copyright (C) 2016 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""File caches."""
+
+from mailman import public
+from zope.interface import Interface
+
+
+@public
+class ICacheManager(Interface):
+    """Manager for managing cached files."""
+
+    def add(key, contents, lifetime=None):
+        """Add the contents to the cache, indexed by the key.
+
+        If there is already some contents cached under the given key, the old
+        contents are overwritten with the new contents.
+
+        :param key: The key to use when storing the contents.
+        :type name: str
+        :param contents: The contents to store in the cache.  If this is a
+            bytes object, it will be stored on disk in binary.  If it's a str,
+            it will be stored in UTF-8.  Either way, the manager will remember
+            the type and return it when you access the file.
+        :type contents: bytes or str
+        :param lifetime: How long should the file be cached for, before it
+            expires (leading to its eventual eviction)?  If not given, the
+            system default lifetime is used.
+        :type lifetime: datetime.timedelta
+        :return: The SHA256 hash under which the file contents are stored.
+        :rtype: str
+        """
+
+    def get(key, *, expunge=False):
+        """Return the contents cached under the given key.
+
+        :param key: The key identifying the contents you want to retrieve.
+        :type key: str
+        :param expunge: A flag indicating whether the file contents should
+            also be removed from the cache.
+        :type expunge: bool
+        :return: The contents of the cached file or None if the given id isn't
+            in the cache (or it's already expired).
+        :rtype: bytes or str, depending on the original contents.
+        """
+
+    def evict():
+        """Evict all files which have expired."""
+
+    def clear():
+        """Clear the entire cache of files."""
diff --git a/src/mailman/interfaces/domain.py b/src/mailman/interfaces/domain.py
index 7be5909a9..b1940f400 100644
--- a/src/mailman/interfaces/domain.py
+++ b/src/mailman/interfaces/domain.py
@@ -69,16 +69,6 @@ class IDomain(Interface):
 
     mail_host = Attribute('The host name for email for this domain.')
 
-    url_host = Attribute(
-        'The host name for the web interface for this domain.')
-
-    base_url = Attribute("""\
-    The base url for the Mailman server at this domain, which includes the
-    scheme and host name.""")
-
-    scheme = Attribute(
-        """The protocol scheme used to contact this list's server.""")
-
     description = Attribute(
         'The human readable description of the domain name.')
 
@@ -91,31 +81,18 @@ class IDomain(Interface):
         The mailing lists are returned in order sorted by list-id.
         """)
 
-    def confirm_url(token=''):
-        """The url used for various forms of confirmation.
-
-        :param token: The confirmation token to use in the url.
-        :type token: string
-        :return: The confirmation url.
-        :rtype: string
-        """
-
 
 @public
 class IDomainManager(Interface):
     """The manager of domains."""
 
-    def add(mail_host, description=None, base_url=None, owners=None):
+    def add(mail_host, description=None, owners=None):
         """Add a new domain.
 
         :param mail_host: The email host name for the domain.
         :type mail_host: string
         :param description: The description of the domain.
         :type description: string
-        :param base_url: The base url, including the scheme for the web
-            interface of the domain.  If not given, it defaults to
-            http://`mail_host`/
-        :type base_url: string
         :param owners: Sequence of owners of the domain, defaults to None,
             meaning the domain does not have owners.
         :type owners: sequence of `IUser` or string emails.
diff --git a/src/mailman/interfaces/mailinglist.py b/src/mailman/interfaces/mailinglist.py
index b5c0e5456..19db025d1 100644
--- a/src/mailman/interfaces/mailinglist.py
+++ b/src/mailman/interfaces/mailinglist.py
@@ -111,6 +111,19 @@ class IMailingList(Interface):
         mailing lists, or in headers, and so forth.  It should be as succinct
         as you can get it, while still identifying what the list is.""")
 
+    info = Attribute("""\
+        A longer description of this mailing list.  This can be any arbitrary
+        text, up to a database-specific maximum length.
+        """)
+
+    preferred_language = Attribute("""\
+        The default language for communications on this mailing list.
+
+        When the list sends out notifications, it will be in this language,
+        unless the recipient is a known user and that user has a preferred
+        language.
+        """)
+
     subject_prefix = Attribute("""\
         The text to insert at the front of the Subject field.
 
@@ -349,29 +362,6 @@ class IMailingList(Interface):
         digest recipients are cleared.
         """)
 
-    # Web access.
-
-    scheme = Attribute(
-        """The protocol scheme used to contact this list's server.
-
-        The web server on this protocol provides the web interface for this
-        mailing list.  The protocol scheme should be 'http' or 'https'.""")
-
-    web_host = Attribute(
-        """This list's web server's domain.
-
-        The read-only domain name of the host to contact for interacting with
-        the web interface of the mailing list.""")
-
-    def script_url(target, context=None):
-        """Return the url to the given script target.
-
-        If 'context' is not given, or is None, then an absolute url is
-        returned.  If context is given, it must be an IMailingListRequest
-        object, and the returned url will be relative to that object's
-        'location' attribute.
-        """
-
     # Autoresponses.
 
     autoresponse_grace_period = Attribute(
@@ -593,183 +583,9 @@ class IMailingList(Interface):
     send_welcome_message = Attribute(
         """Flag indicating whether a welcome message should be sent.""")
 
-    welcome_message_uri = Attribute(
-        """URI for the list's welcome message.
-
-        This can be any URI supported by `urllib2` with the addition of
-        `mailman:` URIs, which reference internal default resources.  This is
-        a template which can include the following placeholders:
-
-        $listname - the FQDN list name for this mailing list.
-        $language - the language code, usually the list's preferred language.
-
-        The resource will be downloaded and cached whenever the welcome
-        message is sent.  The resource at this URI can contain the following
-        placeholders, which are also filled in through values on the mailing
-        list:
-
-        $fqdn_listname    - the FQDN list name for this mailing list.
-        $list_name        - the human readable name for the mailing list.
-        $listinfo_uri     - the URI to the list's information page.
-        $list_requests    - the address to the list's `-request` address.
-        $user_name        - the name of the subscribing user.
-        $user_address     - the email address of the subscribing user.
-        $user_options_uri - the URI to this member's options page.
-        """)
-
     send_goodbye_message = Attribute(
         """Flag indicating whether a goodbye message should be sent.""")
 
-    goodbye_message_uri = Attribute(
-        """URI for the list's goodbye message.
-
-        This can be any URI supported by `urllib2` with the addition of
-        `mailman:` URIs, which reference internal default resources.  This is
-        a template which can include the following placeholders:
-
-        $listname - the FQDN list name for this mailing list.
-        $language - the language code, usually the list's preferred language.
-
-        The resource will be downloaded and cached whenever the goodbye
-        message is sent.  The resource at this URI can contain the following
-        placeholders, which are also filled in through values on the mailing
-        list:
-
-        $fqdn_listname    - the FQDN list name for this mailing list.
-        $list_name        - the human readable name for the mailing list.
-        $listinfo_uri     - the URI to the list's information page.
-        $list_requests    - the address to the list's `-request` address.
-        $user_name        - the name of the subscribing user.
-        $user_address     - the email address of the subscribing user.
-        $user_options_uri - the URI to this member's options page.
-        """)
-
-    # Decorators.
-
-    header_uri = Attribute(
-        """URI for the header decorator on regular delivery messages.
-
-        This can be any URI supported by `urllib2` with the addition of
-        `mailman:` URIs, which reference internal default resources.  This is
-        a template which can include the following placeholders:
-
-        $listname - the FQDN list name for this mailing list.
-        $language - the language code, usually the list's preferred language.
-
-        The resource will be downloaded and cached whenever the decorator is
-        needed.  The resource at this URI can contain the following
-        placeholders, which are also filled in through values on the mailing
-        list:
-
-        $fqdn_listname    - the FQDN list name for this mailing list.
-        $list_name        - the human readable name for the mailing list.
-        $host_name        - the mailing list's host name
-        $listinfo_uri     - the URI to the list's information page.
-        $list_requests    - the address to the list's `-request` address.
-        $description      - the mailing list's description
-        $info             - additional mailing list's information
-
-        Personalized messages will also have these placeholders available:
-
-        $user_name        - the name of the subscribing user.
-        $user_address     - the email address of the subscribing user.
-        $user_options_uri - the URI to this member's options page.
-        """
-        )
-
-    footer_uri = Attribute(
-        """URI for the footer decorator on regular delivery messages.
-
-        This can be any URI supported by `urllib2` with the addition of
-        `mailman:` URIs, which reference internal default resources.  This is
-        a template which can include the following placeholders:
-
-        $listname - the FQDN list name for this mailing list.
-        $language - the language code, usually the list's preferred language.
-
-        The resource will be downloaded and cached whenever the decorator is
-        needed.  The resource at this URI can contain the following
-        placeholders, which are also filled in through values on the mailing
-        list:
-
-        $fqdn_listname    - the FQDN list name for this mailing list.
-        $list_name        - the human readable name for the mailing list.
-        $host_name        - the mailing list's host name
-        $listinfo_uri     - the URI to the list's information page.
-        $list_requests    - the address to the list's `-request` address.
-        $description      - the mailing list's description
-        $info             - additional mailing list's information
-
-        Personalized messages will also have these placeholders available:
-
-        $user_name        - the name of the subscribing user.
-        $user_address     - the email address of the subscribing user.
-        $user_options_uri - the URI to this member's options page.
-        """
-        )
-
-    digest_header_uri = Attribute(
-        """URI for the header decorator on digest messages.
-
-        This can be any URI supported by `urllib2` with the addition of
-        `mailman:` URIs, which reference internal default resources.  This is
-        a template which can include the following placeholders:
-
-        $listname - the FQDN list name for this mailing list.
-        $language - the language code, usually the list's preferred language.
-
-        The resource will be downloaded and cached whenever the decorator is
-        needed.  The resource at this URI can contain the following
-        placeholders, which are also filled in through values on the mailing
-        list:
-
-        $fqdn_listname    - the FQDN list name for this mailing list.
-        $list_name        - the human readable name for the mailing list.
-        $host_name        - the mailing list's host name
-        $listinfo_uri     - the URI to the list's information page.
-        $list_requests    - the address to the list's `-request` address.
-        $description      - the mailing list's description
-        $info             - additional mailing list's information
-
-        Personalized messages will also have these placeholders available:
-
-        $user_name        - the name of the subscribing user.
-        $user_address     - the email address of the subscribing user.
-        $user_options_uri - the URI to this member's options page.
-        """
-        )
-
-    digest_footer_uri = Attribute(
-        """URI for the footer decorator on digest messages.
-
-        This can be any URI supported by `urllib2` with the addition of
-        `mailman:` URIs, which reference internal default resources.  This is
-        a template which can include the following placeholders:
-
-        $listname - the FQDN list name for this mailing list.
-        $language - the language code, usually the list's preferred language.
-
-        The resource will be downloaded and cached whenever the decorator is
-        needed.  The resource at this URI can contain the following
-        placeholders, which are also filled in through values on the mailing
-        list:
-
-        $fqdn_listname    - the FQDN list name for this mailing list.
-        $list_name        - the human readable name for the mailing list.
-        $host_name        - the mailing list's host name
-        $listinfo_uri     - the URI to the list's information page.
-        $list_requests    - the address to the list's `-request` address.
-        $description      - the mailing list's description
-        $info             - additional mailing list's information
-
-        Personalized messages will also have these placeholders available:
-
-        $user_name        - the name of the subscribing user.
-        $user_address     - the email address of the subscribing user.
-        $user_options_uri - the URI to this member's options page.
-        """
-        )
-
 
 @public
 class IAcceptableAlias(Interface):
diff --git a/src/mailman/interfaces/member.py b/src/mailman/interfaces/member.py
index 367838e21..094652521 100644
--- a/src/mailman/interfaces/member.py
+++ b/src/mailman/interfaces/member.py
@@ -174,6 +174,16 @@ class IMember(Interface):
         address, it will be an ``IUser``.
         """)
 
+    display_name = Attribute(
+        """The best match of the member's display name.
+
+        This will be `subscriber.display_name` if available, which means it
+        will either be the display name of the address or user that's
+        subscribed.  If unavailable, and the address is the subscriber, then
+        the linked user's display name is given, if available.  When all else
+        fails, the empty string is returned.
+        """)
+
     preferences = Attribute(
         """This member's preferences.""")
 
@@ -258,11 +268,3 @@ class IMember(Interface):
         4. System default
 
         XXX I'm not sure this is the right place to put this.""")
-
-    options_url = Attribute(
-        """Return the url for the given member's option page.
-
-        XXX This needs a serious re-think in the face of the unified user
-        database, since a member's options aren't tied to any specific mailing
-        list.  So in what part of the web-space does the user's options live?
-        """)
diff --git a/src/mailman/interfaces/template.py b/src/mailman/interfaces/template.py
new file mode 100644
index 000000000..442f1eeda
--- /dev/null
+++ b/src/mailman/interfaces/template.py
@@ -0,0 +1,199 @@
+# Copyright (C) 2012-2016 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Template downloader with cache."""
+
+from mailman import public
+from zope.interface import Attribute, Interface
+
+
+@public
+class ITemplate(Interface):
+    """A template record."""
+
+    name = Attribute("""The template name.""")
+
+    context = Attribute("""\
+        The template context.  This may be a list-id, a domain mail host name,
+        or None (for the global context).
+        """)
+
+    uri = Attribute("""The template uri.""")
+
+    username = Attribute("""\
+        Optional username used to retrieved the contents referenced by the
+        template uri.  This can be None if no username/password is needed.""")
+
+    password = Attribute("""\
+        Optional password used to retrieved the contents referenced by the
+        template uri.  This can be None if no username/password is needed.""")
+
+
+@public
+class ITemplateLoader(Interface):
+    """The template downloader utility."""
+
+    def get(name, context=None, **kws):
+        """Find the named template for the given context.
+
+        This search for a named template using a stacked strategy.  If the
+        template has been registered for the given context in the template
+        manager, that template is used.  If not, a file system based template
+        is used as a fallback.  If nothing can be found, the empty string is
+        returned.
+
+        Use the ``ITemplateManager`` to set specific template locations.  The
+        fallbacks all use the ``mailman:`` scheme to use in-tree defaults.
+
+        :param name: The name of the template to find.
+        :type name: str
+        :param context: The context in which to find the template.  This can
+            be either an IMailingList, an IDomain, or None for the global/site
+            context.  The template will be searched in order from most
+            specific to least specific, i.e. from list-id, to domain, to
+            global.
+        :type context: IMailingList, IDomain, None
+        :param kws: Additional URL substitution variables.  Once a URL for the
+            given name and context is identified, these are used to fill in
+            placeholders in the URL before the template is retrieved.
+            `list_id`, `list_name`, and `mail_host` will automatically be
+            filled in if available depending on the `context`.  This
+            dictionary is passed directly to the underlying
+            ``ITemplatemanager.get()`` call.
+        :type kws: dict
+        :return: The found template or its fallback.
+        :rtype: str
+        """
+
+
+@public
+class ITemplateManager(Interface):
+    """Manager/loader for notification templates."""
+
+    def set(name, context, uri, username=None, password=''):
+        """Set a template mapping from name to URI.
+
+        The URI may be cached for some length of time defined by the system.
+
+        :param name: The template name, including any extension.
+        :type name: str
+        :param context: The context for this name->URI mapping.  This can be a
+            list-id, domain mail host name, or None (for global context).
+        :type context: str
+        :param uri: The URI of the template.  Normal http: and https: URIs can
+            be used, as well as special mailman: URIs which reference internal
+            resources.
+        :type uri: str
+        :param username: Optional user name for Basic Auth on the URI.
+        :type username: str
+        :param password: Optional password for Basic Auth on the URI.
+        :type username: str
+        """
+
+    def get(name, context, **kws):
+        """Return the contents for the given context and name.
+
+        `context` can be a list-id, domain mail host name, or "*" (for global
+        context).  A search will be performed from the named context up to the
+        global context.  For example, if a List-Id is given but no template
+        for that mailing list under that name is registered, the mailing
+        list's domain is search, then the global context is searched.
+
+        If the URI mapped to this mailing list/name pair is not yet retrieved,
+        it is downloaded first.  If the cache lifetime has expired, it will be
+        downloaded again.  Otherwise the cached version will be returned.
+
+        :param name: The template name, including any extension.
+        :type name: str
+        :param context: The context for this name->URI mapping.  This can be a
+            List-ID, domain mail host name, or None (for global context).
+        :type context: str
+        :param kws: A substitution dictionary that is interpolated into the
+            url to retrieve the contents of the template.  Passing in a
+            different dictionary than before can cause a new template to be
+            downloaded.
+        :type kws: dict
+        :return: The resource mapped to the given name, or None if not found.
+        :rtype: str or None
+        """
+
+    def raw(name, context):
+        """Return the raw template information for the given context and name.
+
+        `context` can be a list-id, domain mail host name, or "*" (for global
+        context).  The raw template matching the given name and context is
+        returned, otherwise None if no such template has been registered.
+
+        :param name: The template name, including any extension.
+        :type name: str
+        :param context: The context for this name->URI mapping.  This can be a
+            List-ID, domain mail host name, or None (for global context).
+        :type context: str
+        :return: The raw template record, or None if not found.
+        :rtype: ITemplate or None
+        """
+
+    def delete(name, context):
+        """Delete the named template and any cached contents.
+
+        :param name: The template name, including any extension.
+        :type name: str
+        :param context: The context for this name->URI mapping.  This can be a
+            List-ID, domain mail host name, or None (for global context).
+        :type context: str
+        """
+
+# Mapping of template names to their in-source file names.  A None value means
+# that there is no file in the tree for that template.
+
+ALL_TEMPLATES = {
+    key: '{}.txt'.format(key)
+    for key in {
+        'domain:admin:notice:new-list',
+        'list:admin:action:post',
+        'list:admin:action:subscribe',
+        'list:admin:action:unsubscribe',
+        'list:admin:notice:subscribe',
+        'list:admin:notice:unrecognized',
+        'list:admin:notice:unsubscribe',
+        'list:member:digest:masthead',
+        'list:user:action:confirm',
+        'list:user:action:unsubscribe',
+        'list:user:notice:hold',
+        'list:user:notice:no-more-today',
+        'list:user:notice:post',
+        'list:user:notice:probe',
+        'list:user:notice:refuse',
+        'list:user:notice:welcome',
+        }
+    }
+
+# These have other names.
+ALL_TEMPLATES.update({
+    'list:member:digest:footer': 'list:member:generic:footer.txt',
+    'list:member:regular:footer': 'list:member:generic:footer.txt',
+    })
+
+# These are some extra supported templates which don't have a mapping to a
+# file in the source tree.
+ALL_TEMPLATES.update({
+    'list:member:digest:header': None,
+    'list:member:regular:header': None,
+    'list:user:notice:goodbye':  None,
+    })
+
+public(ALL_TEMPLATES=ALL_TEMPLATES)
diff --git a/src/mailman/interfaces/templates.py b/src/mailman/interfaces/templates.py
deleted file mode 100644
index 40bc86d2a..000000000
--- a/src/mailman/interfaces/templates.py
+++ /dev/null
@@ -1,40 +0,0 @@
-# Copyright (C) 2012-2016 by the Free Software Foundation, Inc.
-#
-# This file is part of GNU Mailman.
-#
-# GNU Mailman is free software: you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by the Free
-# Software Foundation, either version 3 of the License, or (at your option)
-# any later version.
-#
-# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-# more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
-
-"""Template downloader with cache."""
-
-from mailman import public
-from zope.interface import Interface
-
-
-@public
-class ITemplateLoader(Interface):
-    """The template downloader utility."""
-
-    def get(uri):
-        """Download the named URI, and return the response and content.
-
-        This API uses `urllib2`_ so consult its documentation for details.
-
-        .. _`urllib2`: http://docs.python.org/library/urllib2.html
-
-        :param uri: The URI of the resource.  These may be any URI supported
-            by `urllib2` and also `mailman:` URIs for internal resources.
-        :type uri: string
-        :return: The template string as a unicode.
-        :rtype: str
-        """