New template system. Closes #249

The new template system is introduced for API 3.1.  See
``src/mailman/rest/docs/templates.rst`` for details.

8 files changed, 790 insertions, 89 deletions

diff --git a/src/mailman/model/cache.py b/src/mailman/model/cache.py
new file mode 100644
index 000000000..d9c0d0445
--- /dev/null
+++ b/src/mailman/model/cache.py
@@ -0,0 +1,160 @@
+# Copyright (C) 2016 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Generic file cache."""
+
+import os
+import hashlib
+
+from contextlib import ExitStack
+from lazr.config import as_timedelta
+from mailman import public
+from mailman.config import config
+from mailman.database.model import Model
+from mailman.database.transaction import dbconnection
+from mailman.interfaces.cache import ICacheManager
+from mailman.utilities.datetime import now
+from sqlalchemy import Boolean, Column, DateTime, Integer, Unicode
+from zope.interface import implementer
+
+
+class CacheEntry(Model):
+    __tablename__ = 'file_cache'
+
+    id = Column(Integer, primary_key=True)
+    key = Column(Unicode)
+    file_id = Column(Unicode)
+    is_bytes = Column(Boolean)
+    created_on = Column(DateTime)
+    expires_on = Column(DateTime)
+
+    @dbconnection
+    def __init__(self, store, key, file_id, is_bytes, lifetime):
+        self.key = key
+        self.file_id = file_id
+        self.is_bytes = is_bytes
+        self.created_on = now()
+        self.expires_on = self.created_on + lifetime
+
+    @dbconnection
+    def update(self, store, is_bytes, lifetime):
+        self.is_bytes = is_bytes
+        self.created_on = now()
+        self.expires_on = self.created_on + lifetime
+
+    @property
+    def is_expired(self):
+        return self.expires_on <= now()
+
+
+@public
+@implementer(ICacheManager)
+class CacheManager:
+    """Manages a cache of files on the file system."""
+
+    @staticmethod
+    def _id_to_path(file_id):
+        dir_1 = file_id[0:2]
+        dir_2 = file_id[2:4]
+        dir_path = os.path.join(config.CACHE_DIR, dir_1, dir_2)
+        file_path = os.path.join(dir_path, file_id)
+        return file_path, dir_path
+
+    @staticmethod
+    def _key_to_file_id(key):
+        # Calculate the file-id/SHA256 hash.  The key must be a string, even
+        # though the hash algorithm requires bytes.
+        hashfood = key.encode('raw-unicode-escape')
+        # Use the hex digest (a str) for readability.
+        return hashlib.sha256(hashfood).hexdigest()
+
+    def _write_contents(self, file_id, contents, is_bytes):
+        # Calculate the file system path by taking the SHA1 hash, stripping
+        # out two levels of directory (to reduce the chance of direntry
+        # exhaustion on some systems).
+        file_path, dir_path = self._id_to_path(file_id)
+        os.makedirs(dir_path, exist_ok=True)
+        # Open the file on the correct mode and write the contents.
+        with ExitStack() as resources:
+            if is_bytes:
+                fp = resources.enter_context(open(file_path, 'wb'))
+            else:
+                fp = resources.enter_context(
+                    open(file_path, 'w', encoding='utf-8'))
+            fp.write(contents)
+
+    @dbconnection
+    def add(self, store, key, contents, lifetime=None):
+        """See `ICacheManager`."""
+        if lifetime is None:
+            lifetime = as_timedelta(config.mailman.cache_life)
+        is_bytes = isinstance(contents, bytes)
+        file_id = self._key_to_file_id(key)
+        # Is there already an unexpired entry under this id in the database?
+        # If the entry doesn't exist, create it.  If it overwrite both the
+        # contents and lifetime.
+        entry = store.query(CacheEntry).filter(
+            CacheEntry.key == key).one_or_none()
+        if entry is None:
+            entry = CacheEntry(key, file_id, is_bytes, lifetime)
+            store.add(entry)
+        else:
+            entry.update(is_bytes, lifetime)
+        self._write_contents(file_id, contents, is_bytes)
+        return file_id
+
+    @dbconnection
+    def get(self, store, key, *, expunge=False):
+        """See `ICacheManager`."""
+        entry = store.query(CacheEntry).filter(
+            CacheEntry.key == key).one_or_none()
+        if entry is None:
+            return None
+        file_path, dir_path = self._id_to_path(entry.file_id)
+        with ExitStack() as resources:
+            if entry.is_bytes:
+                fp = resources.enter_context(open(file_path, 'rb'))
+            else:
+                fp = resources.enter_context(
+                    open(file_path, 'r', encoding='utf-8'))
+            contents = fp.read()
+        # Do we expunge the cache file?
+        if expunge:
+            store.delete(entry)
+            os.remove(file_path)
+        return contents
+
+    @dbconnection
+    def evict(self, store):
+        """See `ICacheManager`."""
+        # Find all the cache entries which have expired.  We can probably do
+        # this more efficiently, but for now there probably aren't that many
+        # cached files.
+        for entry in store.query(CacheEntry):
+            if entry.is_expired:
+                file_path, dir_path = self._id_to_path(entry.file_id)
+                os.remove(file_path)
+                store.delete(entry)
+
+    @dbconnection
+    def clear(self, store):
+        # Delete all the entries.  We can probably do this more efficiently,
+        # but for now there probably aren't that many cached files.
+        for entry in store.query(CacheEntry):
+            file_path, dir_path = self._id_to_path(entry.file_id)
+            os.remove(file_path)
+            store.delete(entry)
diff --git a/src/mailman/model/docs/domains.rst b/src/mailman/model/docs/domains.rst
index c0d65cee7..dc117a98e 100644
--- a/src/mailman/model/docs/domains.rst
+++ b/src/mailman/model/docs/domains.rst
@@ -32,24 +32,24 @@ Adding a domain requires some basic information, of which the email host name
 is the only required piece.  The other parts are inferred from that.
 
     >>> manager.add('example.org')
-    <Domain example.org, base_url: http://example.org>
+    <Domain example.org>
     >>> show_domains()
-    <Domain example.org, base_url: http://example.org>
+    <Domain example.org>
 
 We can remove domains too.
 
     >>> manager.remove('example.org')
-    <Domain example.org, base_url: http://example.org>
+    <Domain example.org>
     >>> show_domains()
     no domains
 
 Sometimes the email host name is different than the base url for hitting the
 web interface for the domain.
 
-    >>> manager.add('example.com', base_url='https://mail.example.com')
-    <Domain example.com, base_url: https://mail.example.com>
+    >>> manager.add('example.com')
+    <Domain example.com>
     >>> show_domains()
-    <Domain example.com, base_url: https://mail.example.com>
+    <Domain example.com>
 
 Domains can have explicit descriptions, and can be created with one or more
 owners.
@@ -57,16 +57,13 @@ owners.
 
     >>> manager.add(
     ...     'example.net',
-    ...     base_url='http://lists.example.net',
     ...     description='The example domain',
     ...     owners=['anne@example.com'])
-    <Domain example.net, The example domain,
-            base_url: http://lists.example.net>
+    <Domain example.net, The example domain>
 
     >>> show_domains(with_owners=True)
-    <Domain example.com, base_url: https://mail.example.com>
-    <Domain example.net, The example domain,
-            base_url: http://lists.example.net>
+    <Domain example.com>
+    <Domain example.net, The example domain>
     - owner: anne@example.com
 
 Domains can have multiple owners, ideally one of the owners should have a
@@ -76,8 +73,8 @@ configuration's default contact address may be used as a fallback.
    >>> net_domain = manager['example.net']
    >>> net_domain.add_owner('bart@example.org')
    >>> show_domains(with_owners=True)
-   <Domain example.com, base_url: https://mail.example.com>
-   <Domain example.net, The example domain, base_url: http://lists.example.net>
+   <Domain example.com>
+   <Domain example.net, The example domain>
    - owner: anne@example.com
    - owner: bart@example.org
 
@@ -114,30 +111,17 @@ In the global domain manager, domains are indexed by their email host name.
     example.net
 
     >>> print(manager['example.net'])
-    <Domain example.net, The example domain,
-            base_url: http://lists.example.net>
+    <Domain example.net, The example domain>
 
 As with dictionaries, you can also get the domain.  If the domain does not
 exist, ``None`` or a default is returned.
 ::
 
     >>> print(manager.get('example.net'))
-    <Domain example.net, The example domain,
-            base_url: http://lists.example.net>
+    <Domain example.net, The example domain>
 
     >>> print(manager.get('doesnotexist.com'))
     None
 
     >>> print(manager.get('doesnotexist.com', 'blahdeblah'))
     blahdeblah
-
-
-Confirmation tokens
-===================
-
-Confirmation tokens can be added to the domain's url to generate the URL to a
-page users can use to confirm their subscriptions.
-
-    >>> domain = manager['example.net']
-    >>> print(domain.confirm_url('abc'))
-    http://lists.example.net/confirm/abc
diff --git a/src/mailman/model/domain.py b/src/mailman/model/domain.py
index 5eb67fa95..575d157cf 100644
--- a/src/mailman/model/domain.py
+++ b/src/mailman/model/domain.py
@@ -28,7 +28,6 @@ from mailman.interfaces.usermanager import IUserManager
 from mailman.model.mailinglist import MailingList
 from sqlalchemy import Column, Integer, Unicode
 from sqlalchemy.orm import relationship
-from urllib.parse import urljoin, urlparse
 from zope.component import getUtility
 from zope.event import notify
 from zope.interface import implementer
@@ -44,7 +43,6 @@ class Domain(Model):
     id = Column(Integer, primary_key=True)
 
     mail_host = Column(Unicode)
-    base_url = Column(Unicode)
     description = Column(Unicode)
     owners = relationship('User',
                           secondary='domain_owner',
@@ -52,7 +50,6 @@ class Domain(Model):
 
     def __init__(self, mail_host,
                  description=None,
-                 base_url=None,
                  owners=None):
         """Create and register a domain.
 
@@ -60,32 +57,15 @@ class Domain(Model):
         :type mail_host: string
         :param description: An optional description of the domain.
         :type description: string
-        :param base_url: The optional base url for the domain, including
-            scheme.  If not given, it will be constructed from the
-            `mail_host` using the http protocol.
-        :type base_url: string
         :param owners: Optional owners of this domain.
         :type owners: sequence of `IUser` or string emails.
         """
         self.mail_host = mail_host
-        self.base_url = (base_url
-                         if base_url is not None
-                         else 'http://' + mail_host)
         self.description = description
         if owners is not None:
             self.add_owners(owners)
 
     @property
-    def url_host(self):
-        """See `IDomain`."""
-        return urlparse(self.base_url).netloc
-
-    @property
-    def scheme(self):
-        """See `IDomain`."""
-        return urlparse(self.base_url).scheme
-
-    @property
     @dbconnection
     def mailing_lists(self, store):
         """See `IDomain`."""
@@ -93,18 +73,12 @@ class Domain(Model):
             MailingList.mail_host == self.mail_host
             ).order_by(MailingList._list_id)
 
-    def confirm_url(self, token=''):
-        """See `IDomain`."""
-        return urljoin(self.base_url, 'confirm/' + token)
-
     def __repr__(self):
         """repr(a_domain)"""
         if self.description is None:
-            return ('<Domain {0.mail_host}, base_url: {0.base_url}>').format(
-                self)
+            return ('<Domain {0.mail_host}>').format(self)
         else:
-            return ('<Domain {0.mail_host}, {0.description}, '
-                    'base_url: {0.base_url}>').format(self)
+            return ('<Domain {0.mail_host}, {0.description}>').format(self)
 
     def add_owner(self, owner):
         """See `IDomain`."""
@@ -140,7 +114,6 @@ class DomainManager:
     def add(self, store,
             mail_host,
             description=None,
-            base_url=None,
             owners=None):
         """See `IDomainManager`."""
         # Be sure the mail_host is not already registered.  This is probably
@@ -149,7 +122,7 @@ class DomainManager:
             raise BadDomainSpecificationError(
                 'Duplicate email host: {}'.format(mail_host))
         notify(DomainCreatingEvent(mail_host))
-        domain = Domain(mail_host, description, base_url, owners)
+        domain = Domain(mail_host, description, owners)
         store.add(domain)
         notify(DomainCreatedEvent(domain))
         return domain
diff --git a/src/mailman/model/mailinglist.py b/src/mailman/model/mailinglist.py
index b147acd72..9d7d63d9b 100644
--- a/src/mailman/model/mailinglist.py
+++ b/src/mailman/model/mailinglist.py
@@ -56,7 +56,6 @@ from sqlalchemy.event import listen
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import relationship
 from sqlalchemy.orm.exc import NoResultFound
-from urllib.parse import urljoin
 from zope.component import getUtility
 from zope.event import notify
 from zope.interface import implementer
@@ -133,8 +132,6 @@ class MailingList(Model):
     default_nonmember_action = Column(Enum(Action))
     description = Column(Unicode)
     digests_enabled = Column(Boolean)
-    digest_footer_uri = Column(Unicode)
-    digest_header_uri = Column(Unicode)
     digest_is_default = Column(Boolean)
     digest_send_periodic = Column(Boolean)
     digest_size_threshold = Column(Float)
@@ -143,12 +140,9 @@ class MailingList(Model):
     emergency = Column(Boolean)
     encode_ascii_prefixes = Column(Boolean)
     first_strip_reply_to = Column(Boolean)
-    footer_uri = Column(Unicode)
     forward_auto_discards = Column(Boolean)
     gateway_to_mail = Column(Boolean)
     gateway_to_news = Column(Boolean)
-    goodbye_message_uri = Column(Unicode)
-    header_uri = Column(Unicode)
     hold_these_nonmembers = Column(PickleType)
     info = Column(Unicode)
     linked_newsgroup = Column(Unicode)
@@ -184,7 +178,6 @@ class MailingList(Model):
     topics = Column(PickleType)
     topics_bodylines_limit = Column(Integer)
     topics_enabled = Column(Boolean)
-    welcome_message_uri = Column(Unicode)
     # ORM relationships.
     header_matches = relationship(
         'HeaderMatch', backref='mailing_list',
@@ -245,22 +238,6 @@ class MailingList(Model):
         return getUtility(IDomainManager)[self.mail_host]
 
     @property
-    def scheme(self):
-        """See `IMailingList`."""
-        return self.domain.scheme
-
-    @property
-    def web_host(self):
-        """See `IMailingList`."""
-        return self.domain.url_host
-
-    def script_url(self, target, context=None):
-        """See `IMailingList`."""
-        # XXX Handle the case for when context is not None; those would be
-        # relative URLs.
-        return urljoin(self.domain.base_url, target + '/' + self.fqdn_listname)
-
-    @property
     def data_path(self):
         """See `IMailingList`."""
         return os.path.join(config.LIST_DATA_DIR, self.list_id)
@@ -314,7 +291,7 @@ class MailingList(Model):
 
     def confirm_address(self, cookie):
         """See `IMailingList`."""
-        local_part = expand(config.mta.verp_confirm_format, dict(
+        local_part = expand(config.mta.verp_confirm_format, self, dict(
             address='{}-confirm'.format(self.list_name),
             cookie=cookie))
         return '{}@{}'.format(local_part, self.mail_host)
diff --git a/src/mailman/model/member.py b/src/mailman/model/member.py
index 26c36b305..3aaa5e392 100644
--- a/src/mailman/model/member.py
+++ b/src/mailman/model/member.py
@@ -131,6 +131,21 @@ class Member(Model):
     def subscriber(self):
         return (self._user if self._address is None else self._address)
 
+    @property
+    def display_name(self):
+        # Try to find a non-empty display name.  We first look at the directly
+        # subscribed record, which will either be the address or the user.
+        # That's handled automatically by going through member.subscriber.  If
+        # that doesn't give us something useful, try whatever user is linked
+        # to the subscriber.
+        if self.subscriber.display_name:
+            return self.subscriber.display_name
+        # If an unlinked address is subscribed tehre will be no .user.
+        elif self.user is not None and self.user.display_name:
+            return self.user.display_name
+        else:
+            return ''
+
     def _lookup(self, preference, default=None):
         pref = getattr(self.preferences, preference)
         if pref is not None:
@@ -180,12 +195,6 @@ class Member(Model):
         """See `IMember`."""
         return self._lookup('delivery_status')
 
-    @property
-    def options_url(self):
-        """See `IMember`."""
-        # XXX Um, this is definitely wrong
-        return 'http://example.com/' + self.address.email
-
     @dbconnection
     def unsubscribe(self, store):
         """See `IMember`."""
diff --git a/src/mailman/model/template.py b/src/mailman/model/template.py
new file mode 100644
index 000000000..96ce43e1f
--- /dev/null
+++ b/src/mailman/model/template.py
@@ -0,0 +1,202 @@
+# Copyright (C) 2016 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Template management."""
+
+import logging
+
+from mailman import public
+from mailman.config import config
+from mailman.database.model import Model
+from mailman.database.transaction import dbconnection
+from mailman.interfaces.cache import ICacheManager
+from mailman.interfaces.domain import IDomain
+from mailman.interfaces.mailinglist import IMailingList
+from mailman.interfaces.template import (
+    ALL_TEMPLATES, ITemplateLoader, ITemplateManager)
+from mailman.utilities import protocols
+from mailman.utilities.i18n import find
+from mailman.utilities.string import expand
+from requests import HTTPError
+from sqlalchemy import Column, Integer, Unicode
+from urllib.error import URLError
+from urllib.parse import urlparse
+from zope.component import getUtility
+from zope.interface import implementer
+
+
+COMMASPACE = ', '
+log = logging.getLogger('mailman.http')
+
+
+class Template(Model):
+    __tablename__ = 'template'
+
+    id = Column(Integer, primary_key=True)
+    name = Column(Unicode)
+    context = Column(Unicode)
+    uri = Column(Unicode)
+    username = Column(Unicode, nullable=True)
+    password = Column(Unicode, nullable=True)
+
+    def __init__(self, name, context, uri, username, password):
+        self.name = name
+        self.context = context
+        self.reset(uri, username, password)
+
+    def reset(self, uri, username, password):
+        self.uri = uri
+        self.username = username
+        self.password = password
+
+
+@public
+@implementer(ITemplateManager)
+class TemplateManager:
+    """Manager of templates, with caching and support for mailman:// URIs."""
+
+    @dbconnection
+    def set(self, store, name, context, uri, username=None, password=''):
+        """See `ITemplateManager`."""
+        # Just record the fact that we have a template set.  Make sure that if
+        # there is an existing template with the same context and name, we
+        # override any of its settings (and evict the cache).
+        template = store.query(Template).filter(
+            Template.name == name,
+            Template.context == context).one_or_none()
+        if template is None:
+            template = Template(name, context, uri, username, password)
+            store.add(template)
+        else:
+            template.reset(uri, username, password)
+
+    @dbconnection
+    def get(self, store, name, context, **kws):
+        """See `ITemplateManager`."""
+        template = store.query(Template).filter(
+            Template.name == name,
+            Template.context == context).one_or_none()
+        if template is None:
+            return None
+        actual_uri = expand(template.uri, None, kws)
+        cache_mgr = getUtility(ICacheManager)
+        contents = cache_mgr.get(actual_uri)
+        if contents is None:
+            # It's likely that the cached contents have expired.
+            auth = {}
+            if template.username is not None:
+                auth['auth'] = (template.username, template.password)
+            try:
+                contents = protocols.get(actual_uri, **auth)
+            except HTTPError as error:
+                # 404/NotFound errors are interpreted as missing templates,
+                # for which we'll return the default (i.e. the empty string).
+                # All other exceptions get passed up the chain.
+                if error.response.status_code != 404:
+                    raise
+                log.exception('Cannot retrieve template at {} ({})'.format(
+                    actual_uri, auth.get('auth', '<no authorization>')))
+                return ''
+            # We don't need to cache mailman: contents since those are already
+            # on the file system.
+            if urlparse(actual_uri).scheme != 'mailman':
+                cache_mgr.add(actual_uri, contents)
+        return contents
+
+    @dbconnection
+    def raw(self, store, name, context):
+        """See `ITemplateManager`."""
+        return store.query(Template).filter(
+            Template.name == name,
+            Template.context == context).one_or_none()
+
+    @dbconnection
+    def delete(self, store, name, context):
+        """See `ITemplateManager`."""
+        template = store.query(Template).filter(
+            Template.name == name,
+            Template.context == context).one_or_none()
+        if template is not None:
+            store.delete(template)
+        # We don't clear the cache entry, we just let it expire.
+
+
+@public
+@implementer(ITemplateLoader)
+class TemplateLoader:
+    """Loader of templates."""
+
+    def get(self, name, context=None, **kws):
+        """See `ITemplateLoader`."""
+        # Gather some additional information based on the context.
+        substitutions = {}
+        if IMailingList.providedBy(context):
+            mlist = context
+            domain = context.domain
+            lookup_contexts = [
+                mlist.list_id,
+                mlist.mail_host,
+                None,
+                ]
+            substitutions.update(dict(
+                list_id=mlist.list_id,
+                # For backward compatibility, we call this $listname.
+                listname=mlist.fqdn_listname,
+                domain_name=domain.mail_host,
+                language=mlist.preferred_language.code,
+                ))
+        elif IDomain.providedBy(context):
+            mlist = None
+            domain = context
+            lookup_contexts = [
+                domain.mail_host,
+                None,
+                ]
+            substitutions['domain_name'] = domain.mail_host
+        elif context is None:
+            mlist = domain = None
+            lookup_contexts = [None]
+        else:
+            raise ValueError('Bad context type: {!r}'.format(context))
+        # The passed in keyword arguments take precedence.
+        substitutions.update(kws)
+        # See if there's a cached template registered for this name and
+        # context, passing in the url substitutions.  This handles http:,
+        # https:, and file: urls.
+        for lookup_context in lookup_contexts:
+            try:
+                contents = getUtility(ITemplateManager).get(
+                    name, lookup_context, **substitutions)
+            except (HTTPError, URLError):
+                pass
+            else:
+                if contents is not None:
+                    return contents
+        # Fallback to searching within the source code.
+        code = substitutions.get('language', config.mailman.default_language)
+        # Find the template, mutating any missing template exception.
+        missing = object()
+        default_uri = ALL_TEMPLATES.get(name, missing)
+        if default_uri is None:
+            return ''
+        elif default_uri is missing:
+            raise URLError('No such file')
+        path, fp = find(default_uri, mlist, code)
+        try:
+            return fp.read()
+        finally:
+            fp.close()
diff --git a/src/mailman/model/tests/test_cache.py b/src/mailman/model/tests/test_cache.py
new file mode 100644
index 000000000..2d910e202
--- /dev/null
+++ b/src/mailman/model/tests/test_cache.py
@@ -0,0 +1,110 @@
+# Copyright (C) 2016 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Test the cache."""
+
+import os
+import unittest
+
+from datetime import timedelta
+from mailman.config import config
+from mailman.interfaces.cache import ICacheManager
+from mailman.testing.helpers import configuration
+from mailman.testing.layers import ConfigLayer
+from mailman.utilities.datetime import factory
+from zope.component import getUtility
+
+
+class TestCache(unittest.TestCase):
+    layer = ConfigLayer
+
+    def setUp(self):
+        self._cachemgr = getUtility(ICacheManager)
+
+    def test_add_str_contents(self):
+        file_id = self._cachemgr.add('abc', 'xyz')
+        self.assertEqual(
+            file_id,
+            'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad')
+        file_path = os.path.join(config.CACHE_DIR, 'ba', '78', file_id)
+        self.assertTrue(os.path.exists(file_path))
+        # The original content was a string.
+        with open(file_path, 'r', encoding='utf-8') as fp:
+            self.assertEqual(fp.read(), 'xyz')
+
+    def test_add_bytes_contents(self):
+        # No name is given so the file is cached by the hash of the contents.
+        file_id = self._cachemgr.add('abc', b'xyz')
+        self.assertEqual(
+            file_id,
+            'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad')
+        file_path = os.path.join(config.CACHE_DIR, 'ba', '78', file_id)
+        self.assertTrue(os.path.exists(file_path))
+        # The original content was a string.
+        with open(file_path, 'br') as fp:
+            self.assertEqual(fp.read(), b'xyz')
+
+    def test_add_overwrite(self):
+        # If the file already exists and hasn't expired, a conflict exception
+        # is raised the second time we try to save it.
+        self._cachemgr.add('abc', 'xyz')
+        self.assertEqual(self._cachemgr.get('abc'), 'xyz')
+        self._cachemgr.add('abc', 'def')
+        self.assertEqual(self._cachemgr.get('abc'), 'def')
+
+    def test_get_str(self):
+        # Store a str, get a str.
+        self._cachemgr.add('abc', 'xyz')
+        contents = self._cachemgr.get('abc')
+        self.assertEqual(contents, 'xyz')
+
+    def test_get_bytes(self):
+        # Store a bytes, get a bytes.
+        self._cachemgr.add('abc', b'xyz')
+        contents = self._cachemgr.get('abc')
+        self.assertEqual(contents, b'xyz')
+
+    def test_get_str_expunge(self):
+        # When the entry is not expunged, it can be gotten multiple times.
+        # Once it's expunged, it's gone.
+        self._cachemgr.add('abc', 'xyz')
+        self.assertEqual(self._cachemgr.get('abc'), 'xyz')
+        self.assertEqual(self._cachemgr.get('abc', expunge=True), 'xyz')
+        self.assertIsNone(self._cachemgr.get('abc'))
+
+    @configuration('mailman', cache_life='1d')
+    def test_evict(self):
+        # Evicting all expired cache entries makes them inaccessible.
+        self._cachemgr.add('abc', 'xyz', lifetime=timedelta(hours=3))
+        self._cachemgr.add('def', 'uvw', lifetime=timedelta(days=3))
+        self.assertEqual(self._cachemgr.get('abc'), 'xyz')
+        self.assertEqual(self._cachemgr.get('def'), 'uvw')
+        factory.fast_forward(days=1)
+        self._cachemgr.evict()
+        self.assertIsNone(self._cachemgr.get('abc'))
+        self.assertEqual(self._cachemgr.get('def'), 'uvw')
+
+    def test_clear(self):
+        # Clearing the cache gets rid of all entries, regardless of lifetime.
+        self._cachemgr.add('abc', 'xyz', lifetime=timedelta(hours=3))
+        self._cachemgr.add('def', 'uvw')
+        self.assertEqual(self._cachemgr.get('abc'), 'xyz')
+        self.assertEqual(self._cachemgr.get('def'), 'uvw')
+        factory.fast_forward(days=1)
+        self._cachemgr.clear()
+        self.assertIsNone(self._cachemgr.get('abc'))
+        self.assertIsNone(self._cachemgr.get('xyz'))
diff --git a/src/mailman/model/tests/test_template.py b/src/mailman/model/tests/test_template.py
new file mode 100644
index 000000000..f16a4aa5c
--- /dev/null
+++ b/src/mailman/model/tests/test_template.py
@@ -0,0 +1,286 @@
+# Copyright (C) 2016 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Test the template manager."""
+
+import unittest
+import threading
+
+from contextlib import ExitStack
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from mailman.app.lifecycle import create_list
+from mailman.config import config
+from mailman.interfaces.domain import IDomainManager
+from mailman.interfaces.template import ITemplateLoader, ITemplateManager
+from mailman.testing.helpers import wait_for_webservice
+from mailman.testing.layers import ConfigLayer
+from mailman.utilities.i18n import find
+from requests import HTTPError
+from tempfile import TemporaryDirectory
+from urllib.error import URLError
+from zope.component import getUtility
+
+# New in Python 3.5.
+try:
+    from http import HTTPStatus
+except ImportError:
+    class HTTPStatus:
+        FORBIDDEN = 403
+        NOT_FOUND = 404
+        OK = 200
+
+
+# We need a web server to vend non-mailman: urls.
+class TestableHandler(BaseHTTPRequestHandler):
+    # Be quiet.
+    def log_request(*args, **kws):
+        pass
+
+    log_error = log_request
+
+    def do_GET(self):
+        if self.path == '/welcome_3.txt':
+            if self.headers['Authorization'] != 'Basic YW5uZTppcyBzcGVjaWFs':
+                self.send_error(HTTPStatus.FORBIDDEN)
+                return
+        response = TEXTS.get(self.path)
+        if response is None:
+            self.send_error(HTTPStatus.NOT_FOUND)
+            return
+        self.send_response(HTTPStatus.OK)
+        self.send_header('Content-Type', 'UTF-8')
+        self.end_headers()
+        self.wfile.write(response.encode('utf-8'))
+
+
+class HTTPLayer(ConfigLayer):
+    httpd = None
+
+    @classmethod
+    def setUp(cls):
+        assert cls.httpd is None, 'Layer already set up'
+        cls.httpd = HTTPServer(('localhost', 8180), TestableHandler)
+        cls._thread = threading.Thread(target=cls.httpd.serve_forever)
+        cls._thread.daemon = True
+        cls._thread.start()
+        wait_for_webservice('localhost', 8180)
+
+    @classmethod
+    def tearDown(cls):
+        assert cls.httpd is not None, 'Layer not set up'
+        cls.httpd.shutdown()
+        cls.httpd.server_close()
+        cls._thread.join()
+
+
+class TestTemplateCache(unittest.TestCase):
+    layer = HTTPLayer
+
+    def setUp(self):
+        self._templatemgr = getUtility(ITemplateManager)
+
+    def test_http_set_get(self):
+        self._templatemgr.set(
+            'list:user:notice:welcome', 'test.example.com',
+            'http://localhost:8180/welcome_1.txt')
+        contents = self._templatemgr.get(
+            'list:user:notice:welcome', 'test.example.com')
+        self.assertEqual(contents, WELCOME_1)
+
+    def test_http_set_override(self):
+        self._templatemgr.set(
+            'list:user:notice:welcome', 'test.example.com',
+            'http://localhost:8180/welcome_1.txt')
+        # Resetting the template with the same context and domain, but a
+        # different url overrides the previous value.
+        self._templatemgr.set(
+            'list:user:notice:welcome', 'test.example.com',
+            'http://localhost:8180/welcome_2.txt')
+        contents = self._templatemgr.get(
+            'list:user:notice:welcome', 'test.example.com')
+        self.assertEqual(contents, WELCOME_2)
+
+    def test_http_get_cached(self):
+        self._templatemgr.set(
+            'list:user:notice:welcome', 'test.example.com',
+            'http://localhost:8180/welcome_1.txt')
+        # The first one warms the cache.
+        self._templatemgr.get('list:user:notice:welcome', 'test.example.com')
+        # The second one hits the cache.
+        contents = self._templatemgr.get(
+            'list:user:notice:welcome', 'test.example.com')
+        self.assertEqual(contents, WELCOME_1)
+
+    def test_http_basic_auth(self):
+        # We get an HTTP error when we forget the username and password.
+        self._templatemgr.set(
+            'list:user:notice:welcome', 'test.example.com',
+            'http://localhost:8180/welcome_3.txt')
+        with self.assertRaises(HTTPError) as cm:
+            self._templatemgr.get(
+                'list:user:notice:welcome', 'test.example.com')
+        self.assertEqual(cm.exception.response.status_code, 403)
+        self.assertEqual(cm.exception.response.reason, 'Forbidden')
+        # But providing the basic auth information let's it work.
+        self._templatemgr.set(
+            'list:user:notice:welcome', 'test.example.com',
+            'http://localhost:8180/welcome_3.txt',
+            username='anne', password='is special')
+        contents = self._templatemgr.get(
+            'list:user:notice:welcome', 'test.example.com')
+        self.assertEqual(contents, WELCOME_3)
+
+    def test_delete(self):
+        self._templatemgr.set(
+            'list:user:notice:welcome', 'test.example.com',
+            'http://localhost:8180/welcome_1.txt')
+        self._templatemgr.get('list:user:notice:welcome', 'test.example.com')
+        self._templatemgr.delete(
+            'list:user:notice:welcome', 'test.example.com')
+        self.assertIsNone(
+            self._templatemgr.get(
+                'list:user:notice:welcome', 'test.example.com'))
+
+    def test_delete_missing(self):
+        self._templatemgr.delete(
+            'list:user:notice:welcome', 'test.example.com')
+        self.assertIsNone(
+            self._templatemgr.get(
+                'list:user:notice:welcome', 'test.example.com'))
+
+    def test_get_keywords(self):
+        self._templatemgr.set(
+            'list:user:notice:welcome', 'test.example.com',
+            'http://localhost:8180/${path}_${number}.txt')
+        contents = self._templatemgr.get(
+            'list:user:notice:welcome', 'test.example.com',
+            path='welcome', number='1')
+        self.assertEqual(contents, WELCOME_1)
+
+    def test_get_different_keywords(self):
+        self._templatemgr.set(
+            'list:user:notice:welcome', 'test.example.com',
+            'http://localhost:8180/${path}_${number}.txt')
+        contents = self._templatemgr.get(
+            'list:user:notice:welcome', 'test.example.com',
+            path='welcome', number='1')
+        self.assertEqual(contents, WELCOME_1)
+        contents = self._templatemgr.get(
+            'list:user:notice:welcome', 'test.example.com',
+            path='welcome', number='2')
+        self.assertEqual(contents, WELCOME_2)
+
+    def test_not_found(self):
+        # A 404 is treated specially, resulting in the empty string.
+        self._templatemgr.set(
+            'list:user:notice:welcome', 'test.example.com',
+            'http://localhost:8180/missing.txt')
+        contents = self._templatemgr.get(
+            'list:user:notice:welcome', 'test.example.com')
+        self.assertEqual(contents, '')
+
+
+class TestTemplateLoader(unittest.TestCase):
+    """Test the template downloader API."""
+
+    layer = HTTPLayer
+
+    def setUp(self):
+        resources = ExitStack()
+        self.addCleanup(resources.close)
+        var_dir = resources.enter_context(TemporaryDirectory())
+        config.push('template config', """\
+        [paths.testing]
+        var_dir: {}
+        """.format(var_dir))
+        resources.callback(config.pop, 'template config')
+        self._mlist = create_list('test@example.com')
+        self._loader = getUtility(ITemplateLoader)
+        self._manager = getUtility(ITemplateManager)
+
+    def test_domain_context(self):
+        self._manager.set(
+            'list:user:notice:welcome', 'example.com',
+            'http://localhost:8180/$domain_name/welcome_4.txt')
+        domain = getUtility(IDomainManager).get('example.com')
+        content = self._loader.get('list:user:notice:welcome', domain)
+        self.assertEqual(content, 'This is a domain welcome.\n')
+
+    def test_domain_content_fallback(self):
+        self._manager.set(
+            'list:user:notice:welcome', 'example.com',
+            'http://localhost:8180/$domain_name/welcome_4.txt')
+        content = self._loader.get('list:user:notice:welcome', self._mlist)
+        self.assertEqual(content, 'This is a domain welcome.\n')
+
+    def test_site_context(self):
+        self._manager.set(
+            'list:user:notice:welcome', None,
+            'http://localhost:8180/welcome_2.txt')
+        content = self._loader.get('list:user:notice:welcome')
+        self.assertEqual(content, "Sure, I guess you're welcome.\n")
+
+    def test_site_context_mailman(self):
+        self._manager.set(
+            'list:user:notice:welcome', None,
+            'mailman:///welcome.txt')
+        template_content = self._loader.get('list:user:notice:welcome')
+        path, fp = find('list:user:notice:welcome.txt')
+        try:
+            found_contents = fp.read()
+        finally:
+            fp.close()
+        self.assertEqual(template_content, found_contents)
+
+    def test_bad_context(self):
+        self.assertRaises(
+            ValueError, self._loader.get, 'list:user:notice:welcome', object())
+
+    def test_no_such_file(self):
+        self.assertRaises(URLError, self._loader.get, 'missing', self._mlist)
+
+    def test_403_forbidden(self):
+        # 404s are swallowed, but not 403s.
+        self._manager.set(
+            'forbidden', 'test.example.com',
+            'http://localhost:8180/welcome_3.txt')
+        self.assertRaises(URLError, self._loader.get, 'forbidden', self._mlist)
+
+
+# Response texts.
+WELCOME_1 = """\
+Welcome to the {fqdn_listname} mailing list!
+"""
+
+WELCOME_2 = """\
+Sure, I guess you're welcome.
+"""
+
+WELCOME_3 = """\
+Well?  Come.
+"""
+
+WELCOME_4 = """\
+This is a domain welcome.
+"""
+
+TEXTS = {
+    '/welcome_1.txt': WELCOME_1,
+    '/welcome_2.txt': WELCOME_2,
+    '/welcome_3.txt': WELCOME_3,
+    '/example.com/welcome_4.txt': WELCOME_4,
+    }