3 files changed, 220 insertions, 96 deletions

diff --git a/src/mailman/utilities/tests/test_import.py b/src/mailman/utilities/tests/test_import.py
index b2d48891b..d2729e8e1 100644
--- a/src/mailman/utilities/tests/test_import.py
+++ b/src/mailman/utilities/tests/test_import.py
@@ -31,22 +31,23 @@ from mailman.interfaces.archiver import ArchivePolicy
 from mailman.interfaces.autorespond import ResponseAction
 from mailman.interfaces.bans import IBanManager
 from mailman.interfaces.bounce import UnrecognizedBounceDisposition
+from mailman.interfaces.domain import IDomainManager
 from mailman.interfaces.languages import ILanguageManager
 from mailman.interfaces.mailinglist import (
     IAcceptableAliasSet, SubscriptionPolicy)
 from mailman.interfaces.member import DeliveryMode, DeliveryStatus
 from mailman.interfaces.nntp import NewsgroupModeration
-from mailman.interfaces.templates import ITemplateLoader
+from mailman.interfaces.template import ITemplateLoader, ITemplateManager
 from mailman.interfaces.usermanager import IUserManager
 from mailman.testing.helpers import LogFileMark
 from mailman.testing.layers import ConfigLayer
 from mailman.utilities.filesystem import makedirs
 from mailman.utilities.importer import (
     Import21Error, check_language_code, import_config_pck)
-from mailman.utilities.string import expand
 from pickle import load
 from pkg_resources import resource_filename
 from unittest import mock
+from urllib.error import URLError
 from zope.component import getUtility
 
 
@@ -638,16 +639,17 @@ class TestConvertToURI(unittest.TestCase):
     #       -> %(listinfo_uri)s
 
     layer = ConfigLayer
+    maxDiff = None
 
     def setUp(self):
         self._mlist = create_list('blank@example.com')
         self._conf_mapping = dict(
-            welcome_msg='welcome_message_uri',
-            goodbye_msg='goodbye_message_uri',
-            msg_header='header_uri',
-            msg_footer='footer_uri',
-            digest_header='digest_header_uri',
-            digest_footer='digest_footer_uri',
+            welcome_msg='list:user:notice:welcome',
+            goodbye_msg='list:user:notice:goodbye',
+            msg_header='list:member:regular:header',
+            msg_footer='list:member:regular:footer',
+            digest_header='list:member:digest:header',
+            digest_footer='list:member:digest:footer',
             )
         self._pckdict = dict()
 
@@ -655,8 +657,7 @@ class TestConvertToURI(unittest.TestCase):
         for oldvar, newvar in self._conf_mapping.items():
             self._pckdict[str(oldvar)] = b'TEST VALUE'
             import_config_pck(self._mlist, self._pckdict)
-            newattr = getattr(self._mlist, newvar)
-            text = decorate(self._mlist, newattr)
+            text = decorate(newvar, self._mlist)
             self.assertEqual(
                 text, 'TEST VALUE',
                 'Old variable %s was not properly imported to %s'
@@ -664,21 +665,13 @@ class TestConvertToURI(unittest.TestCase):
 
     def test_substitutions(self):
         test_text = ('UNIT TESTING %(real_name)s mailing list\n'
-                     '%(real_name)s@%(host_name)s\n'
-                     '%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s')
+                     '%(real_name)s@%(host_name)s')
         expected_text = ('UNIT TESTING $display_name mailing list\n'
-                         '$fqdn_listname\n'
-                         '$listinfo_uri')
+                         '$listname')
         for oldvar, newvar in self._conf_mapping.items():
             self._pckdict[str(oldvar)] = str(test_text)
             import_config_pck(self._mlist, self._pckdict)
-            newattr = getattr(self._mlist, newvar)
-            template_uri = expand(newattr, dict(
-                listname=self._mlist.fqdn_listname,
-                language=self._mlist.preferred_language.code,
-                ))
-            loader = getUtility(ITemplateLoader)
-            text = loader.get(template_uri)
+            text = getUtility(ITemplateLoader).get(newvar, self._mlist)
             self.assertEqual(
                 text, expected_text,
                 'Old variables were not converted for %s' % newvar)
@@ -689,35 +682,53 @@ class TestConvertToURI(unittest.TestCase):
             '_______________________________________________\n'
             '%(real_name)s mailing list\n'
             '%(real_name)s@%(host_name)s\n'
-            '%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s'
+            '%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s\n'
             )
+        loader = getUtility(ITemplateLoader)
         for oldvar in ('msg_footer', 'digest_footer'):
             newvar = self._conf_mapping[oldvar]
             self._pckdict[str(oldvar)] = str(default_msg_footer)
-            old_value = getattr(self._mlist, newvar)
+            try:
+                old_value = loader.get(newvar, self._mlist)
+            except URLError:
+                old_value = None
             import_config_pck(self._mlist, self._pckdict)
-            new_value = getattr(self._mlist, newvar)
+            try:
+                new_value = loader.get(newvar, self._mlist)
+            except URLError:
+                new_value = None
             self.assertEqual(
                 old_value, new_value,
-                'Default value was not preserved for %s' % newvar)
+                '{} changed unexpectedly: {} != {}'.format(
+                    newvar, old_value, new_value))
 
     def test_keep_default_if_fqdn_changed(self):
         # Use case: importing the old a@ex.com into b@ex.com.  We can't check
         # if it changed from the default so don't import.  We may do more harm
         # than good and it's easy to change if needed.
         test_value = b'TEST-VALUE'
+        # We need an IDomain for this mail_host.
+        getUtility(IDomainManager).add('test.example.com')
+        manager = getUtility(ITemplateManager)
         for oldvar, newvar in self._conf_mapping.items():
             self._mlist.mail_host = 'example.com'
             self._pckdict['mail_host'] = b'test.example.com'
             self._pckdict[str(oldvar)] = test_value
-            old_value = getattr(self._mlist, newvar)
+            try:
+                old_value = manager.get(newvar, 'blank.example.com')
+            except URLError:
+                old_value = None
             # Suppress warning messages in the test output.
             with mock.patch('sys.stderr'):
                 import_config_pck(self._mlist, self._pckdict)
-            new_value = getattr(self._mlist, newvar)
+            try:
+                new_value = manager.get(newvar, 'test.example.com')
+            except URLError:
+                new_value = None
             self.assertEqual(
                 old_value, new_value,
-                'Default value was not preserved for %s' % newvar)
+                '{} changed unexpectedly: {} != {}'.format(
+                    newvar, old_value, new_value))
 
     def test_unicode(self):
         # non-ascii templates
@@ -725,10 +736,11 @@ class TestConvertToURI(unittest.TestCase):
             self._pckdict[str(oldvar)] = b'Ol\xe1!'
         import_config_pck(self._mlist, self._pckdict)
         for oldvar, newvar in self._conf_mapping.items():
-            newattr = getattr(self._mlist, newvar)
-            text = decorate(self._mlist, newattr)
+            text = decorate(newvar, self._mlist)
             expected = u'Ol\ufffd!'
-            self.assertEqual(text, expected)
+            self.assertEqual(
+                text, expected,
+                '{} -> {} did not get converted'.format(oldvar, newvar))
 
     def test_unicode_in_default(self):
         # What if the default template is already in UTF-8?   For example, if
@@ -736,13 +748,13 @@ class TestConvertToURI(unittest.TestCase):
         footer = b'\xe4\xb8\xad $listinfo_uri'
         footer_path = os.path.join(
             config.VAR_DIR, 'templates', 'lists',
-            'blank@example.com', 'en', 'footer-generic.txt')
+            'blank@example.com', 'en', 'footer.txt')
         makedirs(os.path.dirname(footer_path))
         with open(footer_path, 'wb') as fp:
             fp.write(footer)
         self._pckdict['msg_footer'] = b'NEW-VALUE'
         import_config_pck(self._mlist, self._pckdict)
-        text = decorate(self._mlist, self._mlist.footer_uri)
+        text = decorate('list:member:regular:footer', self._mlist)
         self.assertEqual(text, 'NEW-VALUE')
 
 
diff --git a/src/mailman/utilities/tests/test_protocols.py b/src/mailman/utilities/tests/test_protocols.py
new file mode 100644
index 000000000..dc5e46e6a
--- /dev/null
+++ b/src/mailman/utilities/tests/test_protocols.py
@@ -0,0 +1,174 @@
+# Copyright (C) 2016 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Test the protocol support.
+
+For convenience, we currently don't test the http: and https: schemes here.
+These are tested fairly well in the template cache tests.  We probably
+eventually want to refactor that for test isolation.
+"""
+
+
+import os
+import unittest
+
+from contextlib import ExitStack
+from mailman.app.lifecycle import create_list
+from mailman.config import config
+from mailman.testing.layers import ConfigLayer
+from mailman.utilities import protocols
+from tempfile import TemporaryDirectory
+from urllib.error import URLError
+
+
+class TestProtocols(unittest.TestCase):
+    layer = ConfigLayer
+
+    def setUp(self):
+        resources = ExitStack()
+        self.addCleanup(resources.close)
+        self.var_dir = resources.enter_context(TemporaryDirectory())
+        config.push('template config', """\
+        [paths.testing]
+        var_dir: {}
+        """.format(self.var_dir))
+        resources.callback(config.pop, 'template config')
+        # Put a demo template in the site directory.
+        path = os.path.join(self.var_dir, 'templates', 'site', 'en')
+        os.makedirs(path)
+        with open(os.path.join(path, 'demo.txt'), 'w') as fp:
+            print('Test content', end='', file=fp)
+        self._mlist = create_list('test@example.com')
+
+    def test_file(self):
+        with TemporaryDirectory() as tempdir:
+            path = os.path.join(tempdir, 'my-file')
+            with open(path, 'w', encoding='utf-8') as fp:
+                print('Some contents', end='', file=fp)
+            contents = protocols.get('file:///{}'.format(path))
+            self.assertEqual(contents, 'Some contents')
+
+    def test_file_binary(self):
+        with TemporaryDirectory() as tempdir:
+            path = os.path.join(tempdir, 'my-file')
+            with open(path, 'wb') as fp:
+                fp.write(b'xxx')
+            contents = protocols.get('file:///{}'.format(path), mode='rb')
+            self.assertEqual(contents, b'xxx')
+
+    def test_file_ascii(self):
+        with TemporaryDirectory() as tempdir:
+            path = os.path.join(tempdir, 'my-file')
+            with open(path, 'w', encoding='us-ascii') as fp:
+                print('Some contents', end='', file=fp)
+            contents = protocols.get('file:///{}'.format(path),
+                                     encoding='us-ascii')
+            self.assertEqual(contents, 'Some contents')
+
+    def test_mailman_internal_uris(self):
+        # mailman://demo.txt
+        content = protocols.get('mailman:///demo.txt')
+        self.assertEqual(content, 'Test content')
+
+    def test_mailman_internal_uris_twice(self):
+        # mailman:///demo.txt
+        content = protocols.get('mailman:///demo.txt')
+        self.assertEqual(content, 'Test content')
+        content = protocols.get('mailman:///demo.txt')
+        self.assertEqual(content, 'Test content')
+
+    def test_mailman_uri_with_language(self):
+        content = protocols.get('mailman:///en/demo.txt')
+        self.assertEqual(content, 'Test content')
+
+    def test_mailman_uri_with_english_fallback(self):
+        content = protocols.get('mailman:///it/demo.txt')
+        self.assertEqual(content, 'Test content')
+
+    def test_mailman_uri_with_list_name(self):
+        content = protocols.get('mailman:///test@example.com/demo.txt')
+        self.assertEqual(content, 'Test content')
+
+    def test_mailman_full_uri(self):
+        content = protocols.get('mailman:///test@example.com/en/demo.txt')
+        self.assertEqual(content, 'Test content')
+
+    def test_mailman_full_uri_with_english_fallback(self):
+        content = protocols.get('mailman:///test@example.com/it/demo.txt')
+        self.assertEqual(content, 'Test content')
+
+    def test_uri_not_found(self):
+        with self.assertRaises(URLError) as cm:
+            protocols.get('mailman:///missing.txt')
+        self.assertEqual(cm.exception.reason, 'No such file')
+
+    def test_shorter_url_error(self):
+        with self.assertRaises(URLError) as cm:
+            protocols.get('mailman:///')
+        self.assertEqual(cm.exception.reason, 'No template specified')
+
+    def test_short_url_error(self):
+        with self.assertRaises(URLError) as cm:
+            protocols.get('mailman://')
+        self.assertEqual(cm.exception.reason, 'No template specified')
+
+    def test_bad_language(self):
+        with self.assertRaises(URLError) as cm:
+            protocols.get('mailman:///xx/demo.txt')
+        self.assertEqual(cm.exception.reason, 'Bad language or list name')
+
+    def test_bad_mailing_list(self):
+        with self.assertRaises(URLError) as cm:
+            protocols.get('mailman:///missing@example.com/demo.txt')
+        self.assertEqual(cm.exception.reason, 'Bad language or list name')
+
+    def test_missing_mailing_list(self):
+        with self.assertRaises(URLError) as cm:
+            protocols.get('mailman:///missing@example.com/it/demo.txt')
+        self.assertEqual(cm.exception.reason, 'Missing list')
+
+    def test_no_such_language(self):
+        with self.assertRaises(URLError) as cm:
+            protocols.get('mailman:///test@example.com/xx/demo.txt')
+        self.assertEqual(cm.exception.reason, 'No such language')
+
+    def test_too_many_path_components(self):
+        with self.assertRaises(URLError) as cm:
+            protocols.get('mailman:///missing@example.com/en/foo/demo.txt')
+        self.assertEqual(cm.exception.reason, 'No such file')
+
+    def test_non_ascii(self):
+        # mailman://demo.txt with non-ascii content.
+        test_text = b'\xe4\xb8\xad'
+        path = os.path.join(self.var_dir, 'templates', 'site', 'it')
+        os.makedirs(path)
+        with open(os.path.join(path, 'demo.txt'), 'wb') as fp:
+            fp.write(test_text)
+        content = protocols.get('mailman:///it/demo.txt')
+        self.assertIsInstance(content, str)
+        self.assertEqual(content, test_text.decode('utf-8'))
+
+    def test_bad_file_keyword(self):
+        self.assertRaises(ValueError, protocols.get, 'file:///etc/passwd',
+                          invalid_keyword='yes')
+
+    def test_bad_mailman_keyword(self):
+        self.assertRaises(ValueError, protocols.get, 'mailman:///demo.text',
+                          invalid_keyword='yes')
+
+    def test_bad_protocol(self):
+        self.assertRaises(URLError, protocols.get, 'unknown:///demo.text')
diff --git a/src/mailman/utilities/tests/test_templates.py b/src/mailman/utilities/tests/test_templates.py
index fcb0ecc1c..4e05f13c6 100644
--- a/src/mailman/utilities/tests/test_templates.py
+++ b/src/mailman/utilities/tests/test_templates.py
@@ -26,7 +26,7 @@ from mailman.app.lifecycle import create_list
 from mailman.config import config
 from mailman.interfaces.languages import ILanguageManager
 from mailman.testing.layers import ConfigLayer
-from mailman.utilities.i18n import TemplateNotFoundError, find, make, search
+from mailman.utilities.i18n import TemplateNotFoundError, find, search
 from pkg_resources import resource_filename
 from zope.component import getUtility
 
@@ -219,65 +219,3 @@ class TestFind(unittest.TestCase):
         with self.assertRaises(TemplateNotFoundError) as cm:
             find('missing.txt', self.mlist)
         self.assertEqual(cm.exception.template_file, 'missing.txt')
-
-
-class TestMake(unittest.TestCase):
-    """Test template interpolation."""
-
-    layer = ConfigLayer
-
-    def setUp(self):
-        self.var_dir = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, self.var_dir)
-        config.push('template config', """\
-        [paths.testing]
-        var_dir: {}
-        """.format(self.var_dir))
-        self.addCleanup(config.pop, 'template config')
-        # The following MUST happen AFTER the push() above since pushing a new
-        # config also clears out the language manager.
-        getUtility(ILanguageManager).add('xx', 'utf-8', 'Xlandia')
-        self.mlist = create_list('test@example.com')
-        self.mlist.preferred_language = 'xx'
-        # Populate the template directories with a few fake templates.
-        path = os.path.join(self.var_dir, 'templates', 'site', 'xx')
-        os.makedirs(path)
-        with open(os.path.join(path, 'nosub.txt'), 'w') as fp:
-            print("""\
-This is a global template.
-It has no substitutions.
-It will be wrapped.
-""", file=fp)
-        with open(os.path.join(path, 'subs.txt'), 'w') as fp:
-            print("""\
-This is a $kind template.
-It has $howmany substitutions.
-It will be wrapped.
-""", file=fp)
-        with open(os.path.join(path, 'nowrap.txt'), 'w') as fp:
-            print("""\
-This is a $kind template.
-It has $howmany substitutions.
-It will not be wrapped.
-""", file=fp)
-
-    def test_no_substitutions(self):
-        self.assertEqual(make('nosub.txt', self.mlist), """\
-This is a global template.  It has no substitutions.  It will be
-wrapped.""")
-
-    def test_substitutions(self):
-        self.assertEqual(make('subs.txt', self.mlist,
-                              kind='very nice',
-                              howmany='a few'), """\
-This is a very nice template.  It has a few substitutions.  It will be
-wrapped.""")
-
-    def test_substitutions_no_wrap(self):
-        self.assertEqual(make('nowrap.txt', self.mlist, wrap=False,
-                              kind='very nice',
-                              howmany='a few'), """\
-This is a very nice template.
-It has a few substitutions.
-It will not be wrapped.
-""")