14 files changed, 73 insertions, 74 deletions

diff --git a/.bzrignore b/.bzrignore
index 862dd0b10..b348eb336 100644
--- a/.bzrignore
+++ b/.bzrignore
@@ -21,3 +21,4 @@ distribute-*.egg
 distribute-*.tar.gz
 .coverage
 htmlcov
+.tox
diff --git a/src/mailman/app/docs/pipelines.rst b/src/mailman/app/docs/pipelines.rst
index 5aaf7cf62..c912cb763 100644
--- a/src/mailman/app/docs/pipelines.rst
+++ b/src/mailman/app/docs/pipelines.rst
@@ -43,14 +43,14 @@ etc.
     X-Mailman-Version: ...
     Precedence: list
     List-Id: <test.example.com>
+    Archived-At: http://lists.example.com/.../4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB
+    List-Archive: <http://lists.example.com/archives/test@example.com>
+    List-Help: <mailto:test-request@example.com?subject=help>
     List-Post: <mailto:test@example.com>
     List-Subscribe: <http://lists.example.com/listinfo/test@example.com>,
      <mailto:test-join@example.com>
-    Archived-At: http://lists.example.com/.../4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB
     List-Unsubscribe: <http://lists.example.com/listinfo/test@example.com>,
      <mailto:test-leave@example.com>
-    List-Archive: <http://lists.example.com/archives/test@example.com>
-    List-Help: <mailto:test-request@example.com?subject=help>
     <BLANKLINE>
     First post!
     <BLANKLINE>
diff --git a/src/mailman/commands/cli_conf.py b/src/mailman/commands/cli_conf.py
index 55f63c712..7fe9fce7d 100644
--- a/src/mailman/commands/cli_conf.py
+++ b/src/mailman/commands/cli_conf.py
@@ -65,19 +65,12 @@ class Conf:
             key-values pair from any section matching the given key will be
             displayed.
             """))
-        command_parser.add_argument(
-            '-t', '--sort',
-            default=False, action='store_true',
-            help=_('Sort the output by sections and keys.'))
 
     def _get_value(self, section, key):
         return getattr(getattr(config, section), key)
 
-    def _sections(self, sort_p):
-        sections = config.schema._section_schemas
-        if sort_p:
-            sections = sorted(sections)
-        return sections
+    def _sections(self):
+        return sorted(config.schema._section_schemas)
 
     def _print_full_syntax(self, section, key, value, output):
         print('[{}] {}: {}'.format(section, key, value), file=output)
@@ -88,10 +81,8 @@ class Conf:
     def _show_section_error(self, section):
         self.parser.error('No such section: {}'.format(section))
 
-    def _print_values_for_section(self, section, output, sort_p):
-        current_section = getattr(config, section)
-        if sort_p:
-            current_section = sorted(current_section)
+    def _print_values_for_section(self, section, output):
+        current_section = sorted(getattr(config, section))
         for key in current_section:
             self._print_full_syntax(section, key,
                                     self._get_value(section, key), output)
@@ -106,7 +97,6 @@ class Conf:
         # Process the command, ignoring the closing of the output file.
         section = args.section
         key = args.key
-        sort_p = args.sort
         # Case 1: Both section and key are given, so we can directly look up
         # the value.
         if section is not None and key is not None:
@@ -119,12 +109,12 @@ class Conf:
         # Case 2: Section is given, key is not given.
         elif section is not None and key is None:
             if self._section_exists(section):
-                self._print_values_for_section(section, output, sort_p)
+                self._print_values_for_section(section, output)
             else:
                 self._show_section_error(section)
         # Case 3: Section is not given, key is given.
         elif section is None and key is not None:
-            for current_section in self._sections(sort_p):
+            for current_section in self._sections():
                 # We have to ensure that the current section actually exists
                 # and that it contains the given key.
                 if (self._section_exists(current_section) and
@@ -137,13 +127,12 @@ class Conf:
         # Case 4: Neither section nor key are given, just display all the
         # sections and their corresponding key/value pairs.
         elif section is None and key is None:
-            for current_section in self._sections(sort_p):
+            for current_section in self._sections():
                 # However, we have to make sure that the current sections and
                 # key which are being looked up actually exist before trying
                 # to print them.
                 if self._section_exists(current_section):
-                    self._print_values_for_section(
-                        current_section, output, sort_p)
+                    self._print_values_for_section(current_section, output)
 
     def process(self, args):
         """See `ICLISubCommand`."""
diff --git a/src/mailman/commands/docs/conf.rst b/src/mailman/commands/docs/conf.rst
index 2f708edc5..0ff5064bb 100644
--- a/src/mailman/commands/docs/conf.rst
+++ b/src/mailman/commands/docs/conf.rst
@@ -14,7 +14,6 @@ a specific key-value pair, or several key-value pairs.
     ...     key = None
     ...     section = None
     ...     output = None
-    ...     sort = False
     >>> from mailman.commands.cli_conf import Conf
     >>> command = Conf()
 
@@ -22,9 +21,9 @@ To get a list of all key-value pairs of any section, you need to call the
 command without any options.
 
     >>> command.process(FakeArgs)
-    [logging.archiver] path: mailman.log
+    [antispam] header_checks:
     ...
-    [passwords] password_length: 8
+    [logging.bounce] level: info
     ...
     [mailman] site_owner: noreply@example.com
     ...
@@ -33,9 +32,9 @@ You can list all the key-value pairs of a specific section.
 
     >>> FakeArgs.section = 'shell'
     >>> command.process(FakeArgs)
-    [shell] use_ipython: no
     [shell] banner: Welcome to the GNU Mailman shell
     [shell] prompt: >>>
+    [shell] use_ipython: no
 
 You can also pass a key and display all key-value pairs matching the given
 key, along with the names of the corresponding sections.
@@ -44,20 +43,20 @@ key, along with the names of the corresponding sections.
     >>> FakeArgs.key = 'path'
     >>> command.process(FakeArgs)
     [logging.archiver] path: mailman.log
-    [logging.locks] path: mailman.log
-    [logging.mischief] path: mailman.log
+    [logging.bounce] path: bounce.log
     [logging.config] path: mailman.log
-    [logging.error] path: mailman.log
-    [logging.smtp] path: smtp.log
     [logging.database] path: mailman.log
+    [logging.debug] path: debug.log
+    [logging.error] path: mailman.log
+    [logging.fromusenet] path: mailman.log
     [logging.http] path: mailman.log
+    [logging.locks] path: mailman.log
+    [logging.mischief] path: mailman.log
     [logging.root] path: mailman.log
-    [logging.fromusenet] path: mailman.log
-    [logging.bounce] path: bounce.log
-    [logging.vette] path: mailman.log
     [logging.runner] path: mailman.log
+    [logging.smtp] path: smtp.log
     [logging.subscribe] path: mailman.log
-    [logging.debug] path: debug.log
+    [logging.vette] path: mailman.log
 
 If you specify both a section and a key, you will get the corresponding value.
 
@@ -66,16 +65,5 @@ If you specify both a section and a key, you will get the corresponding value.
     >>> command.process(FakeArgs)
     noreply@example.com
 
-You can also sort the output.  The output is first sorted by section, then by
-key.
-
-    >>> FakeArgs.key = None
-    >>> FakeArgs.section = 'shell'
-    >>> FakeArgs.sort = True
-    >>> command.process(FakeArgs)
-    [shell] banner: Welcome to the GNU Mailman shell
-    [shell] prompt: >>>
-    [shell] use_ipython: no
-
 
 .. _`Postfix command postconf(1)`: http://www.postfix.org/postconf.1.html
diff --git a/src/mailman/handlers/rfc_2369.py b/src/mailman/handlers/rfc_2369.py
index 3ac721d19..9078fd863 100644
--- a/src/mailman/handlers/rfc_2369.py
+++ b/src/mailman/handlers/rfc_2369.py
@@ -94,8 +94,9 @@ def process(mlist, msg, msgdata):
                 if permalink is not None:
                     headers['Archived-At'] = permalink
     # XXX RFC 2369 also defines a List-Owner header which we are not currently
-    # supporting, but should.
-    for h, v in headers.items():
+    # supporting, but should.  We need to add these headers in a predictable
+    # order.
+    for h, v in sorted(headers.items()):
         # First we delete any pre-existing headers because the RFC permits
         # only one copy of each, and we want to be sure it's ours.
         del msg[h]
diff --git a/src/mailman/mta/docs/bulk.rst b/src/mailman/mta/docs/bulk.rst
index f0c4e0132..f2a76229b 100644
--- a/src/mailman/mta/docs/bulk.rst
+++ b/src/mailman/mta/docs/bulk.rst
@@ -194,7 +194,7 @@ message sent, with all the recipients packed into the envelope recipients
     <BLANKLINE>
     This is a test.
 
-The ``X-RcptTo:`` header contains the set of recipients, in random order.
+The ``X-RcptTo:`` header contains the set of recipients, in sorted order.
 
     >>> len(messages[0]['x-rcptto'].split(','))
     100
diff --git a/src/mailman/rest/docs/helpers.rst b/src/mailman/rest/docs/helpers.rst
index 7405fc98f..0acbb5f45 100644
--- a/src/mailman/rest/docs/helpers.rst
+++ b/src/mailman/rest/docs/helpers.rst
@@ -45,7 +45,7 @@ gets modified to contain the etag under the ``http_etag`` key.
     >>> resource = dict(geddy='bass', alex='guitar', neil='drums')
     >>> json_data = etag(resource)
     >>> print(resource['http_etag'])
-    "43942176d8d5bb4414ccf35e2720ccd5251e66da"
+    "96e036d66248cab746b7d97047e08896fcfb2493"
 
 For convenience, the etag function also returns the JSON representation of the
 dictionary after tagging, since that's almost always what you want.
@@ -58,7 +58,7 @@ dictionary after tagging, since that's almost always what you want.
     >>> dump_msgdata(data)
     alex     : guitar
     geddy    : bass
-    http_etag: "43942176d8d5bb4414ccf35e2720ccd5251e66da"
+    http_etag: "96e036d66248cab746b7d97047e08896fcfb2493"
     neil     : drums
 
 
diff --git a/src/mailman/rest/docs/preferences.rst b/src/mailman/rest/docs/preferences.rst
index 8694364a4..f4839b1f4 100644
--- a/src/mailman/rest/docs/preferences.rst
+++ b/src/mailman/rest/docs/preferences.rst
@@ -162,7 +162,7 @@ deleted.
     >>> dump_json('http://localhost:9001/3.0/addresses/anne@example.com'
     ...           '/preferences')
     acknowledge_posts: True
-    http_etag: "5219245d1eea98bc107032013af20ef91bfb5c51"
+    http_etag: "1ff07b0367bede79ade27d217e12df3915aaee2b"
     preferred_language: ja
     self_link: http://localhost:9001/3.0/addresses/anne@example.com/preferences
 
diff --git a/src/mailman/rest/helpers.py b/src/mailman/rest/helpers.py
index 025ad1779..db3e43af0 100644
--- a/src/mailman/rest/helpers.py
+++ b/src/mailman/rest/helpers.py
@@ -38,6 +38,7 @@ from cStringIO import StringIO
 from datetime import datetime, timedelta
 from enum import Enum
 from lazr.config import as_boolean
+from pprint import pformat
 from restish import http
 from restish.http import Response
 from restish.resource import MethodDecorator
@@ -89,11 +90,12 @@ class ExtendedEncoder(json.JSONEncoder):
 def etag(resource):
     """Calculate the etag and return a JSON representation.
 
-    The input is a dictionary representing the resource.  This dictionary must
-    not contain an `http_etag` key.  This function calculates the etag by
-    using the sha1 hexdigest of the repr of the dictionary.  It then inserts
-    this value under the `http_etag` key, and returns the JSON representation
-    of the modified dictionary.
+    The input is a dictionary representing the resource.  This
+    dictionary must not contain an `http_etag` key.  This function
+    calculates the etag by using the sha1 hexdigest of the
+    pretty-printed (and thus key-sorted and predictable) representation
+    of the dictionary.  It then inserts this value under the `http_etag`
+    key, and returns the JSON representation of the modified dictionary.
 
     :param resource: The original resource representation.
     :type resource: dictionary
@@ -101,8 +103,10 @@ def etag(resource):
     :rtype string
     """
     assert 'http_etag' not in resource, 'Resource already etagged'
-    etag = hashlib.sha1(repr(resource)).hexdigest()
-
+    # Calculate the tag from a predictable (i.e. sorted) representation of the
+    # dictionary.  The actual details aren't so important.  pformat() is
+    # guaranteed to sort the keys.
+    etag = hashlib.sha1(pformat(resource)).hexdigest()
     resource['http_etag'] = '"{0}"'.format(etag)
     return json.dumps(resource, cls=ExtendedEncoder)
 
diff --git a/src/mailman/runners/docs/outgoing.rst b/src/mailman/runners/docs/outgoing.rst
index a3220e423..d4a20d497 100644
--- a/src/mailman/runners/docs/outgoing.rst
+++ b/src/mailman/runners/docs/outgoing.rst
@@ -81,7 +81,7 @@ Every recipient got the same copy of the message.
     Message-ID: <first>
     X-Peer: ...
     X-MailFrom: test-bounces@example.com
-    X-RcptTo: cperson@example.com, bperson@example.com, aperson@example.com
+    X-RcptTo: aperson@example.com, bperson@example.com, cperson@example.com
     <BLANKLINE>
     First post!
 
@@ -222,7 +222,7 @@ VERP'd.
     1
 
     >>> show_headers(messages)
-    cperson@example.com, bperson@example.com, aperson@example.com
+    aperson@example.com, bperson@example.com, cperson@example.com
     test-bounces@example.com
 
     # Perform post-delivery bookkeeping.
@@ -242,7 +242,7 @@ The second message sent to the list is also not VERP'd.
     1
 
     >>> show_headers(messages)
-    cperson@example.com, bperson@example.com, aperson@example.com
+    aperson@example.com, bperson@example.com, cperson@example.com
     test-bounces@example.com
 
     # Perform post-delivery bookkeeping.
@@ -281,7 +281,7 @@ The next one is back to bulk delivery.
     1
 
     >>> show_headers(messages)
-    cperson@example.com, bperson@example.com, aperson@example.com
+    aperson@example.com, bperson@example.com, cperson@example.com
     test-bounces@example.com
 
     >>> config.pop('verp occasionally')
@@ -394,7 +394,7 @@ Neither the first message...
     1
 
     >>> show_headers(messages)
-    cperson@example.com, bperson@example.com, aperson@example.com
+    aperson@example.com, bperson@example.com, cperson@example.com
     test-bounces@example.com
 
 ...nor the second message is VERP'd.
@@ -409,5 +409,5 @@ Neither the first message...
     1
 
     >>> show_headers(messages)
-    cperson@example.com, bperson@example.com, aperson@example.com
+    aperson@example.com, bperson@example.com, cperson@example.com
     test-bounces@example.com
diff --git a/src/mailman/testing/mta.py b/src/mailman/testing/mta.py
index 7481e0093..875647485 100644
--- a/src/mailman/testing/mta.py
+++ b/src/mailman/testing/mta.py
@@ -161,6 +161,11 @@ class ConnectionCountingServer(QueueServer):
         log.info('[ConnectionCountingServer] accepted: %s', address)
         StatisticsChannel(self, connection, address)
 
+    def process_message(self, peer, mailfrom, rcpttos, data):
+        # Provide a guaranteed order to recpttos.
+        QueueServer.process_message(
+            self, peer, mailfrom, sorted(rcpttos), data)
+
     def reset(self):
         """See `lazr.smtp.server.Server`."""
         QueueServer.reset(self)
diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py
index baaa0e020..0523bb06f 100644
--- a/src/mailman/utilities/importer.py
+++ b/src/mailman/utilities/importer.py
@@ -308,13 +308,15 @@ def import_config_pck(mlist, config_dict):
         'digest_header': 'digest_header_uri',
         'digest_footer': 'digest_footer_uri',
         }
-    # The best we can do is convert only the most common ones.
-    convert_placeholders = {
-        '%(real_name)s': '$display_name',
-        '%(real_name)s@%(host_name)s': '$fqdn_listname',
-        '%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s':
-            '$listinfo_uri',
-    }
+    # The best we can do is convert only the most common ones.  These are
+    # order dependent; the longer substitution with the common prefix must
+    # show up earlier.
+    convert_placeholders = [
+        ('%(real_name)s@%(host_name)s', '$fqdn_listname'),
+        ('%(real_name)s', '$display_name'),
+        ('%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s',
+         '$listinfo_uri'),
+        ]
     # Collect defaults.
     defaults = {}
     for oldvar, newvar in convert_to_uri.items():
@@ -339,7 +341,7 @@ def import_config_pck(mlist, config_dict):
             continue
         text = config_dict[oldvar]
         text = text.decode('utf-8', 'replace')
-        for oldph, newph in convert_placeholders.items():
+        for oldph, newph in convert_placeholders:
             text = text.replace(oldph, newph)
         default_value, default_text  = defaults.get(newvar, (None, None))
         if not text and not (default_value or default_text):
diff --git a/src/mailman/utilities/tests/test_import.py b/src/mailman/utilities/tests/test_import.py
index e6eb9344c..be09a3256 100644
--- a/src/mailman/utilities/tests/test_import.py
+++ b/src/mailman/utilities/tests/test_import.py
@@ -501,8 +501,9 @@ class TestConvertToURI(unittest.TestCase):
                 ))
             loader = getUtility(ITemplateLoader)
             text = loader.get(template_uri)
-            self.assertEqual(text, expected_text,
-                    'Old variables were not converted for %s' % newvar)
+            self.assertEqual(
+                text, expected_text,
+                'Old variables were not converted for %s' % newvar)
 
     def test_keep_default(self):
         # If the value was not changed from MM2.1's default, don't import it.
diff --git a/tox.ini b/tox.ini
new file mode 100644
index 000000000..3a9eb91d4
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,8 @@
+[tox]
+envlist = py27
+recreate = True
+
+[testenv]
+commands = python -m nose2 -v
+#sitepackages = True
+usedevelop = True