Several important cleanups.

* Turn on absolute_import and unicode_literals everywhere, and deal with the
  aftermath.
* Use 'except X as Y' everywhere.
* Make the module prologues much more consistent.
* Use '{}'.format() consistently, except for logger interface.
* Because of the problems with calling ** args with unicode keywords, hide
  calls to Template.substitute() behind an API.

5 files changed, 200 insertions, 160 deletions

diff --git a/mailman/docs/archivers.txt b/mailman/docs/archivers.txt
index 489e3f15b..ef36a25ac 100644
--- a/mailman/docs/archivers.txt
+++ b/mailman/docs/archivers.txt
@@ -77,10 +77,10 @@ archiver; by enabling it messages to public lists will get sent there
 automatically.
 
     >>> archiver = archivers['mail-archive']
-    >>> archiver.list_url(mlist)
-    'http://go.mail-archive.dev/test%40example.com'
-    >>> archiver.permalink(mlist, msg)
-    'http://go.mail-archive.dev/ZaXPPxRMM9_hFZL4vTRlQlBx8pc='
+    >>> print archiver.list_url(mlist)
+    http://go.mail-archive.dev/test%40example.com
+    >>> print archiver.permalink(mlist, msg)
+    http://go.mail-archive.dev/ZaXPPxRMM9_hFZL4vTRlQlBx8pc=
 
 To archive the message, the archiver actually mails the message to a special
 address at the Mail-Archive.
@@ -137,23 +137,23 @@ Additionally, this archiver can handle malformed Message-IDs.
     >>> mlist.archive_private = False
     >>> del msg['message-id']
     >>> msg['Message-ID'] = '12345>'
-    >>> archiver.permalink(mlist, msg)
-    'http://go.mail-archive.dev/bXvG32YzcDEIVDaDLaUSVQekfo8='
+    >>> print archiver.permalink(mlist, msg)
+    http://go.mail-archive.dev/bXvG32YzcDEIVDaDLaUSVQekfo8=
 
     >>> del msg['message-id']
     >>> msg['Message-ID'] = '<12345'
-    >>> archiver.permalink(mlist, msg)
-    'http://go.mail-archive.dev/9rockPrT1Mm-jOsLWS6_hseR_OY='
+    >>> print archiver.permalink(mlist, msg)
+    http://go.mail-archive.dev/9rockPrT1Mm-jOsLWS6_hseR_OY=
 
     >>> del msg['message-id']
     >>> msg['Message-ID'] = '12345'
-    >>> archiver.permalink(mlist, msg)
-    'http://go.mail-archive.dev/ZaXPPxRMM9_hFZL4vTRlQlBx8pc='
+    >>> print archiver.permalink(mlist, msg)
+    http://go.mail-archive.dev/ZaXPPxRMM9_hFZL4vTRlQlBx8pc=
 
     >>> del msg['message-id']
     >>> msg['Message-ID'] = '    12345    '
-    >>> archiver.permalink(mlist, msg)
-    'http://go.mail-archive.dev/ZaXPPxRMM9_hFZL4vTRlQlBx8pc='
+    >>> print archiver.permalink(mlist, msg)
+    http://go.mail-archive.dev/ZaXPPxRMM9_hFZL4vTRlQlBx8pc=
 
 
 MHonArc
@@ -162,8 +162,8 @@ MHonArc
 The MHonArc archiver <http://www.mhonarc.org> is also available.
 
     >>> archiver = archivers['mhonarc']
-    >>> archiver.name
-    'mhonarc'
+    >>> print archiver.name
+    mhonarc
 
 Messages sent to a local MHonArc instance are added to its archive via a
 subprocess call.
diff --git a/mailman/docs/chains.txt b/mailman/docs/chains.txt
index e9c5d74e4..b6e75e6e1 100644
--- a/mailman/docs/chains.txt
+++ b/mailman/docs/chains.txt
@@ -19,10 +19,10 @@ The Discard chain simply throws the message away.
     >>> chain = config.chains['discard']
     >>> verifyObject(IChain, chain)
     True
-    >>> chain.name
-    'discard'
-    >>> chain.description
-    u'Discard a message and stop processing.'
+    >>> print chain.name
+    discard
+    >>> print chain.description
+    Discard a message and stop processing.
 
     >>> from mailman.app.lifecycle import create_list
     >>> mlist = create_list(u'_xtest@example.com')
@@ -58,10 +58,10 @@ this action.
     >>> chain = config.chains['reject']
     >>> verifyObject(IChain, chain)
     True
-    >>> chain.name
-    'reject'
-    >>> chain.description
-    u'Reject/bounce a message and stop processing.'
+    >>> print chain.name
+    reject
+    >>> print chain.description
+    Reject/bounce a message and stop processing.
     >>> file_pos = fp.tell()
     >>> process(mlist, msg, {}, 'reject')
     >>> fp.seek(file_pos)
@@ -104,10 +104,10 @@ sender and the list moderators.
     >>> chain = config.chains['hold']
     >>> verifyObject(IChain, chain)
     True
-    >>> chain.name
-    'hold'
-    >>> chain.description
-    u'Hold a message and stop processing.'
+    >>> print chain.name
+    hold
+    >>> print chain.description
+    Hold a message and stop processing.
 
     >>> file_pos = fp.tell()
     >>> process(mlist, msg, {}, 'hold')
@@ -231,7 +231,7 @@ first item is a type code and the second item is a message id.
     [(u'id', ...), (u'type', u'held message')]
 
 The message itself is held in the message store.
- 
+
     >>> rkey, rdata = config.db.requests.get_list_requests(mlist).get_request(
     ...     data['id'])
     >>> msg = config.db.message_store.get_message_by_id(
@@ -256,10 +256,10 @@ processed and sent on to the list membership.
     >>> chain = config.chains['accept']
     >>> verifyObject(IChain, chain)
     True
-    >>> chain.name
-    'accept'
-    >>> chain.description
-    u'Accept a message.'
+    >>> print chain.name
+    accept
+    >>> print chain.description
+    Accept a message.
     >>> file_pos = fp.tell()
     >>> process(mlist, msg, {}, 'accept')
     >>> fp.seek(file_pos)
@@ -297,10 +297,10 @@ the Hold handler from previous versions of Mailman.
     >>> chain = config.chains['built-in']
     >>> verifyObject(IChain, chain)
     True
-    >>> chain.name
-    'built-in'
-    >>> chain.description
-    u'The built-in moderation chain.'
+    >>> print chain.name
+    built-in
+    >>> print chain.description
+    The built-in moderation chain.
 
 The previously created message is innocuous enough that it should pass through
 all default rules.  This message will end up in the pipeline queue.
@@ -320,7 +320,7 @@ all default rules.  This message will end up in the pipeline queue.
     X-Message-ID-Hash: RXJU4JL6N2OUN3OYMXXPPSCR7P7JE2BW
     X-Mailman-Rule-Misses: approved; emergency; loop; administrivia;
         implicit-dest;
-    	max-recipients; max-size; news-moderation; no-subject;
+        max-recipients; max-size; news-moderation; no-subject;
         suspicious-header
     <BLANKLINE>
     An important message.
@@ -331,7 +331,15 @@ hit and all rules that have missed.
 
     >>> sorted(qdata['rule_hits'])
     []
-    >>> sorted(qdata['rule_misses'])
-    ['administrivia', 'approved', 'emergency', 'implicit-dest', 'loop',
-     'max-recipients', 'max-size', 'news-moderation', 'no-subject',
-     'suspicious-header']
+    >>> for rule_name in sorted(qdata['rule_misses']):
+    ...     print rule_name
+    administrivia
+    approved
+    emergency
+    implicit-dest
+    loop
+    max-recipients
+    max-size
+    news-moderation
+    no-subject
+    suspicious-header
diff --git a/mailman/docs/pipelines.txt b/mailman/docs/pipelines.txt
index 205d210b1..0e6dad8e8 100644
--- a/mailman/docs/pipelines.txt
+++ b/mailman/docs/pipelines.txt
@@ -62,11 +62,11 @@ etc.
 And the message metadata has information about recipients and other stuff.
 However there are currently no recipients for this message.
 
-    >>> sorted(msgdata.items())
-    [('original_sender', u'aperson@example.com'),
-     ('origsubj', u'My first post'),
-     ('recips', set([])),
-     ('stripped_subject', <email.header.Header instance at ...>)]
+    >>> dump_msgdata(msgdata)
+    original_sender : aperson@example.com
+    origsubj        : My first post
+    recips          : set([])
+    stripped_subject: My first post
 
 And the message is now sitting in various other processing queues.
 
@@ -98,12 +98,13 @@ And the message is now sitting in various other processing queues.
     <BLANKLINE>
     First post!
     <BLANKLINE>
-    >>> print sorted(messages[0].msgdata.items())
-    [('_parsemsg', False), ('original_sender', u'aperson@example.com'),
-     ('origsubj', u'My first post'),
-     ('received_time', ...), ('recips', set([])),
-     ('stripped_subject', <email.header.Header instance at ...>),
-     ('version', 3)]
+    >>> dump_msgdata(messages[0].msgdata)
+    _parsemsg       : False
+    original_sender : aperson@example.com
+    origsubj        : My first post
+    recips          : set([])
+    stripped_subject: My first post
+    version         : 3
 
 This mailing list is not linked to an NNTP newsgroup, so there's nothing in
 the outgoing nntp queue.
@@ -141,13 +142,14 @@ This is the message that will actually get delivered to end recipients.
     <BLANKLINE>
     First post!
     <BLANKLINE>
-    >>> print sorted(messages[0].msgdata.items())
-    [('_parsemsg', False), ('listname', u'xtest@example.com'),
-     ('original_sender', u'aperson@example.com'),
-     ('origsubj', u'My first post'), ('received_time', ...),
-     ('recips', set([])),
-     ('stripped_subject', <email.header.Header instance at ...>),
-     ('version', 3)]
+    >>> dump_msgdata(messages[0].msgdata)
+    _parsemsg       : False
+    listname        : xtest@example.com
+    original_sender : aperson@example.com
+    origsubj        : My first post
+    recips          : set([])
+    stripped_subject: My first post
+    version         : 3
 
 There's now one message in the digest mailbox, getting ready to be sent.
 
diff --git a/mailman/docs/registration.txt b/mailman/docs/registration.txt
index 2e04f7ddf..d243188bc 100644
--- a/mailman/docs/registration.txt
+++ b/mailman/docs/registration.txt
@@ -158,13 +158,12 @@ message is sent to the user in order to verify the registered address.
     <BLANKLINE>
         postmaster@mail.example.com
     <BLANKLINE>
-    >>> sorted(qdata.items())
-    [('_parsemsg', False),
-     ('nodecorate', True),
-     ('received_time', ...),
-     ('recips', [u'aperson@example.com']),
-     ('reduced_list_headers', True),
-     ('version', 3)]
+    >>> dump_msgdata(qdata)
+    _parsemsg           : False
+    nodecorate          : True
+    recips              : [u'aperson@example.com']
+    reduced_list_headers: True
+    version             : 3
 
 The confirmation token shows up in several places, each of which provides an
 easy way for the user to complete the confirmation.  The token will always
diff --git a/mailman/docs/requests.txt b/mailman/docs/requests.txt
index aa6f031bd..87b835fb8 100644
--- a/mailman/docs/requests.txt
+++ b/mailman/docs/requests.txt
@@ -10,9 +10,10 @@ Here is a helper function for printing out held requests.
     >>> def show_holds(requests):
     ...     for request in requests.held_requests:
     ...         key, data = requests.get_request(request.id)
+    ...         print request.id, str(request.request_type), key
     ...         if data is not None:
-    ...             data = sorted(data.items())
-    ...         print request.id, str(request.request_type), key, data
+    ...             for key in sorted(data):
+    ...                 print '    {0}: {1}'.format(key, data[key])
 
 And another helper for displaying messages in the virgin queue.
 
@@ -80,10 +81,10 @@ And of course, now we can see that there are four requests being held.
     >>> requests.count_of(RequestType.unsubscription)
     1
     >>> show_holds(requests)
-    1 RequestType.held_message hold_1 None
-    2 RequestType.subscription hold_2 None
-    3 RequestType.unsubscription hold_3 None
-    4 RequestType.held_message hold_4 None
+    1 RequestType.held_message hold_1
+    2 RequestType.subscription hold_2
+    3 RequestType.unsubscription hold_3
+    4 RequestType.held_message hold_4
 
 If we try to hold a request with a bogus type, we get an exception.
 
@@ -101,11 +102,13 @@ We can hold requests with additional data.
     >>> requests.count
     5
     >>> show_holds(requests)
-    1 RequestType.held_message hold_1 None
-    2 RequestType.subscription hold_2 None
-    3 RequestType.unsubscription hold_3 None
-    4 RequestType.held_message hold_4 None
-    5 RequestType.held_message hold_5 [(u'bar', u'no'), (u'foo', u'yes')]
+    1 RequestType.held_message hold_1
+    2 RequestType.subscription hold_2
+    3 RequestType.unsubscription hold_3
+    4 RequestType.held_message hold_4
+    5 RequestType.held_message hold_5
+        bar: no
+        foo: yes
 
 
 Getting requests
@@ -116,8 +119,8 @@ of the request data we want.  This returns a 2-tuple of the key and data we
 originally held.
 
     >>> key, data = requests.get_request(2)
-    >>> key
-    u'hold_2'
+    >>> print key
+    hold_2
 
 Because we did not store additional data with request 2, it comes back as None
 now.
@@ -128,10 +131,11 @@ now.
 However, if we ask for a request that had data, we'd get it back now.
 
     >>> key, data = requests.get_request(5)
-    >>> key
-    u'hold_5'
-    >>> sorted(data.items())
-    [(u'bar', u'no'), (u'foo', u'yes')]
+    >>> print key
+    hold_5
+    >>> dump_msgdata(data)
+    bar: no
+    foo: yes
 
 If we ask for a request that is not in the database, we get None back.
 
@@ -150,12 +154,15 @@ over by type.
     >>> for request in requests.of_type(RequestType.held_message):
     ...     assert request.request_type is RequestType.held_message
     ...     key, data = requests.get_request(request.id)
+    ...     print request.id, key
     ...     if data is not None:
-    ...         data = sorted(data.items())
-    ...     print request.id, key, data
-    1 hold_1 None
-    4 hold_4 None
-    5 hold_5 [(u'bar', u'no'), (u'foo', u'yes')]
+    ...         for key in sorted(data):
+    ...             print '    {0}: {1}'.format(key, data[key])
+    1 hold_1
+    4 hold_4
+    5 hold_5
+    bar: no
+    foo: yes
 
 
 Deleting requests
@@ -168,10 +175,12 @@ database.
     >>> requests.count
     4
     >>> show_holds(requests)
-    1 RequestType.held_message hold_1 None
-    3 RequestType.unsubscription hold_3 None
-    4 RequestType.held_message hold_4 None
-    5 RequestType.held_message hold_5 [(u'bar', u'no'), (u'foo', u'yes')]
+    1 RequestType.held_message hold_1
+    3 RequestType.unsubscription hold_3
+    4 RequestType.held_message hold_4
+    5 RequestType.held_message hold_5
+        bar: no
+        foo: yes
     >>> print requests.get_request(2)
     None
 
@@ -286,14 +295,13 @@ The message can be rejected, meaning it is bounced back to the sender.
     <BLANKLINE>
         alist-owner@example.com
     <BLANKLINE>
-    >>> sorted(qdata.items())
-    [('_parsemsg', False),
-     ('listname', u'alist@example.com'),
-     ('nodecorate', True),
-     ('received_time', ...),
-     ('recips', [u'aperson@example.org']),
-     ('reduced_list_headers', True),
-     ('version', 3)]
+    >>> dump_msgdata(qdata)
+    _parsemsg           : False
+    listname            : alist@example.com
+    nodecorate          : True
+    recips              : [u'aperson@example.org']
+    reduced_list_headers: True
+    version             : 3
 
 Or the message can be approved.  This actually places the message back into
 the incoming queue for further processing, however the message metadata
@@ -313,11 +321,12 @@ indicates that the message has been approved.
     <BLANKLINE>
     Here's something important about our mailing list.
     <BLANKLINE>
-    >>> sorted(qdata.items())
-    [('_parsemsg', False),
-     (u'approved', True), ('moderator_approved', True),
-     (u'received_time', 123.45), (u'sender', u'aperson@example.com'),
-     ('version', 3)]
+    >>> dump_msgdata(qdata)
+    _parsemsg         : False
+    approved          : True
+    moderator_approved: True
+    sender            : aperson@example.com
+    version           : 3
 
 In addition to any of the above dispositions, the message can also be
 preserved for further study.  Ordinarily the message is removed from the
@@ -386,11 +395,13 @@ moderators.
     <BLANKLINE>
     Here's something important about our mailing list.
     <BLANKLINE>
-    >>> sorted(qdata.items())
-    [('_parsemsg', False), ('listname', u'alist@example.com'),
-    ('nodecorate', True), ('received_time', ...),
-    ('recips', [u'zperson@example.com']),
-    ('reduced_list_headers', True), ('version', 3)]
+    >>> dump_msgdata(qdata)
+    _parsemsg           : False
+    listname            : alist@example.com
+    nodecorate          : True
+    recips              : [u'zperson@example.com']
+    reduced_list_headers: True
+    version             : 3
 
 
 Holding subscription requests
@@ -454,13 +465,14 @@ queue when the message is held.
     <BLANKLINE>
     to process the request.
     <BLANKLINE>
-    >>> sorted(qdata.items())
-    [('_parsemsg', False),
-    ('listname', u'alist@example.com'),
-    ('nodecorate', True),
-    ('received_time', ...),
-    ('recips', [u'alist-owner@example.com']),
-    ('reduced_list_headers', True), ('tomoderators', True), ('version', 3)]
+    >>> dump_msgdata(qdata)
+    _parsemsg           : False
+    listname            : alist@example.com
+    nodecorate          : True
+    recips              : [u'alist-owner@example.com']
+    reduced_list_headers: True
+    tomoderators        : True
+    version             : 3
 
 Once held, the moderator can select one of several dispositions.  The most
 trivial is to simply defer a decision for now.
@@ -508,12 +520,13 @@ subscriber.
     <BLANKLINE>
         alist-owner@example.com
     <BLANKLINE>
-    >>> sorted(qdata.items())
-    [('_parsemsg', False), ('listname', u'alist@example.com'),
-    ('nodecorate', True),
-    ('received_time', ...),
-    ('recips', [u'cperson@example.org']),
-    ('reduced_list_headers', True), ('version', 3)]
+    >>> dump_msgdata(qdata)
+    _parsemsg           : False
+    listname            : alist@example.com
+    nodecorate          : True
+    recips              : [u'cperson@example.org']
+    reduced_list_headers: True
+    version             : 3
 
 The subscription can also be accepted.  This subscribes the address to the
 mailing list.
@@ -551,11 +564,14 @@ subscription and the fact that they may need to approve it.
     <BLANKLINE>
     to process the request.
     <BLANKLINE>
-    >>> sorted(qdata.items())
-    [('_parsemsg', False), ('listname', u'alist@example.com'),
-     ('nodecorate', True), ('received_time', ...),
-     ('recips', [u'alist-owner@example.com']),
-     ('reduced_list_headers', True), ('tomoderators', True), ('version', 3)]
+    >>> dump_msgdata(qdata)
+    _parsemsg           : False
+    listname            : alist@example.com
+    nodecorate          : True
+    recips              : [u'alist-owner@example.com']
+    reduced_list_headers: True
+    tomoderators        : True
+    version             : 3
 
 Accept the subscription request.
 
@@ -621,11 +637,14 @@ The welcome message is sent to the person who just subscribed.
     this email is not included here.  There is also a button on your
     options page that will send your current password to you.
     <BLANKLINE>
-    >>> sorted(welcome_qdata.items())
-    [('_parsemsg', False), ('listname', u'alist@example.com'),
-     ('nodecorate', True), ('received_time', ...),
-     ('recips', [u'fperson@example.org']),
-     ('reduced_list_headers', True), ('verp', False), ('version', 3)]
+    >>> dump_msgdata(welcome_qdata)
+    _parsemsg           : False
+    listname            : alist@example.com
+    nodecorate          : True
+    recips              : [u'fperson@example.org']
+    reduced_list_headers: True
+    verp                : False
+    version             : 3
 
 The admin message is sent to the moderators.
 
@@ -643,11 +662,14 @@ The admin message is sent to the moderators.
     Frank Person <fperson@example.org> has been successfully subscribed to
     A Test List.
     <BLANKLINE>
-    >>> sorted(admin_qdata.items())
-    [('_parsemsg', False), ('envsender', 'changeme@example.com'),
-     ('listname', u'alist@example.com'),
-     ('nodecorate', True), ('received_time', ...),
-     ('recips', []), ('reduced_list_headers', True), ('version', 3)]
+    >>> dump_msgdata(admin_qdata)
+    _parsemsg           : False
+    envsender           : changeme@example.com
+    listname            : alist@example.com
+    nodecorate          : True
+    recips              : []
+    reduced_list_headers: True
+    version             : 3
 
 Frank Person is now a member of the mailing list.
 
@@ -714,12 +736,14 @@ unsubscription holds can send the list's moderators an immediate notification.
     <BLANKLINE>
     to process the request.
     <BLANKLINE>
-    >>> sorted(qdata.items())
-    [('_parsemsg', False),
-     ('listname', u'alist@example.com'), ('nodecorate', True),
-     ('received_time', ...),
-     ('recips', [u'alist-owner@example.com']),
-     ('reduced_list_headers', True), ('tomoderators', True), ('version', 3)]
+    >>> dump_msgdata(qdata)
+    _parsemsg           : False
+    listname            : alist@example.com
+    nodecorate          : True
+    recips              : [u'alist-owner@example.com']
+    reduced_list_headers: True
+    tomoderators        : True
+    version             : 3
 
 There are now two addresses with held unsubscription requests.  As above, one
 of the actions we can take is to defer to the decision.
@@ -770,12 +794,14 @@ and the person remains a member of the mailing list.
     <BLANKLINE>
         alist-owner@example.com
     <BLANKLINE>
-    >>> sorted(qdata.items())
-    [('_parsemsg', False),
-     ('listname', u'alist@example.com'),
-     ('nodecorate', True), ('received_time', ...),
-     ('recips', [u'hperson@example.com']),
-     ('reduced_list_headers', True), ('version', 3)]
+    >>> dump_msgdata(qdata)
+    _parsemsg           : False
+    listname            : alist@example.com
+    nodecorate          : True
+    recips              : [u'hperson@example.com']
+    reduced_list_headers: True
+    version             : 3
+
     >>> mlist.members.get_member(u'hperson@example.com')
     <Member: hperson@example.com on alist@example.com as MemberRole.member>
 
@@ -823,12 +849,14 @@ The goodbye message...
     <BLANKLINE>
     So long!
     <BLANKLINE>
-    >>> sorted(goodbye_qdata.items())
-    [('_parsemsg', False),
-     ('listname', u'alist@example.com'),
-     ('nodecorate', True), ('received_time', ...),
-     ('recips', [u'gperson@example.com']),
-     ('reduced_list_headers', True), ('verp', False), ('version', 3)]
+    >>> dump_msgdata(goodbye_qdata)
+    _parsemsg           : False
+    listname            : alist@example.com
+    nodecorate          : True
+    recips              : [u'gperson@example.com']
+    reduced_list_headers: True
+    verp                : False
+    version             : 3
 
 ...and the admin message.
 
@@ -845,8 +873,11 @@ The goodbye message...
     <BLANKLINE>
     gperson@example.com has been removed from A Test List.
     <BLANKLINE>
-    >>> sorted(admin_qdata.items())
-    [('_parsemsg', False), ('envsender', 'changeme@example.com'),
-     ('listname', u'alist@example.com'),
-     ('nodecorate', True), ('received_time', ...),
-     ('recips', []), ('reduced_list_headers', True), ('version', 3)]
+    >>> dump_msgdata(admin_qdata)
+    _parsemsg           : False
+    envsender           : changeme@example.com
+    listname            : alist@example.com
+    nodecorate          : True
+    recips              : []
+    reduced_list_headers: True
+    version             : 3