From 44e43727be13e3554342c2b5b75b7dc42abdb18c Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Sun, 30 Nov 2014 21:51:03 -0500 Subject: Checkpointing. By using `six` I think I have most of the imports squared away. There's probably still uses of `unicode` built-ins that need fixing. The idea is to first get the test suite running (which it doesn't yet), and then to fix tests. There's a bug in lazr.config which requires us to patch it for now. --- setup.py | 1 + src/mailman/app/moderator.py | 8 ++--- src/mailman/app/notifications.py | 5 ++- src/mailman/app/templates.py | 36 +++++++++---------- src/mailman/app/tests/test_templates.py | 20 +++++------ src/mailman/archiving/prototype.py | 9 +++-- src/mailman/archiving/tests/test_prototype.py | 8 ++--- src/mailman/bin/export.py | 2 +- src/mailman/commands/cli_import.py | 7 ++-- src/mailman/commands/cli_inject.py | 9 +++-- src/mailman/commands/cli_qfile.py | 8 ++--- src/mailman/commands/docs/echo.rst | 2 +- src/mailman/commands/docs/help.rst | 8 ++--- src/mailman/commands/docs/membership.rst | 18 +++++----- src/mailman/commands/tests/test_conf.py | 2 +- src/mailman/commands/tests/test_help.py | 8 ++--- src/mailman/config/config.py | 49 ++++++++++++++------------ src/mailman/config/tests/test_configuration.py | 5 +-- src/mailman/core/initialize.py | 6 ++-- src/mailman/core/runner.py | 15 ++++---- src/mailman/core/switchboard.py | 21 +++++------ src/mailman/database/sqlite.py | 2 +- src/mailman/email/message.py | 10 +++--- src/mailman/handlers/cook_headers.py | 4 +-- src/mailman/handlers/decorate.py | 7 ++-- src/mailman/handlers/docs/subject-munging.rst | 16 ++++----- src/mailman/handlers/tests/test_recipients.py | 5 +-- src/mailman/model/domain.py | 9 +++-- src/mailman/model/mailinglist.py | 19 +++++----- src/mailman/model/messagestore.py | 5 ++- src/mailman/model/requests.py | 17 ++++----- src/mailman/options.py | 2 +- src/mailman/rest/addresses.py | 6 ++-- src/mailman/rest/configuration.py | 26 +++++++------- src/mailman/rest/docs/helpers.rst | 5 +-- src/mailman/rest/domains.py | 10 +++--- src/mailman/rest/lists.py | 6 ++-- src/mailman/rest/members.py | 12 ++++--- src/mailman/rest/tests/test_addresses.py | 5 ++- src/mailman/rest/tests/test_domains.py | 5 ++- src/mailman/rest/tests/test_lists.py | 5 ++- src/mailman/rest/tests/test_membership.py | 5 ++- src/mailman/rest/tests/test_moderation.py | 3 +- src/mailman/rest/tests/test_preferences.py | 2 +- src/mailman/rest/tests/test_root.py | 2 +- src/mailman/rest/tests/test_users.py | 5 ++- src/mailman/rest/users.py | 12 ++++--- src/mailman/rest/validator.py | 2 +- src/mailman/runners/command.py | 18 +++++----- src/mailman/runners/digest.py | 6 ++-- src/mailman/runners/nntp.py | 4 +-- src/mailman/runners/tests/test_nntp.py | 6 ++-- src/mailman/testing/helpers.py | 11 +++--- src/mailman/testing/layers.py | 5 +-- src/mailman/testing/mta.py | 3 +- src/mailman/utilities/email.py | 2 +- src/mailman/utilities/i18n.py | 4 ++- src/mailman/utilities/importer.py | 2 +- src/mailman/utilities/tests/test_import.py | 15 ++++---- 59 files changed, 265 insertions(+), 265 deletions(-) diff --git a/setup.py b/setup.py index 78787ed06..c516d744d 100644 --- a/setup.py +++ b/setup.py @@ -105,6 +105,7 @@ case second `m'. Any other spelling is incorrect.""", 'mock', 'nose2', 'passlib', + 'six', 'sqlalchemy', 'zope.component', 'zope.configuration', diff --git a/src/mailman/app/moderator.py b/src/mailman/app/moderator.py index 78332b84a..9a65207a6 100644 --- a/src/mailman/app/moderator.py +++ b/src/mailman/app/moderator.py @@ -31,12 +31,11 @@ __all__ = [ ] +import six import time import logging from email.utils import formataddr, formatdate, getaddresses, make_msgid -from zope.component import getUtility - from mailman.app.membership import add_member, delete_member from mailman.app.notifications import send_admin_subscription_notice from mailman.config import config @@ -51,6 +50,7 @@ from mailman.interfaces.messages import IMessageStore from mailman.interfaces.requests import IListRequests, RequestType from mailman.utilities.datetime import now from mailman.utilities.i18n import make +from zope.component import getUtility NL = '\n' @@ -86,8 +86,8 @@ def hold_message(mlist, msg, msgdata=None, reason=None): # Message-ID header. message_id = msg.get('message-id') if message_id is None: - msg['Message-ID'] = message_id = unicode(make_msgid()) - assert isinstance(message_id, unicode), ( + msg['Message-ID'] = message_id = make_msgid().decode('utf-8') + assert isinstance(message_id, six.text_type), ( 'Message-ID is not a unicode: %s' % message_id) getUtility(IMessageStore).add(msg) # Prepare the message metadata with some extra information needed only by diff --git a/src/mailman/app/notifications.py b/src/mailman/app/notifications.py index 1fa1fe01e..99cbf0d0e 100644 --- a/src/mailman/app/notifications.py +++ b/src/mailman/app/notifications.py @@ -31,9 +31,6 @@ import logging from email.utils import formataddr from lazr.config import as_boolean -from urllib2 import URLError -from zope.component import getUtility - from mailman.config import config from mailman.core.i18n import _ from mailman.email.message import OwnerNotification, UserNotification @@ -41,6 +38,8 @@ from mailman.interfaces.member import DeliveryMode from mailman.interfaces.templates import ITemplateLoader from mailman.utilities.i18n import make from mailman.utilities.string import expand, wrap +from six.moves.urllib_error import URLError +from zope.component import getUtility log = logging.getLogger('mailman.error') diff --git a/src/mailman/app/templates.py b/src/mailman/app/templates.py index 742584b49..d0c278e3a 100644 --- a/src/mailman/app/templates.py +++ b/src/mailman/app/templates.py @@ -25,22 +25,22 @@ __all__ = [ ] -import urllib2 - from contextlib import closing -from urllib import addinfourl -from urlparse import urlparse -from zope.component import getUtility -from zope.interface import implementer - -from mailman.utilities.i18n import TemplateNotFoundError, find from mailman.interfaces.languages import ILanguageManager from mailman.interfaces.listmanager import IListManager from mailman.interfaces.templates import ITemplateLoader +from mailman.utilities.i18n import TemplateNotFoundError, find +from six.moves.urllib_error import URLError +from six.moves.urllib_parse import urlparse +from six.moves.urllib_request import ( + BaseHandler, build_opener, install_opener, urlopen) +from six.moves.urllib_response import addinfourl +from zope.component import getUtility +from zope.interface import implementer -class MailmanHandler(urllib2.BaseHandler): +class MailmanHandler(BaseHandler): # Handle internal mailman: URLs. def mailman_open(self, req): # Parse urls of the form: @@ -57,7 +57,7 @@ class MailmanHandler(urllib2.BaseHandler): # path components are legal, filter them out. parts = filter(None, parsed.path.split('/')) if len(parts) == 0: - raise urllib2.URLError('No template specified') + raise URLError('No template specified') elif len(parts) == 1: template = parts[0] elif len(parts) == 2: @@ -69,25 +69,25 @@ class MailmanHandler(urllib2.BaseHandler): language = getUtility(ILanguageManager).get(part0) mlist = getUtility(IListManager).get(part0) if language is None and mlist is None: - raise urllib2.URLError('Bad language or list name') + raise URLError('Bad language or list name') elif mlist is None: code = language.code elif len(parts) == 3: fqdn_listname, code, template = parts mlist = getUtility(IListManager).get(fqdn_listname) if mlist is None: - raise urllib2.URLError('Missing list') + raise URLError('Missing list') language = getUtility(ILanguageManager).get(code) if language is None: - raise urllib2.URLError('No such language') + raise URLError('No such language') code = language.code else: - raise urllib2.URLError('No such file') + raise URLError('No such file') # Find the template, mutating any missing template exception. try: path, fp = find(template, mlist, code) except TemplateNotFoundError: - raise urllib2.URLError('No such file') + raise URLError('No such file') return addinfourl(fp, {}, original_url) @@ -97,10 +97,10 @@ class TemplateLoader: """Loader of templates, with caching and support for mailman:// URIs.""" def __init__(self): - opener = urllib2.build_opener(MailmanHandler()) - urllib2.install_opener(opener) + opener = build_opener(MailmanHandler()) + install_opener(opener) def get(self, uri): """See `ITemplateLoader`.""" - with closing(urllib2.urlopen(uri)) as fp: + with closing(urlopen(uri)) as fp: return fp.read().decode('utf-8') diff --git a/src/mailman/app/tests/test_templates.py b/src/mailman/app/tests/test_templates.py index afde68647..40ec6d234 100644 --- a/src/mailman/app/tests/test_templates.py +++ b/src/mailman/app/tests/test_templates.py @@ -26,18 +26,18 @@ __all__ = [ import os +import six import shutil -import urllib2 import tempfile import unittest -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.config import config from mailman.interfaces.languages import ILanguageManager from mailman.interfaces.templates import ITemplateLoader from mailman.testing.layers import ConfigLayer +from six.moves.urllib_error import URLError +from zope.component import getUtility @@ -98,32 +98,32 @@ class TestTemplateLoader(unittest.TestCase): self.assertEqual(content, 'Test content') def test_uri_not_found(self): - with self.assertRaises(urllib2.URLError) as cm: + with self.assertRaises(URLError) as cm: self._loader.get('mailman:///missing.txt') self.assertEqual(cm.exception.reason, 'No such file') def test_shorter_url_error(self): - with self.assertRaises(urllib2.URLError) as cm: + with self.assertRaises(URLError) as cm: self._loader.get('mailman:///') self.assertEqual(cm.exception.reason, 'No template specified') def test_short_url_error(self): - with self.assertRaises(urllib2.URLError) as cm: + with self.assertRaises(URLError) as cm: self._loader.get('mailman://') self.assertEqual(cm.exception.reason, 'No template specified') def test_bad_language(self): - with self.assertRaises(urllib2.URLError) as cm: + with self.assertRaises(URLError) as cm: self._loader.get('mailman:///xx/demo.txt') self.assertEqual(cm.exception.reason, 'Bad language or list name') def test_bad_mailing_list(self): - with self.assertRaises(urllib2.URLError) as cm: + with self.assertRaises(URLError) as cm: self._loader.get('mailman:///missing@example.com/demo.txt') self.assertEqual(cm.exception.reason, 'Bad language or list name') def test_too_many_path_components(self): - with self.assertRaises(urllib2.URLError) as cm: + with self.assertRaises(URLError) as cm: self._loader.get('mailman:///missing@example.com/en/foo/demo.txt') self.assertEqual(cm.exception.reason, 'No such file') @@ -135,5 +135,5 @@ class TestTemplateLoader(unittest.TestCase): with open(os.path.join(path, 'demo.txt'), 'w') as fp: print(test_text, end='', file=fp) content = self._loader.get('mailman:///it/demo.txt') - self.assertTrue(isinstance(content, unicode)) + self.assertIsInstance(content, six.text_type) self.assertEqual(content, test_text.decode('utf-8')) diff --git a/src/mailman/archiving/prototype.py b/src/mailman/archiving/prototype.py index 153c44b69..e564b40b1 100644 --- a/src/mailman/archiving/prototype.py +++ b/src/mailman/archiving/prototype.py @@ -30,14 +30,13 @@ import errno import logging from datetime import timedelta -from mailbox import Maildir -from urlparse import urljoin - from flufl.lock import Lock, TimeOutError -from zope.interface import implementer - +from mailbox import Maildir from mailman.config import config from mailman.interfaces.archiver import IArchiver +from six.moves.urllib_parse import urljoin +from zope.interface import implementer + log = logging.getLogger('mailman.error') diff --git a/src/mailman/archiving/tests/test_prototype.py b/src/mailman/archiving/tests/test_prototype.py index fba46ea4b..6bc4f25b4 100644 --- a/src/mailman/archiving/tests/test_prototype.py +++ b/src/mailman/archiving/tests/test_prototype.py @@ -89,13 +89,13 @@ but the water deserves to be swum. def _find(self, path): all_filenames = set() for dirpath, dirnames, filenames in os.walk(path): - if not isinstance(dirpath, unicode): - dirpath = unicode(dirpath) + if isinstance(dirpath, bytes): + dirpath = dirpath.decode('utf-8') all_filenames.add(dirpath) for filename in filenames: new_filename = filename - if not isinstance(filename, unicode): - new_filename = unicode(filename) + if isinstance(filename, bytes): + new_filename = filename.decode('utf-8') all_filenames.add(os.path.join(dirpath, new_filename)) return all_filenames diff --git a/src/mailman/bin/export.py b/src/mailman/bin/export.py index a5400a9bc..1ee9f31e1 100644 --- a/src/mailman/bin/export.py +++ b/src/mailman/bin/export.py @@ -134,7 +134,7 @@ class XMLDumper(object): print >> self._fp, '<%s%s/>' % (_name, attrs) else: # The value might contain angle brackets. - value = escape(unicode(_value)) + value = escape(_value.decode('utf-8')) print >> self._fp, '<%s%s>%s' % (_name, attrs, value, _name) def _do_list_categories(self, mlist, k, subcat=None): diff --git a/src/mailman/commands/cli_import.py b/src/mailman/commands/cli_import.py index 5e25cd4fe..b53faea96 100644 --- a/src/mailman/commands/cli_import.py +++ b/src/mailman/commands/cli_import.py @@ -26,16 +26,15 @@ __all__ = [ import sys -import cPickle - -from zope.component import getUtility -from zope.interface import implementer from mailman.core.i18n import _ from mailman.database.transaction import transactional from mailman.interfaces.command import ICLISubCommand from mailman.interfaces.listmanager import IListManager from mailman.utilities.importer import import_config_pck, Import21Error +from six.moves import cPickle +from zope.component import getUtility +from zope.interface import implementer diff --git a/src/mailman/commands/cli_inject.py b/src/mailman/commands/cli_inject.py index 07ef0ec6c..9339dc074 100644 --- a/src/mailman/commands/cli_inject.py +++ b/src/mailman/commands/cli_inject.py @@ -27,14 +27,13 @@ __all__ = [ import sys -from zope.component import getUtility -from zope.interface import implementer - from mailman.app.inject import inject_text from mailman.config import config from mailman.core.i18n import _ from mailman.interfaces.command import ICLISubCommand from mailman.interfaces.listmanager import IListManager +from zope.component import getUtility +from zope.interface import implementer @@ -49,7 +48,7 @@ class Inject: self.parser = parser command_parser.add_argument( '-q', '--queue', - type=unicode, help=_(""" + help=_(""" The name of the queue to inject the message to. QUEUE must be one of the directories inside the qfiles directory. If omitted, the incoming queue is used.""")) @@ -59,7 +58,7 @@ class Inject: help=_('Show a list of all available queue names and exit.')) command_parser.add_argument( '-f', '--filename', - type=unicode, help=_(""" + help=_(""" Name of file containing the message to inject. If not given, or '-' (without the quotes) standard input is used.""")) # Required positional argument. diff --git a/src/mailman/commands/cli_qfile.py b/src/mailman/commands/cli_qfile.py index 986898bee..c4ff66aea 100644 --- a/src/mailman/commands/cli_qfile.py +++ b/src/mailman/commands/cli_qfile.py @@ -25,14 +25,12 @@ __all__ = [ ] -import cPickle - -from pprint import PrettyPrinter -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.interfaces.command import ICLISubCommand from mailman.utilities.interact import interact +from pprint import PrettyPrinter +from six.moves import cPickle +from zope.interface import implementer m = [] diff --git a/src/mailman/commands/docs/echo.rst b/src/mailman/commands/docs/echo.rst index 32399ebfc..686accf2c 100644 --- a/src/mailman/commands/docs/echo.rst +++ b/src/mailman/commands/docs/echo.rst @@ -24,7 +24,7 @@ The original message is ignored, but the results receive the echoed command. >>> from mailman.email.message import Message >>> print(command.process(mlist, Message(), {}, ('foo', 'bar'), results)) ContinueProcessing.yes - >>> print(unicode(results)) + >>> print(results.decode('utf-8')) The results of your email command are provided below. echo foo bar diff --git a/src/mailman/commands/docs/help.rst b/src/mailman/commands/docs/help.rst index bbd6c8c09..35ba87caa 100644 --- a/src/mailman/commands/docs/help.rst +++ b/src/mailman/commands/docs/help.rst @@ -25,7 +25,7 @@ short description of each of them. >>> from mailman.email.message import Message >>> print(help.process(mlist, Message(), {}, (), results)) ContinueProcessing.yes - >>> print(unicode(results)) + >>> print(results.decode('utf-8')) The results of your email command are provided below. confirm - Confirm a subscription request. @@ -44,19 +44,19 @@ With an argument, you can get more detailed help about a specific command. >>> results = Results() >>> print(help.process(mlist, Message(), {}, ('help',), results)) ContinueProcessing.yes - >>> print(unicode(results)) + >>> print(results.decode('utf-8')) The results of your email command are provided below. help [command] Get help about available email commands. - + Some commands have even more detailed help. >>> results = Results() >>> print(help.process(mlist, Message(), {}, ('join',), results)) ContinueProcessing.yes - >>> print(unicode(results)) + >>> print(results.decode('utf-8')) The results of your email command are provided below. join [digest=] diff --git a/src/mailman/commands/docs/membership.rst b/src/mailman/commands/docs/membership.rst index aa3ab97e6..3fe9b05ba 100644 --- a/src/mailman/commands/docs/membership.rst +++ b/src/mailman/commands/docs/membership.rst @@ -45,7 +45,7 @@ If that's missing though, then an error is returned. >>> from mailman.email.message import Message >>> print(join.process(mlist, Message(), {}, (), results)) ContinueProcessing.no - >>> print(unicode(results)) + >>> print(results.decode('utf-8')) The results of your email command are provided below. join: No valid address found to subscribe @@ -60,7 +60,7 @@ The ``subscribe`` command is an alias. >>> results = Results() >>> print(subscribe.process(mlist, Message(), {}, (), results)) ContinueProcessing.no - >>> print(unicode(results)) + >>> print(results.decode('utf-8')) The results of your email command are provided below. subscribe: No valid address found to subscribe @@ -79,7 +79,7 @@ When the message has a From field, that address will be subscribed. >>> results = Results() >>> print(join.process(mlist, msg, {}, (), results)) ContinueProcessing.yes - >>> print(unicode(results)) + >>> print(results.decode('utf-8')) The results of your email command are provided below. Confirmation email sent to Anne Person @@ -150,7 +150,7 @@ list. >>> results = Results() >>> print(confirm.process(mlist, msg, {}, (token,), results)) ContinueProcessing.yes - >>> print(unicode(results)) + >>> print(results.decode('utf-8')) The results of your email command are provided below. Confirmed @@ -208,7 +208,7 @@ list. >>> results = Results() >>> print(confirm.process(mlist_2, msg, {}, (token,), results)) ContinueProcessing.yes - >>> print(unicode(results)) + >>> print(results.decode('utf-8')) The results of your email command are provided below. Confirmed @@ -241,7 +241,7 @@ is sent a confirmation message for her request. >>> results = Results() >>> print(leave.process(mlist_2, msg, {}, (), results)) ContinueProcessing.yes - >>> print(unicode(results)) + >>> print(results.decode('utf-8')) The results of your email command are provided below. Anne Person left baker@example.com @@ -278,7 +278,7 @@ to unsubscribe Anne from the alpha mailing list. >>> print(leave.process(mlist, msg, {}, (), results)) ContinueProcessing.no - >>> print(unicode(results)) + >>> print(results.decode('utf-8')) The results of your email command are provided below. Invalid or unverified email address: anne.person@example.org @@ -299,7 +299,7 @@ unsubscribe her from the list. >>> print(leave.process(mlist, msg, {}, (), results)) ContinueProcessing.yes - >>> print(unicode(results)) + >>> print(results.decode('utf-8')) The results of your email command are provided below. Anne Person left alpha@example.com @@ -354,7 +354,7 @@ a user of the system. >>> print(confirm.process(mlist, msg, {}, (token,), results)) ContinueProcessing.yes - >>> print(unicode(results)) + >>> print(results.decode('utf-8')) The results of your email command are provided below. Confirmed diff --git a/src/mailman/commands/tests/test_conf.py b/src/mailman/commands/tests/test_conf.py index 12ed5c537..cc0f61ba2 100644 --- a/src/mailman/commands/tests/test_conf.py +++ b/src/mailman/commands/tests/test_conf.py @@ -31,9 +31,9 @@ import mock import tempfile import unittest -from StringIO import StringIO from mailman.commands.cli_conf import Conf from mailman.testing.layers import ConfigLayer +from six import StringIO diff --git a/src/mailman/commands/tests/test_help.py b/src/mailman/commands/tests/test_help.py index 3c7d1ae9f..74eaae84e 100644 --- a/src/mailman/commands/tests/test_help.py +++ b/src/mailman/commands/tests/test_help.py @@ -47,11 +47,11 @@ class TestHelp(unittest.TestCase): def test_too_many_arguments(self): # Error message when too many help arguments are given. results = Results() - status = self._help.process(self._mlist, Message(), {}, + status = self._help.process(self._mlist, Message(), {}, ('more', 'than', 'one'), results) self.assertEqual(status, ContinueProcessing.no) - self.assertEqual(unicode(results), """\ + self.assertEqual(results.decode('utf-8'), """\ The results of your email command are provided below. help: too many arguments: more than one @@ -60,10 +60,10 @@ help: too many arguments: more than one def test_no_such_command(self): # Error message when asking for help on an existent command. results = Results() - status = self._help.process(self._mlist, Message(), {}, + status = self._help.process(self._mlist, Message(), {}, ('doesnotexist',), results) self.assertEqual(status, ContinueProcessing.no) - self.assertEqual(unicode(results), """\ + self.assertEqual(results.decode('utf-8'), """\ The results of your email command are provided below. help: no such command: doesnotexist diff --git a/src/mailman/config/config.py b/src/mailman/config/config.py index 649d6c5e1..c92485176 100644 --- a/src/mailman/config/config.py +++ b/src/mailman/config/config.py @@ -30,11 +30,12 @@ __all__ = [ import os import sys -from ConfigParser import SafeConfigParser from flufl.lock import Lock from lazr.config import ConfigSchema, as_boolean -from pkg_resources import resource_stream, resource_string +from pkg_resources import resource_filename, resource_string +from six.moves.configparser import ConfigParser, RawConfigParser from string import Template +from unittest.mock import patch from zope.component import getUtility from zope.event import notify from zope.interface import implementer @@ -66,6 +67,11 @@ MAILMAN_CFG_TEMPLATE = """\ # enabled: yes # recipient: your.address@your.domain""" +class _NonStrictRawConfigParser(RawConfigParser): + def __init__(self, *args, **kws): + kws['strict'] = False + super().__init__(*args, **kws) + @implementer(IConfiguration) @@ -99,30 +105,27 @@ class Configuration: def load(self, filename=None): """Load the configuration from the schema and config files.""" - schema_file = config_file = None - try: - schema_file = resource_stream('mailman.config', 'schema.cfg') - schema = ConfigSchema('schema.cfg', schema_file) - # If a configuration file was given, load it now too. First, load - # the absolute minimum default configuration, then if a - # configuration filename was given by the user, push it. - config_file = resource_stream('mailman.config', 'mailman.cfg') - self._config = schema.loadFile(config_file, 'mailman.cfg') - if filename is not None: - self.filename = filename - with open(filename) as user_config: - self._config.push(filename, user_config.read()) - finally: - if schema_file: - schema_file.close() - if config_file: - config_file.close() - self._post_process() + schema_file = resource_filename('mailman.config', 'schema.cfg') + schema = ConfigSchema(schema_file) + # If a configuration file was given, load it now too. First, load + # the absolute minimum default configuration, then if a + # configuration filename was given by the user, push it. + config_file = resource_filename('mailman.config', 'mailman.cfg') + self._config = schema.load(config_file) + if filename is not None: + self.filename = filename + with open(filename) as user_config: + self.push(filename, user_config.read()) def push(self, config_name, config_string): """Push a new configuration onto the stack.""" self._clear() - self._config.push(config_name, config_string) + # In Python 3, the RawConfigParser() must be created with + # strict=False, otherwise we'll get a DuplicateSectionError. + # See https://bugs.launchpad.net/lazr.config/+bug/1397779 + with patch('lazr.config._config.RawConfigParser', + _NonStrictRawConfigParser): + self._config.push(config_name, config_string) self._post_process() def pop(self, config_name): @@ -305,7 +308,7 @@ def external_configuration(path): """ # Is the context coming from a file system or Python path? cfg_path = expand_path(path) - parser = SafeConfigParser() + parser = ConfigParser() files = parser.read(cfg_path) if files != [cfg_path]: raise MissingConfigurationFileError(path) diff --git a/src/mailman/config/tests/test_configuration.py b/src/mailman/config/tests/test_configuration.py index f3a49d64f..b411f9615 100644 --- a/src/mailman/config/tests/test_configuration.py +++ b/src/mailman/config/tests/test_configuration.py @@ -28,6 +28,7 @@ __all__ = [ import os +import six import mock import tempfile import unittest @@ -79,12 +80,12 @@ class TestExternal(unittest.TestCase): def test_load_external_by_filename_as_string(self): filename = resource_filename('mailman.config', 'postfix.cfg') contents = load_external(filename, encoding='utf-8') - self.assertIsInstance(contents, unicode) + self.assertIsInstance(contents, six.text_type) self.assertEqual(contents[:9], '[postfix]') def test_load_external_by_path_as_string(self): contents = load_external('python:mailman.config.postfix', 'utf-8') - self.assertIsInstance(contents, unicode) + self.assertIsInstance(contents, six.text_type) self.assertEqual(contents[:9], '[postfix]') def test_external_configuration_by_filename(self): diff --git a/src/mailman/core/initialize.py b/src/mailman/core/initialize.py index c2395db10..6c7196990 100644 --- a/src/mailman/core/initialize.py +++ b/src/mailman/core/initialize.py @@ -39,7 +39,7 @@ __all__ = [ import os import sys -from pkg_resources import resource_string +from pkg_resources import resource_string as resource_bytes from zope.component import getUtility from zope.configuration import xmlconfig @@ -109,8 +109,8 @@ def initialize_1(config_path=None): :param config_path: The path to the configuration file. :type config_path: string """ - zcml = resource_string('mailman.config', 'configure.zcml') - xmlconfig.string(zcml) + zcml = resource_bytes('mailman.config', 'configure.zcml') + xmlconfig.string(zcml.decode('utf-8')) # By default, set the umask so that only owner and group can read and # write our files. Specifically we must have g+rw and we probably want # o-rwx although I think in most cases it doesn't hurt if other can read diff --git a/src/mailman/core/runner.py b/src/mailman/core/runner.py index 81a2ea3d1..d6aad2b07 100644 --- a/src/mailman/core/runner.py +++ b/src/mailman/core/runner.py @@ -30,12 +30,7 @@ import signal import logging import traceback -from cStringIO import StringIO from lazr.config import as_boolean, as_timedelta -from zope.component import getUtility -from zope.event import notify -from zope.interface import implementer - from mailman.config import config from mailman.core.i18n import _ from mailman.core.logging import reopen @@ -44,6 +39,10 @@ from mailman.interfaces.languages import ILanguageManager from mailman.interfaces.listmanager import IListManager from mailman.interfaces.runner import IRunner, RunnerCrashEvent from mailman.utilities.string import expand +from six.moves import cStringIO as StringIO +from zope.component import getUtility +from zope.event import notify +from zope.interface import implementer dlog = logging.getLogger('mailman.debug') @@ -218,11 +217,11 @@ class Runner: # them out of our sight. # # Find out which mailing list this message is destined for. + mlist = None missing = object() listname = msgdata.get('listname', missing) - mlist = (None - if listname is missing - else getUtility(IListManager).get(unicode(listname))) + if listname is missing: + mlist = getUtility(IListManager).get(listname.decode('utf-8')) if mlist is None: elog.error( '%s runner "%s" shunting message for missing list: %s', diff --git a/src/mailman/core/switchboard.py b/src/mailman/core/switchboard.py index 2e8ef24a7..78a12616e 100644 --- a/src/mailman/core/switchboard.py +++ b/src/mailman/core/switchboard.py @@ -37,22 +37,22 @@ import os import time import email import pickle -import cPickle import hashlib import logging -from zope.interface import implementer - from mailman.config import config from mailman.email.message import Message from mailman.interfaces.configuration import ConfigurationUpdatedEvent from mailman.interfaces.switchboard import ISwitchboard from mailman.utilities.filesystem import makedirs from mailman.utilities.string import expand +from six.moves import cPickle +from zope.interface import implementer -# 20 bytes of all bits set, maximum hashlib.sha.digest() value. -shamax = 0xffffffffffffffffffffffffffffffffffffffffL +# 20 bytes of all bits set, maximum hashlib.sha.digest() value. We do it this +# way for Python 2/3 compatibility. +shamax = int('0xffffffffffffffffffffffffffffffffffffffff', 16) # Small increment to add to time in case two entries have the same time. This # prevents skipping one of two entries with the same time until the next pass. DELTA = .0001 @@ -92,7 +92,7 @@ class Switchboard: self.queue_directory = queue_directory # If configured to, create the directory if it doesn't yet exist. if config.create_paths: - makedirs(self.queue_directory, 0770) + makedirs(self.queue_directory, 0o770) # Fast track for no slices self._lower = None self._upper = None @@ -123,7 +123,7 @@ class Switchboard: msgsave = cPickle.dumps(_msg, protocol) # listname is unicode but the input to the hash function must be an # 8-bit string (eventually, a bytes object). - hashfood = msgsave + listname.encode('utf-8') + repr(now) + hashfood = msgsave + listname + repr(now) # Encode the current time into the file name for FIFO sorting. The # file name consists of two parts separated by a '+': the received # time for this message (i.e. when it first showed up on this system) @@ -207,13 +207,13 @@ class Switchboard: # Throw out any files which don't match our bitrange. BAW: test # performance and end-cases of this algorithm. MAS: both # comparisons need to be <= to get complete range. - if lower is None or (lower <= long(digest, 16) <= upper): + if lower is None or (lower <= int(digest, 16) <= upper): key = float(when) while key in times: key += DELTA times[key] = filebase # FIFO sort - return [times[key] for key in sorted(times)] + return [times[k] for k in sorted(times)] def recover_backup_files(self): """See `ISwitchboard`.""" @@ -228,7 +228,8 @@ class Switchboard: dst = os.path.join(self.queue_directory, filebase + '.pck') with open(src, 'rb+') as fp: try: - msg = cPickle.load(fp) + # Throw away the message object. + cPickle.load(fp) data_pos = fp.tell() data = cPickle.load(fp) except Exception as error: diff --git a/src/mailman/database/sqlite.py b/src/mailman/database/sqlite.py index db7860390..8540846e1 100644 --- a/src/mailman/database/sqlite.py +++ b/src/mailman/database/sqlite.py @@ -28,7 +28,7 @@ __all__ = [ import os from mailman.database.base import SABaseDatabase -from urlparse import urlparse +from six.moves.urllib_parse import urlparse diff --git a/src/mailman/email/message.py b/src/mailman/email/message.py index f3a44e63c..4d26ca9c4 100644 --- a/src/mailman/email/message.py +++ b/src/mailman/email/message.py @@ -56,15 +56,15 @@ class Message(email.message.Message): def __getitem__(self, key): # Ensure that header values are unicodes. value = email.message.Message.__getitem__(self, key) - if isinstance(value, str): - return unicode(value, 'ascii') + if isinstance(value, bytes): + return value.decode('ascii') return value def get(self, name, failobj=None): # Ensure that header values are unicodes. value = email.message.Message.get(self, name, failobj) - if isinstance(value, str): - return unicode(value, 'ascii') + if isinstance(value, bytes): + return value.decode('ascii') return value def get_all(self, name, failobj=None): @@ -73,7 +73,7 @@ class Message(email.message.Message): all_values = email.message.Message.get_all(self, name, missing) if all_values is missing: return failobj - return [(unicode(value, 'ascii') if isinstance(value, str) else value) + return [(value.decode('ascii') if isinstance(value, bytes) else value) for value in all_values] # BAW: For debugging w/ bin/dumpdb. Apparently pprint uses repr. diff --git a/src/mailman/handlers/cook_headers.py b/src/mailman/handlers/cook_headers.py index d5d096448..5fdfc238b 100644 --- a/src/mailman/handlers/cook_headers.py +++ b/src/mailman/handlers/cook_headers.py @@ -201,7 +201,7 @@ def prefix_subject(mlist, msg, msgdata): # range. It is safe to use unicode string when manupilating header # contents with re module. It would be best to return unicode in # ch_oneline() but here is temporary solution. - subject = unicode(subject, cset) + subject = subject.decode(cset) # If the subject_prefix contains '%d', it is replaced with the # mailing list sequential number. Sequential number format allows # '%d' or '%05d' like pattern. @@ -270,7 +270,7 @@ def ch_oneline(headerstr): else: cset = 'utf-8' h = make_header(d) - ustr = unicode(h) + ustr = h.decode('utf-8') oneline = ''.join(ustr.splitlines()) return oneline.encode(cset, 'replace'), cset except (LookupError, UnicodeError, ValueError, HeaderParseError): diff --git a/src/mailman/handlers/decorate.py b/src/mailman/handlers/decorate.py index bf8454232..03f0c009f 100644 --- a/src/mailman/handlers/decorate.py +++ b/src/mailman/handlers/decorate.py @@ -31,15 +31,14 @@ import re import logging from email.mime.text import MIMEText -from urllib2 import URLError -from zope.component import getUtility -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.email.message import Message from mailman.interfaces.handler import IHandler from mailman.interfaces.templates import ITemplateLoader from mailman.utilities.string import expand +from six.moves.urllib_error import URLError +from zope.component import getUtility +from zope.interface import implementer log = logging.getLogger('mailman.error') diff --git a/src/mailman/handlers/docs/subject-munging.rst b/src/mailman/handlers/docs/subject-munging.rst index 538ad99c7..072e80d17 100644 --- a/src/mailman/handlers/docs/subject-munging.rst +++ b/src/mailman/handlers/docs/subject-munging.rst @@ -124,7 +124,7 @@ set than the encoded header. >>> process(mlist, msg, {}) >>> print(msg['subject']) [XTest] =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= - >>> unicode(msg['subject']) + >>> msg['subject'].decode('utf-8') u'[XTest] \u30e1\u30fc\u30eb\u30de\u30f3' @@ -180,7 +180,7 @@ in the subject prefix, and the subject is encoded non-ASCII. >>> process(mlist, msg, {}) >>> print(msg['subject']) [XTest 456] =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= - >>> unicode(msg['subject']) + >>> msg['subject'].decode('utf-8') u'[XTest 456] \u30e1\u30fc\u30eb\u30de\u30f3' Even more fun is when the internationalized ``Subject`` header already has a @@ -194,10 +194,8 @@ prefix, possibly with a different posting number. >>> print(msg['subject']) [XTest 456] Re: =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= -.. - # XXX This requires Python email patch #1681333 to succeed. - # >>> unicode(msg['subject']) - # u'[XTest 456] Re: \u30e1\u30fc\u30eb\u30de\u30f3' + >>> msg['subject'].decode('utf-8') + u'[XTest 456] Re: \u30e1\u30fc\u30eb\u30de\u30f3' As before, old style subject prefixes are re-ordered. @@ -210,10 +208,8 @@ As before, old style subject prefixes are re-ordered. [XTest 456] Re: =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= -.. - # XXX This requires Python email patch #1681333 to succeed. - # >>> unicode(msg['subject']) - # u'[XTest 456] Re: \u30e1\u30fc\u30eb\u30de\u30f3' + >>> msg['subject'].decode('utf-8') + u'[XTest 456] Re: \u30e1\u30fc\u30eb\u30de\u30f3' In this test case, we get an extra space between the prefix and the original diff --git a/src/mailman/handlers/tests/test_recipients.py b/src/mailman/handlers/tests/test_recipients.py index afe533a7e..8f2a9d47d 100644 --- a/src/mailman/handlers/tests/test_recipients.py +++ b/src/mailman/handlers/tests/test_recipients.py @@ -26,15 +26,16 @@ __all__ = [ ] +import six import unittest -from zope.component import getUtility from mailman.app.lifecycle import create_list from mailman.config import config from mailman.interfaces.member import DeliveryMode, DeliveryStatus, MemberRole from mailman.interfaces.usermanager import IUserManager from mailman.testing.helpers import specialized_message_from_string as mfs from mailman.testing.layers import ConfigLayer +from zope.component import getUtility @@ -218,4 +219,4 @@ site_owner: siteadmin@example.com finally: config.pop('test_site_admin_unicode') self.assertEqual(len(msgdata['recipients']), 1) - self.assertIsInstance(list(msgdata['recipients'])[0], unicode) + self.assertIsInstance(list(msgdata['recipients'])[0], six.text_type) diff --git a/src/mailman/model/domain.py b/src/mailman/model/domain.py index 8290cb755..a9020e816 100644 --- a/src/mailman/model/domain.py +++ b/src/mailman/model/domain.py @@ -26,17 +26,16 @@ __all__ = [ ] -from sqlalchemy import Column, Integer, Unicode -from urlparse import urljoin, urlparse -from zope.event import notify -from zope.interface import implementer - from mailman.database.model import Model from mailman.database.transaction import dbconnection from mailman.interfaces.domain import ( BadDomainSpecificationError, DomainCreatedEvent, DomainCreatingEvent, DomainDeletedEvent, DomainDeletingEvent, IDomain, IDomainManager) from mailman.model.mailinglist import MailingList +from six.moves.urllib_parse import urljoin, urlparse +from sqlalchemy import Column, Integer, Unicode +from zope.event import notify +from zope.interface import implementer diff --git a/src/mailman/model/mailinglist.py b/src/mailman/model/mailinglist.py index 761a78b94..c55786fe8 100644 --- a/src/mailman/model/mailinglist.py +++ b/src/mailman/model/mailinglist.py @@ -27,16 +27,6 @@ __all__ = [ import os -from sqlalchemy import ( - Boolean, Column, DateTime, Float, ForeignKey, Integer, Interval, - LargeBinary, PickleType, Unicode) -from sqlalchemy.event import listen -from sqlalchemy.orm import relationship -from urlparse import urljoin -from zope.component import getUtility -from zope.event import notify -from zope.interface import implementer - from mailman.config import config from mailman.database.model import Model from mailman.database.transaction import dbconnection @@ -65,6 +55,15 @@ from mailman.model.mime import ContentFilter from mailman.model.preferences import Preferences from mailman.utilities.filesystem import makedirs from mailman.utilities.string import expand +from six.moves.urllib_parse import urljoin +from sqlalchemy import ( + Boolean, Column, DateTime, Float, ForeignKey, Integer, Interval, + LargeBinary, PickleType, Unicode) +from sqlalchemy.event import listen +from sqlalchemy.orm import relationship +from zope.component import getUtility +from zope.event import notify +from zope.interface import implementer SPACE = ' ' diff --git a/src/mailman/model/messagestore.py b/src/mailman/model/messagestore.py index 19fa8133f..19b87c610 100644 --- a/src/mailman/model/messagestore.py +++ b/src/mailman/model/messagestore.py @@ -27,16 +27,15 @@ __all__ = [ import os import errno import base64 +import pickle import hashlib -import cPickle as pickle - -from zope.interface import implementer from mailman.config import config from mailman.database.transaction import dbconnection from mailman.interfaces.messages import IMessageStore from mailman.model.message import Message from mailman.utilities.filesystem import makedirs +from zope.interface import implementer # It could be very bad if you have already stored files and you change this diff --git a/src/mailman/model/requests.py b/src/mailman/model/requests.py index 6b130196d..24575d3c8 100644 --- a/src/mailman/model/requests.py +++ b/src/mailman/model/requests.py @@ -24,18 +24,19 @@ __all__ = [ ] -from cPickle import dumps, loads -from datetime import timedelta -from sqlalchemy import Column, ForeignKey, Integer, LargeBinary, Unicode -from sqlalchemy.orm import relationship -from zope.component import getUtility -from zope.interface import implementer +import six +from datetime import timedelta from mailman.database.model import Model from mailman.database.transaction import dbconnection from mailman.database.types import Enum from mailman.interfaces.pending import IPendable, IPendings from mailman.interfaces.requests import IListRequests, RequestType +from six.moves.cPickle import dumps, loads +from sqlalchemy import Column, ForeignKey, Integer, LargeBinary, Unicode +from sqlalchemy.orm import relationship +from zope.component import getUtility +from zope.interface import implementer @@ -50,8 +51,8 @@ class DataPendable(dict): # such a way that it will be properly reconstituted when unpended. clean_mapping = {} for key, value in mapping.items(): - assert isinstance(key, basestring) - if not isinstance(value, unicode): + assert isinstance(key, six.string_types) + if not isinstance(value, six.text_type): key = '_pck_' + key value = dumps(value).decode('raw-unicode-escape') clean_mapping[key] = value diff --git a/src/mailman/options.py b/src/mailman/options.py index a4f553a09..07565f611 100644 --- a/src/mailman/options.py +++ b/src/mailman/options.py @@ -42,7 +42,7 @@ from mailman.version import MAILMAN_VERSION def check_unicode(option, opt, value): """Check that the value is a unicode string.""" - if isinstance(value, unicode): + if not isinstance(value, bytes): return value try: return value.decode(sys.getdefaultencoding()) diff --git a/src/mailman/rest/addresses.py b/src/mailman/rest/addresses.py index fa3d099b6..7923d8bf9 100644 --- a/src/mailman/rest/addresses.py +++ b/src/mailman/rest/addresses.py @@ -27,6 +27,8 @@ __all__ = [ ] +import six + from operator import attrgetter from zope.component import getUtility @@ -186,8 +188,8 @@ class UserAddresses(_AddressBase): not_found(response) return user_manager = getUtility(IUserManager) - validator = Validator(email=unicode, - display_name=unicode, + validator = Validator(email=six.text_type, + display_name=six.text_type, _optional=('display_name',)) try: address = user_manager.create_address(**validator(request)) diff --git a/src/mailman/rest/configuration.py b/src/mailman/rest/configuration.py index b432268c7..6ea78f90e 100644 --- a/src/mailman/rest/configuration.py +++ b/src/mailman/rest/configuration.py @@ -25,6 +25,8 @@ __all__ = [ ] +import six + from lazr.config import as_boolean, as_timedelta from mailman.config import config from mailman.core.errors import ( @@ -61,7 +63,7 @@ class AcceptableAliases(GetterSetter): alias_set = IAcceptableAliasSet(mlist) alias_set.clear() for alias in value: - alias_set.add(unicode(alias)) + alias_set.add(alias.decode('utf-8')) @@ -71,13 +73,13 @@ class AcceptableAliases(GetterSetter): def pipeline_validator(pipeline_name): """Convert the pipeline name to a string, but only if it's known.""" if pipeline_name in config.pipelines: - return unicode(pipeline_name) + return pipeline_name.decode('utf-8') raise ValueError('Unknown pipeline: {}'.format(pipeline_name)) def list_of_unicode(values): """Turn a list of things into a list of unicodes.""" - return [unicode(value) for value in values] + return [value.decode('utf-8') for value in values] @@ -106,9 +108,9 @@ ATTRIBUTES = dict( autorespond_postings=GetterSetter(enum_validator(ResponseAction)), autorespond_requests=GetterSetter(enum_validator(ResponseAction)), autoresponse_grace_period=GetterSetter(as_timedelta), - autoresponse_owner_text=GetterSetter(unicode), - autoresponse_postings_text=GetterSetter(unicode), - autoresponse_request_text=GetterSetter(unicode), + autoresponse_owner_text=GetterSetter(six.text_type), + autoresponse_postings_text=GetterSetter(six.text_type), + autoresponse_request_text=GetterSetter(six.text_type), archive_policy=GetterSetter(enum_validator(ArchivePolicy)), bounces_address=GetterSetter(None), collapse_alternatives=GetterSetter(as_boolean), @@ -116,7 +118,7 @@ ATTRIBUTES = dict( created_at=GetterSetter(None), default_member_action=GetterSetter(enum_validator(Action)), default_nonmember_action=GetterSetter(enum_validator(Action)), - description=GetterSetter(unicode), + description=GetterSetter(six.text_type), digest_last_sent_at=GetterSetter(None), digest_size_threshold=GetterSetter(float), filter_content=GetterSetter(as_boolean), @@ -135,21 +137,21 @@ ATTRIBUTES = dict( post_id=GetterSetter(None), posting_address=GetterSetter(None), posting_pipeline=GetterSetter(pipeline_validator), - display_name=GetterSetter(unicode), + display_name=GetterSetter(six.text_type), reply_goes_to_list=GetterSetter(enum_validator(ReplyToMunging)), - reply_to_address=GetterSetter(unicode), + reply_to_address=GetterSetter(six.text_type), request_address=GetterSetter(None), scheme=GetterSetter(None), send_welcome_message=GetterSetter(as_boolean), - subject_prefix=GetterSetter(unicode), + subject_prefix=GetterSetter(six.text_type), volume=GetterSetter(None), web_host=GetterSetter(None), - welcome_message_uri=GetterSetter(unicode), + welcome_message_uri=GetterSetter(six.text_type), ) VALIDATORS = ATTRIBUTES.copy() -for attribute, gettersetter in VALIDATORS.items(): +for attribute, gettersetter in list(VALIDATORS.items()): if gettersetter.decoder is None: del VALIDATORS[attribute] diff --git a/src/mailman/rest/docs/helpers.rst b/src/mailman/rest/docs/helpers.rst index 5bcf5cad4..2dd65bbb8 100644 --- a/src/mailman/rest/docs/helpers.rst +++ b/src/mailman/rest/docs/helpers.rst @@ -69,8 +69,9 @@ Another helper unpacks ``POST`` and ``PUT`` request variables, validating and converting their values. :: + >>> import six >>> from mailman.rest.validator import Validator - >>> validator = Validator(one=int, two=unicode, three=bool) + >>> validator = Validator(one=int, two=six.text_type, three=bool) >>> class FakeRequest: ... params = None @@ -119,7 +120,7 @@ Extra keys are also not allowed. However, if optional keys are missing, it's okay. :: - >>> validator = Validator(one=int, two=unicode, three=bool, + >>> validator = Validator(one=int, two=six.text_type, three=bool, ... four=int, five=int, ... _optional=('four', 'five')) diff --git a/src/mailman/rest/domains.py b/src/mailman/rest/domains.py index 5d36dcab9..bd221abeb 100644 --- a/src/mailman/rest/domains.py +++ b/src/mailman/rest/domains.py @@ -26,6 +26,8 @@ __all__ = [ ] +import six + from mailman.interfaces.domain import ( BadDomainSpecificationError, IDomainManager) from mailman.rest.helpers import ( @@ -99,10 +101,10 @@ class AllDomains(_DomainBase): """Create a new domain.""" domain_manager = getUtility(IDomainManager) try: - validator = Validator(mail_host=unicode, - description=unicode, - base_url=unicode, - contact_address=unicode, + validator = Validator(mail_host=six.text_type, + description=six.text_type, + base_url=six.text_type, + contact_address=six.text_type, _optional=('description', 'base_url', 'contact_address')) domain = domain_manager.add(**validator(request)) diff --git a/src/mailman/rest/lists.py b/src/mailman/rest/lists.py index 580b6e898..feaa96323 100644 --- a/src/mailman/rest/lists.py +++ b/src/mailman/rest/lists.py @@ -30,6 +30,8 @@ __all__ = [ ] +import six + from lazr.config import as_boolean from operator import attrgetter from zope.component import getUtility @@ -204,8 +206,8 @@ class AllLists(_ListBase): def on_post(self, request, response): """Create a new mailing list.""" try: - validator = Validator(fqdn_listname=unicode, - style_name=unicode, + validator = Validator(fqdn_listname=six.text_type, + style_name=six.text_type, _optional=('style_name',)) mlist = create_list(**validator(request)) except ListAlreadyExistsError: diff --git a/src/mailman/rest/members.py b/src/mailman/rest/members.py index 4d1c87b73..b63f65658 100644 --- a/src/mailman/rest/members.py +++ b/src/mailman/rest/members.py @@ -28,6 +28,8 @@ __all__ = [ ] +import six + from uuid import UUID from operator import attrgetter from zope.component import getUtility @@ -176,7 +178,7 @@ class AMember(_MemberBase): return try: values = Validator( - address=unicode, + address=six.text_type, delivery_mode=enum_validator(DeliveryMode), _optional=('address', 'delivery_mode'))(request) except ValueError as error: @@ -207,9 +209,9 @@ class AllMembers(_MemberBase): service = getUtility(ISubscriptionService) try: validator = Validator( - list_id=unicode, + list_id=six.text_type, subscriber=subscriber_validator, - display_name=unicode, + display_name=six.text_type, delivery_mode=enum_validator(DeliveryMode), role=enum_validator(MemberRole), _optional=('delivery_mode', 'display_name', 'role')) @@ -256,8 +258,8 @@ class FindMembers(_MemberBase): """Find a member""" service = getUtility(ISubscriptionService) validator = Validator( - list_id=unicode, - subscriber=unicode, + list_id=six.text_type, + subscriber=six.text_type, role=enum_validator(MemberRole), _optional=('list_id', 'subscriber', 'role')) try: diff --git a/src/mailman/rest/tests/test_addresses.py b/src/mailman/rest/tests/test_addresses.py index f4aeb3013..29e09355e 100644 --- a/src/mailman/rest/tests/test_addresses.py +++ b/src/mailman/rest/tests/test_addresses.py @@ -27,15 +27,14 @@ __all__ = [ import unittest -from urllib2 import HTTPError -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.database.transaction import transaction from mailman.interfaces.usermanager import IUserManager from mailman.testing.helpers import call_api from mailman.testing.layers import RESTLayer from mailman.utilities.datetime import now +from six.moves.urllib_error import HTTPError +from zope.component import getUtility diff --git a/src/mailman/rest/tests/test_domains.py b/src/mailman/rest/tests/test_domains.py index 44cf11ef3..48f9c4fe3 100644 --- a/src/mailman/rest/tests/test_domains.py +++ b/src/mailman/rest/tests/test_domains.py @@ -27,14 +27,13 @@ __all__ = [ import unittest -from urllib2 import HTTPError -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.database.transaction import transaction from mailman.interfaces.listmanager import IListManager from mailman.testing.helpers import call_api from mailman.testing.layers import RESTLayer +from six.moves.urllib_error import HTTPError +from zope.component import getUtility diff --git a/src/mailman/rest/tests/test_lists.py b/src/mailman/rest/tests/test_lists.py index ba6f6ea59..e8cc54d4b 100644 --- a/src/mailman/rest/tests/test_lists.py +++ b/src/mailman/rest/tests/test_lists.py @@ -30,14 +30,13 @@ __all__ = [ import unittest -from urllib2 import HTTPError -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.database.transaction import transaction from mailman.interfaces.usermanager import IUserManager from mailman.testing.helpers import call_api from mailman.testing.layers import RESTLayer +from six.moves.urllib_error import HTTPError +from zope.component import getUtility diff --git a/src/mailman/rest/tests/test_membership.py b/src/mailman/rest/tests/test_membership.py index 3c7d0520b..6b40fbb01 100644 --- a/src/mailman/rest/tests/test_membership.py +++ b/src/mailman/rest/tests/test_membership.py @@ -28,9 +28,6 @@ __all__ = [ import unittest -from urllib2 import HTTPError -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.config import config from mailman.database.transaction import transaction @@ -41,6 +38,8 @@ from mailman.testing.helpers import ( from mailman.runners.incoming import IncomingRunner from mailman.testing.layers import ConfigLayer, RESTLayer from mailman.utilities.datetime import now +from six.moves.urllib_error import HTTPError +from zope.component import getUtility diff --git a/src/mailman/rest/tests/test_moderation.py b/src/mailman/rest/tests/test_moderation.py index c0ec4755a..0e2528b0f 100644 --- a/src/mailman/rest/tests/test_moderation.py +++ b/src/mailman/rest/tests/test_moderation.py @@ -26,8 +26,6 @@ __all__ = [ import unittest -from urllib2 import HTTPError - from mailman.app.lifecycle import create_list from mailman.app.moderator import hold_message, hold_subscription from mailman.config import config @@ -36,6 +34,7 @@ from mailman.interfaces.member import DeliveryMode from mailman.testing.helpers import ( call_api, specialized_message_from_string as mfs) from mailman.testing.layers import RESTLayer +from six.moves.urllib_error import HTTPError diff --git a/src/mailman/rest/tests/test_preferences.py b/src/mailman/rest/tests/test_preferences.py index 91a066cff..06e0b035b 100644 --- a/src/mailman/rest/tests/test_preferences.py +++ b/src/mailman/rest/tests/test_preferences.py @@ -32,7 +32,7 @@ from mailman.database.transaction import transaction from mailman.interfaces.usermanager import IUserManager from mailman.testing.helpers import call_api from mailman.testing.layers import RESTLayer -from urllib2 import HTTPError +from six.moves.urllib_error import HTTPError from zope.component import getUtility diff --git a/src/mailman/rest/tests/test_root.py b/src/mailman/rest/tests/test_root.py index d4d25ede0..c5787bcb1 100644 --- a/src/mailman/rest/tests/test_root.py +++ b/src/mailman/rest/tests/test_root.py @@ -35,7 +35,7 @@ from mailman.config import config from mailman.core.system import system from mailman.testing.helpers import call_api from mailman.testing.layers import RESTLayer -from urllib2 import HTTPError +from six.moves.urllib_error import HTTPError diff --git a/src/mailman/rest/tests/test_users.py b/src/mailman/rest/tests/test_users.py index 10cc724a3..1936fdc68 100644 --- a/src/mailman/rest/tests/test_users.py +++ b/src/mailman/rest/tests/test_users.py @@ -30,15 +30,14 @@ __all__ = [ import os import unittest -from urllib2 import HTTPError -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.config import config from mailman.database.transaction import transaction from mailman.interfaces.usermanager import IUserManager from mailman.testing.helpers import call_api, configuration from mailman.testing.layers import RESTLayer +from six.moves.urllib_error import HTTPError +from zope.component import getUtility diff --git a/src/mailman/rest/users.py b/src/mailman/rest/users.py index cfea36cfa..7e6e70c5a 100644 --- a/src/mailman/rest/users.py +++ b/src/mailman/rest/users.py @@ -27,6 +27,8 @@ __all__ = [ ] +import six + from passlib.utils import generate_password as generate from uuid import UUID from zope.component import getUtility @@ -58,7 +60,7 @@ class PasswordEncrypterGetterSetter(GetterSetter): ATTRIBUTES = dict( - display_name=GetterSetter(unicode), + display_name=GetterSetter(six.text_type), cleartext_password=PasswordEncrypterGetterSetter(), ) @@ -105,9 +107,9 @@ class AllUsers(_UserBase): def on_post(self, request, response): """Create a new user.""" try: - validator = Validator(email=unicode, - display_name=unicode, - password=unicode, + validator = Validator(email=six.text_type, + display_name=six.text_type, + password=six.text_type, _optional=('display_name', 'password')) arguments = validator(request) except ValueError as error: @@ -253,7 +255,7 @@ class Login: # We do not want to encrypt the plaintext password given in the POST # data. That would hash the password, but we need to have the # plaintext in order to pass into passlib. - validator = Validator(cleartext_password=GetterSetter(unicode)) + validator = Validator(cleartext_password=GetterSetter(six.text_type)) try: values = validator(request) except ValueError as error: diff --git a/src/mailman/rest/validator.py b/src/mailman/rest/validator.py index cbcc5f652..74a8c0be4 100644 --- a/src/mailman/rest/validator.py +++ b/src/mailman/rest/validator.py @@ -62,7 +62,7 @@ def subscriber_validator(subscriber): try: return UUID(int=int(subscriber)) except ValueError: - return unicode(subscriber) + return subscriber.decode('utf-8') def language_validator(code): diff --git a/src/mailman/runners/command.py b/src/mailman/runners/command.py index 3d91f663a..54ab03687 100644 --- a/src/mailman/runners/command.py +++ b/src/mailman/runners/command.py @@ -31,9 +31,9 @@ __all__ = [ # -owner. import re +import six import logging -from StringIO import StringIO from email.errors import HeaderParseError from email.header import decode_header, make_header from email.iterators import typed_subpart_iterator @@ -76,7 +76,7 @@ class CommandFinder: # Extract the subject header and do RFC 2047 decoding. raw_subject = msg.get('subject', '') try: - subject = unicode(make_header(decode_header(raw_subject))) + subject = make_header(decode_header(raw_subject)).decode('utf-8') # Mail commands must be ASCII. self.command_lines.append(subject.encode('us-ascii')) except (HeaderParseError, UnicodeError, LookupError): @@ -84,7 +84,7 @@ class CommandFinder: # subject is a unicode object, convert it to ASCII ignoring all # bogus characters. Otherwise, there's nothing in the subject # that we can use. - if isinstance(raw_subject, unicode): + if isinstance(raw_subject, six.text_type): safe_subject = raw_subject.encode('us-ascii', 'ignore') self.command_lines.append(safe_subject) # Find the first text/plain part of the message. @@ -100,7 +100,7 @@ class CommandFinder: return body = part.get_payload(decode=True) # text/plain parts better have string payloads. - assert isinstance(body, basestring), 'Non-string decoded payload' + assert isinstance(body, six.string_types), 'Non-string decoded payload' lines = body.splitlines() # Use no more lines than specified max_lines = int(config.mailman.email_commands_max_lines) @@ -118,7 +118,7 @@ class CommandFinder: # Ensure that all the parts are unicodes. Since we only accept # ASCII commands and arguments, ignore anything else. parts = [(part - if isinstance(part, unicode) + if isinstance(part, six.text_type) else part.decode('ascii', 'ignore')) for part in parts] yield parts @@ -130,20 +130,20 @@ class Results: """The email command results.""" def __init__(self, charset='us-ascii'): - self._output = StringIO() + self._output = six.StringIO() self.charset = charset print(_("""\ The results of your email command are provided below. """), file=self._output) def write(self, text): - if not isinstance(text, unicode): + if isinstance(text, bytes): text = text.decode(self.charset, 'ignore') self._output.write(text) def __unicode__(self): value = self._output.getvalue() - assert isinstance(value, unicode), 'Not a unicode: %r' % value + assert isinstance(value, six.text_type), 'Not a unicode: %r' % value return value @@ -231,7 +231,7 @@ class CommandRunner(Runner): # Find a charset for the response body. Try the original message's # charset first, then ascii, then latin-1 and finally falling back to # utf-8. - reply_body = unicode(results) + reply_body = results.decode('utf-8') for charset in (results.charset, 'us-ascii', 'latin-1'): try: reply_body.encode(charset) diff --git a/src/mailman/runners/digest.py b/src/mailman/runners/digest.py index 98fe82f39..628a08e0c 100644 --- a/src/mailman/runners/digest.py +++ b/src/mailman/runners/digest.py @@ -28,8 +28,6 @@ __all__ = [ import re import logging -# cStringIO doesn't support unicode. -from StringIO import StringIO from copy import deepcopy from email.header import Header from email.message import Message @@ -37,8 +35,6 @@ from email.mime.message import MIMEMessage from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.utils import formatdate, getaddresses, make_msgid -from urllib2 import URLError - from mailman.config import config from mailman.core.i18n import _ from mailman.core.runner import Runner @@ -47,6 +43,8 @@ from mailman.interfaces.member import DeliveryMode, DeliveryStatus from mailman.utilities.i18n import make from mailman.utilities.mailbox import Mailbox from mailman.utilities.string import oneline, wrap +from six.moves import cStringIO as StringIO +from six.moves.urllib_error import URLError log = logging.getLogger('mailman.error') diff --git a/src/mailman/runners/nntp.py b/src/mailman/runners/nntp.py index 493f8d09a..7d9c189cc 100644 --- a/src/mailman/runners/nntp.py +++ b/src/mailman/runners/nntp.py @@ -31,11 +31,11 @@ import socket import logging import nntplib -from cStringIO import StringIO - from mailman.config import config from mailman.core.runner import Runner from mailman.interfaces.nntp import NewsgroupModeration +from six.moves import cStringIO as StringIO + COMMA = ',' COMMASPACE = ', ' diff --git a/src/mailman/runners/tests/test_nntp.py b/src/mailman/runners/tests/test_nntp.py index 3570d1a6f..191dd2657 100644 --- a/src/mailman/runners/tests/test_nntp.py +++ b/src/mailman/runners/tests/test_nntp.py @@ -304,7 +304,7 @@ Testing # and make some simple checks that the message is what we expected. conn_mock.quit.assert_called_once_with() - @mock.patch('nntplib.NNTP', side_effect=nntplib.error_temp) + @mock.patch('nntplib.NNTP', side_effect=nntplib.NNTPTemporaryError) def test_connect_with_nntplib_failure(self, class_mock): self._nntpq.enqueue(self._msg, {}, listname='test@example.com') mark = LogFileMark('mailman.error') @@ -341,7 +341,7 @@ Testing self.assertEqual(messages[0].msgdata['listname'], 'test@example.com') self.assertEqual(messages[0].msg['subject'], 'A newsgroup posting') - @mock.patch('nntplib.NNTP', side_effect=nntplib.error_temp) + @mock.patch('nntplib.NNTP', side_effect=nntplib.NNTPTemporaryError) def test_connection_never_gets_quit_after_failures(self, class_mock): # The NNTP connection doesn't get closed after a unsuccessful # connection, since there's nothing to close. @@ -361,7 +361,7 @@ Testing # The NNTP connection does get closed after a unsuccessful post. # Add a side-effect to the instance mock's .post() method. conn_mock = class_mock() - conn_mock.post.side_effect = nntplib.error_temp + conn_mock.post.side_effect = nntplib.NNTPTemporaryError self._nntpq.enqueue(self._msg, {}, listname='test@example.com') self._runner.run() # The connection object's post() method was called once with a diff --git a/src/mailman/testing/helpers.py b/src/mailman/testing/helpers.py index b0fe14a0d..1de0e98cf 100644 --- a/src/mailman/testing/helpers.py +++ b/src/mailman/testing/helpers.py @@ -59,8 +59,8 @@ from contextlib import contextmanager from email import message_from_string from httplib2 import Http from lazr.config import as_timedelta -from urllib import urlencode -from urllib2 import HTTPError +from six.moves.urllib_error import HTTPError +from six.moves.urllib_parse import urlencode from zope import event from zope.component import getUtility @@ -342,7 +342,7 @@ def call_api(url, data=None, method=None, username=None, password=None): if len(content) == 0: return None, response # XXX Workaround http://bugs.python.org/issue10038 - content = unicode(content) + content = content.decode('utf-8') return json.loads(content), response @@ -506,9 +506,8 @@ def specialized_message_from_string(unicode_text): """ # This mimic what Switchboard.dequeue() does when parsing a message from # text into a Message instance. - text = unicode_text.encode('ascii') - original_size = len(text) - message = message_from_string(text, Message) + original_size = len(unicode_text) + message = message_from_string(unicode_text, Message) message.original_size = original_size return message diff --git a/src/mailman/testing/layers.py b/src/mailman/testing/layers.py index 74ad99dc8..8ec6c307f 100644 --- a/src/mailman/testing/layers.py +++ b/src/mailman/testing/layers.py @@ -46,7 +46,7 @@ import datetime import tempfile from lazr.config import as_boolean -from pkg_resources import resource_string +from pkg_resources import resource_string as resource_bytes from textwrap import dedent from zope.component import getUtility @@ -132,7 +132,8 @@ class ConfigLayer(MockAndMonkeyLayer): configuration: {1} """.format(cls.var_dir, postfix_cfg)) # Read the testing config and push it. - test_config += resource_string('mailman.testing', 'testing.cfg') + more = resource_bytes('mailman.testing', 'testing.cfg') + test_config += more.decode('utf-8') config.create_paths = True config.push('test config', test_config) # Initialize everything else. diff --git a/src/mailman/testing/mta.py b/src/mailman/testing/mta.py index 875647485..234540e98 100644 --- a/src/mailman/testing/mta.py +++ b/src/mailman/testing/mta.py @@ -27,10 +27,9 @@ __all__ = [ import logging -from Queue import Empty, Queue - from lazr.smtptest.controller import QueueController from lazr.smtptest.server import Channel, QueueServer +from six.moves.queue import Empty, Queue from zope.interface import implementer from mailman.interfaces.mta import IMailTransportAgentLifecycle diff --git a/src/mailman/utilities/email.py b/src/mailman/utilities/email.py index 7025ddb89..a15c86fb5 100644 --- a/src/mailman/utilities/email.py +++ b/src/mailman/utilities/email.py @@ -68,7 +68,7 @@ def add_message_hash(msg): message_id = message_id[1:-1] else: message_id = message_id.strip() - digest = sha1(message_id).digest() + digest = sha1(message_id.encode('utf-8')).digest() message_id_hash = b32encode(digest) del msg['x-message-id-hash'] msg['X-Message-ID-Hash'] = message_id_hash diff --git a/src/mailman/utilities/i18n.py b/src/mailman/utilities/i18n.py index e22bd6c18..e9136837f 100644 --- a/src/mailman/utilities/i18n.py +++ b/src/mailman/utilities/i18n.py @@ -29,6 +29,7 @@ __all__ = [ import os +import six import sys import errno @@ -203,7 +204,8 @@ def make(template_file, mlist=None, language=None, wrap=True, template = _(fp.read()[:-1]) finally: fp.close() - assert isinstance(template, unicode), 'Translated template is not unicode' + assert isinstance(template, six.text_type), ( + 'Translated template is not unicode') text = expand(template, kw) if wrap: return wrap_text(text) diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py index cc8a0cf44..5cc52fef7 100644 --- a/src/mailman/utilities/importer.py +++ b/src/mailman/utilities/importer.py @@ -48,7 +48,7 @@ from mailman.interfaces.nntp import NewsgroupModeration from mailman.interfaces.usermanager import IUserManager from mailman.utilities.filesystem import makedirs from mailman.utilities.i18n import search -from urllib2 import URLError +from six.moves.urllib_error import URLError from zope.component import getUtility diff --git a/src/mailman/utilities/tests/test_import.py b/src/mailman/utilities/tests/test_import.py index 42608ae45..de40a3ca1 100644 --- a/src/mailman/utilities/tests/test_import.py +++ b/src/mailman/utilities/tests/test_import.py @@ -27,14 +27,14 @@ __all__ = [ import os +import six import mock -import cPickle import unittest from datetime import timedelta, datetime from enum import Enum from pkg_resources import resource_filename -from sqlalchemy.exc import IntegrityError +from six.moves.cPickle import load from zope.component import getUtility from mailman.app.lifecycle import create_list @@ -78,7 +78,7 @@ class TestBasicImport(unittest.TestCase): self._mlist = create_list('blank@example.com') pickle_file = resource_filename('mailman.testing', 'config.pck') with open(pickle_file) as fp: - self._pckdict = cPickle.load(fp) + self._pckdict = load(fp) def _import(self): import_config_pck(self._mlist, self._pckdict) @@ -188,7 +188,7 @@ class TestBasicImport(unittest.TestCase): # moderator_password must not be unicode self._pckdict[b'mod_password'] = b'TESTVALUE' self._import() - self.assertFalse(isinstance(self._mlist.moderator_password, unicode)) + self.assertNotIsInstance(self._mlist.moderator_password, six.text_type) self.assertEqual(self._mlist.moderator_password, b'TESTVALUE') def test_newsgroup_moderation(self): @@ -263,9 +263,10 @@ class TestBasicImport(unittest.TestCase): # Suppress warning messages in test output. with mock.patch('sys.stderr'): self._import() - self.assertEqual(self._mlist.info, - unicode(self._pckdict[b'info'], 'ascii', 'replace'), - "We don't fall back to replacing non-ascii chars") + self.assertEqual( + self._mlist.info, + self._pckdict[b'info'].decode('ascii', 'replace'), + "We don't fall back to replacing non-ascii chars") def test_preferred_language(self): self._pckdict[b'preferred_language'] = b'ja' -- cgit v1.2.3-70-g09d2