diff options
375 files changed, 2709 insertions, 2960 deletions
diff --git a/.bzrignore b/.bzrignore index b348eb336..b687d7d83 100644 --- a/.bzrignore +++ b/.bzrignore @@ -22,3 +22,4 @@ distribute-*.tar.gz .coverage htmlcov .tox +__pycache__ diff --git a/coverage.ini b/coverage.ini index ec0e846c7..58861564d 100644 --- a/coverage.ini +++ b/coverage.ini @@ -4,7 +4,7 @@ parallel = true omit = setup* */showme.py - .tox/coverage/lib/python2.7/site-packages/* + .tox/coverage/lib/python3.4/site-packages/* [paths] source = @@ -15,17 +15,15 @@ # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see <http://www.gnu.org/licenses/>. -# Do *not* import unicode_literals. This breaks setuptools. -from __future__ import absolute_import, print_function - import re import sys from setuptools import setup, find_packages from string import Template -if sys.hexversion < 0x20700f0: - print('Mailman requires at least Python 2.7') + +if sys.hexversion < 0x30400f0: + print('Mailman requires at least Python 3.4') sys.exit(1) @@ -105,6 +103,7 @@ case second `m'. Any other spelling is incorrect.""", 'mock', 'nose2', 'passlib', + 'six', 'sqlalchemy', 'zope.component', 'zope.configuration', diff --git a/src/mailman/__init__.py b/src/mailman/__init__.py index db7befab7..74040d211 100644 --- a/src/mailman/__init__.py +++ b/src/mailman/__init__.py @@ -17,13 +17,6 @@ """The `mailman` package.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type -__all__ = [ - ] - - import sys diff --git a/src/mailman/app/bounces.py b/src/mailman/app/bounces.py index b0a316ad6..ebfe63cff 100644 --- a/src/mailman/app/bounces.py +++ b/src/mailman/app/bounces.py @@ -17,9 +17,6 @@ """Application level bounce handling.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ProbeVERP', 'StandardVERP', @@ -36,10 +33,6 @@ import logging from email.mime.message import MIMEMessage from email.mime.text import MIMEText from email.utils import parseaddr -from string import Template -from zope.component import getUtility -from zope.interface import implementer - from mailman.config import config from mailman.core.i18n import _ from mailman.email.message import OwnerNotification, UserNotification @@ -50,6 +43,10 @@ from mailman.interfaces.subscriptions import ISubscriptionService from mailman.utilities.email import split_email from mailman.utilities.i18n import make from mailman.utilities.string import oneline +from string import Template +from zope.component import getUtility +from zope.interface import implementer + log = logging.getLogger('mailman.config') elog = logging.getLogger('mailman.error') @@ -71,8 +68,8 @@ def bounce_message(mlist, msg, error=None): :type error: Exception """ # Bounce a message back to the sender, with an error message if provided - # in the exception argument. - if msg.sender is None: + # in the exception argument. .sender might be None or the empty string. + if not msg.sender: # We can't bounce the message if we don't know who it's supposed to go # to. return diff --git a/src/mailman/app/commands.py b/src/mailman/app/commands.py index a0f717138..cfa672de5 100644 --- a/src/mailman/app/commands.py +++ b/src/mailman/app/commands.py @@ -17,19 +17,15 @@ """Initialize the email commands.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'initialize', ] -from zope.interface.verify import verifyObject - from mailman.config import config from mailman.interfaces.command import IEmailCommand from mailman.utilities.modules import find_components +from zope.interface.verify import verifyObject diff --git a/src/mailman/app/docs/hooks.rst b/src/mailman/app/docs/hooks.rst index eb6cbac05..ba9bb249e 100644 --- a/src/mailman/app/docs/hooks.rst +++ b/src/mailman/app/docs/hooks.rst @@ -18,12 +18,12 @@ Hooks name an importable callable so it must be accessible on ``sys.path``. ... counter = 1 ... def pre_hook(): ... global counter - ... print 'pre-hook:', counter + ... print('pre-hook:', counter) ... counter += 1 ... ... def post_hook(): ... global counter - ... print 'post-hook:', counter + ... print('post-hook:', counter) ... counter += 1 ... """, file=fp) >>> fp.close() @@ -61,6 +61,7 @@ script that will produce no output to force the hooks to run. ... proc = subprocess.Popen( ... [exe, 'lists', '--domain', 'ignore', '-q'], ... cwd=ConfigLayer.root_directory, env=env, + ... universal_newlines=True, ... stdout=subprocess.PIPE, stderr=subprocess.PIPE) ... stdout, stderr = proc.communicate() ... assert proc.returncode == 0, stderr diff --git a/src/mailman/app/docs/pipelines.rst b/src/mailman/app/docs/pipelines.rst index adcdd1ea5..dfdc6d70c 100644 --- a/src/mailman/app/docs/pipelines.rst +++ b/src/mailman/app/docs/pipelines.rst @@ -45,9 +45,9 @@ etc. To: test@example.com Message-ID: <first> X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB - Subject: [Test] My first post X-Mailman-Version: ... Precedence: list + Subject: [Test] My first post List-Id: <test.example.com> Archived-At: http://lists.example.com/.../4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB List-Archive: <http://lists.example.com/archives/test@example.com> @@ -67,7 +67,7 @@ However there are currently no recipients for this message. >>> dump_msgdata(msgdata) original_sender : aperson@example.com original_subject: My first post - recipients : set([]) + recipients : set() stripped_subject: My first post After pipeline processing, the message is now sitting in various other @@ -84,9 +84,9 @@ processing queues. To: test@example.com Message-ID: <first> X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB - Subject: [Test] My first post X-Mailman-Version: ... Precedence: list + Subject: [Test] My first post List-Id: <test.example.com> ... <BLANKLINE> @@ -97,7 +97,7 @@ processing queues. _parsemsg : False original_sender : aperson@example.com original_subject: My first post - recipients : set([]) + recipients : set() stripped_subject: My first post version : 3 @@ -121,9 +121,9 @@ delivered to end recipients. To: test@example.com Message-ID: <first> X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB - Subject: [Test] My first post X-Mailman-Version: ... Precedence: list + Subject: [Test] My first post List-Id: <test.example.com> ... <BLANKLINE> @@ -132,10 +132,10 @@ delivered to end recipients. >>> dump_msgdata(messages[0].msgdata) _parsemsg : False - listname : test@example.com + listid : test.example.com original_sender : aperson@example.com original_subject: My first post - recipients : set([]) + recipients : set() stripped_subject: My first post version : 3 @@ -152,9 +152,9 @@ There's now one message in the digest mailbox, getting ready to be sent. To: test@example.com Message-ID: <first> X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB - Subject: [Test] My first post X-Mailman-Version: ... Precedence: list + Subject: [Test] My first post List-Id: <test.example.com> ... <BLANKLINE> diff --git a/src/mailman/app/docs/subscriptions.rst b/src/mailman/app/docs/subscriptions.rst index 8c3d8b28d..eaccdc3cc 100644 --- a/src/mailman/app/docs/subscriptions.rst +++ b/src/mailman/app/docs/subscriptions.rst @@ -67,13 +67,6 @@ New members can also be added by providing an existing user id instead of an email address. However, the user must have a preferred email address. :: - >>> service.join('test.example.com', bart.user.user_id, - ... role=MemberRole.owner) - Traceback (most recent call last): - ... - MissingPreferredAddressError: User must have a preferred address: - <User "Bart Person" (2) at ...> - >>> from mailman.utilities.datetime import now >>> address = list(bart.user.addresses)[0] >>> address.verified_on = now() diff --git a/src/mailman/app/domain.py b/src/mailman/app/domain.py index 7ad976699..a8a2cd71a 100644 --- a/src/mailman/app/domain.py +++ b/src/mailman/app/domain.py @@ -17,18 +17,14 @@ """Application level domain support.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'handle_DomainDeletingEvent', ] -from zope.component import getUtility - from mailman.interfaces.domain import DomainDeletingEvent from mailman.interfaces.listmanager import IListManager +from zope.component import getUtility diff --git a/src/mailman/app/events.py b/src/mailman/app/events.py index 16817c202..0b7f2309e 100644 --- a/src/mailman/app/events.py +++ b/src/mailman/app/events.py @@ -17,22 +17,18 @@ """Global events.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'initialize', ] -from zope import event - from mailman.app import ( domain, membership, moderator, registrar, subscriptions) from mailman.core import i18n, switchboard from mailman.languages import manager as language_manager from mailman.styles import manager as style_manager from mailman.utilities import passwords +from zope import event diff --git a/src/mailman/app/inject.py b/src/mailman/app/inject.py index 4c182657d..7e8c359ea 100644 --- a/src/mailman/app/inject.py +++ b/src/mailman/app/inject.py @@ -17,9 +17,6 @@ """Inject a message into a queue.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'inject_message', 'inject_text', @@ -28,7 +25,6 @@ __all__ = [ from email import message_from_string from email.utils import formatdate, make_msgid - from mailman.config import config from mailman.email.message import Message from mailman.utilities.email import add_message_hash @@ -53,6 +49,8 @@ def inject_message(mlist, msg, recipients=None, switchboard=None, **kws): :type switchboard: string :param kws: Additional values for the message metadata. :type kws: dictionary + :return: filebase of enqueued message + :rtype: string """ if switchboard is None: switchboard = 'in' @@ -66,13 +64,13 @@ def inject_message(mlist, msg, recipients=None, switchboard=None, **kws): msg['Date'] = formatdate(localtime=True) msg.original_size = len(msg.as_string()) msgdata = dict( - listname=mlist.fqdn_listname, + listid=mlist.list_id, original_size=msg.original_size, ) msgdata.update(kws) if recipients is not None: msgdata['recipients'] = recipients - config.switchboards[switchboard].enqueue(msg, **msgdata) + return config.switchboards[switchboard].enqueue(msg, **msgdata) @@ -95,6 +93,8 @@ def inject_text(mlist, text, recipients=None, switchboard=None, **kws): :type switchboard: string :param kws: Additional values for the message metadata. :type kws: dictionary + :return: filebase of enqueued message + :rtype: string """ message = message_from_string(text, Message) - inject_message(mlist, message, recipients, switchboard, **kws) + return inject_message(mlist, message, recipients, switchboard, **kws) diff --git a/src/mailman/app/lifecycle.py b/src/mailman/app/lifecycle.py index 8110fe69d..bef8320d0 100644 --- a/src/mailman/app/lifecycle.py +++ b/src/mailman/app/lifecycle.py @@ -17,9 +17,6 @@ """Application level list creation.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'create_list', 'remove_list', @@ -31,8 +28,6 @@ import errno import shutil import logging -from zope.component import getUtility - from mailman.config import config from mailman.interfaces.address import IEmailValidator from mailman.interfaces.domain import ( @@ -42,6 +37,7 @@ from mailman.interfaces.member import MemberRole from mailman.interfaces.styles import IStyleManager from mailman.interfaces.usermanager import IUserManager from mailman.utilities.modules import call_name +from zope.component import getUtility log = logging.getLogger('mailman.error') diff --git a/src/mailman/app/membership.py b/src/mailman/app/membership.py index 4ec6b7878..0a6c8b971 100644 --- a/src/mailman/app/membership.py +++ b/src/mailman/app/membership.py @@ -17,9 +17,6 @@ """Application support for membership management.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'add_member', 'delete_member', @@ -28,8 +25,6 @@ __all__ = [ from email.utils import formataddr -from zope.component import getUtility - from mailman.app.notifications import ( send_goodbye_message, send_welcome_message) from mailman.config import config @@ -40,6 +35,7 @@ from mailman.interfaces.member import ( MemberRole, MembershipIsBannedError, NotAMemberError, SubscriptionEvent) from mailman.interfaces.usermanager import IUserManager from mailman.utilities.i18n import make +from zope.component import getUtility diff --git a/src/mailman/app/moderator.py b/src/mailman/app/moderator.py index 105e53617..d4c5b1036 100644 --- a/src/mailman/app/moderator.py +++ b/src/mailman/app/moderator.py @@ -17,9 +17,6 @@ """Application support for moderators.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'handle_ListDeletingEvent', 'handle_message', @@ -35,8 +32,6 @@ import time import logging from email.utils import formataddr, formatdate, getaddresses, make_msgid -from zope.component import getUtility - from mailman.app.membership import add_member, delete_member from mailman.app.notifications import send_admin_subscription_notice from mailman.config import config @@ -51,6 +46,7 @@ from mailman.interfaces.messages import IMessageStore from mailman.interfaces.requests import IListRequests, RequestType from mailman.utilities.datetime import now from mailman.utilities.i18n import make +from zope.component import getUtility NL = '\n' @@ -86,14 +82,14 @@ def hold_message(mlist, msg, msgdata=None, reason=None): # Message-ID header. message_id = msg.get('message-id') if message_id is None: - msg['Message-ID'] = message_id = make_msgid().decode('ascii') + msg['Message-ID'] = message_id = make_msgid() elif isinstance(message_id, bytes): message_id = message_id.decode('ascii') getUtility(IMessageStore).add(msg) # Prepare the message metadata with some extra information needed only by # the moderation interface. msgdata['_mod_message_id'] = message_id - msgdata['_mod_fqdn_listname'] = mlist.fqdn_listname + msgdata['_mod_listid'] = mlist.list_id msgdata['_mod_sender'] = msg.sender msgdata['_mod_subject'] = msg.get('subject', _('(no subject)')) msgdata['_mod_reason'] = reason @@ -134,7 +130,7 @@ def handle_message(mlist, id, action, # Start by getting the message from the message store. msg = message_store.get_message_by_id(message_id) # Delete moderation-specific entries from the message metadata. - for key in msgdata.keys(): + for key in list(msgdata): if key.startswith('_mod_'): del msgdata[key] # Add some metadata to indicate this message has now been approved. diff --git a/src/mailman/app/notifications.py b/src/mailman/app/notifications.py index 1fa1fe01e..163b02653 100644 --- a/src/mailman/app/notifications.py +++ b/src/mailman/app/notifications.py @@ -17,9 +17,6 @@ """Sending notifications.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'send_admin_subscription_notice', 'send_goodbye_message', @@ -31,9 +28,6 @@ import logging from email.utils import formataddr from lazr.config import as_boolean -from urllib2 import URLError -from zope.component import getUtility - from mailman.config import config from mailman.core.i18n import _ from mailman.email.message import OwnerNotification, UserNotification @@ -41,6 +35,8 @@ from mailman.interfaces.member import DeliveryMode from mailman.interfaces.templates import ITemplateLoader from mailman.utilities.i18n import make from mailman.utilities.string import expand, wrap +from six.moves.urllib_error import URLError +from zope.component import getUtility log = logging.getLogger('mailman.error') @@ -141,7 +137,6 @@ def send_admin_subscription_notice(mlist, address, display_name, language): """ with _.using(mlist.preferred_language.code): subject = _('$mlist.display_name subscription notification') - display_name = display_name.encode(language.charset, 'replace') text = make('adminsubscribeack.txt', mailing_list=mlist, listname=mlist.display_name, diff --git a/src/mailman/app/registrar.py b/src/mailman/app/registrar.py index aa4e35483..fd84f7aa0 100644 --- a/src/mailman/app/registrar.py +++ b/src/mailman/app/registrar.py @@ -17,9 +17,6 @@ """Implementation of the IUserRegistrar interface.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Registrar', 'handle_ConfirmationNeededEvent', @@ -28,10 +25,6 @@ __all__ = [ import logging -from zope.component import getUtility -from zope.event import notify -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.email.message import UserNotification from mailman.interfaces.address import IEmailValidator @@ -42,6 +35,9 @@ from mailman.interfaces.registrar import ConfirmationNeededEvent, IRegistrar from mailman.interfaces.templates import ITemplateLoader from mailman.interfaces.usermanager import IUserManager from mailman.utilities.datetime import now +from zope.component import getUtility +from zope.event import notify +from zope.interface import implementer log = logging.getLogger('mailman.error') diff --git a/src/mailman/app/replybot.py b/src/mailman/app/replybot.py index 4ade73faf..ca563ea0a 100644 --- a/src/mailman/app/replybot.py +++ b/src/mailman/app/replybot.py @@ -21,9 +21,6 @@ # mailing list. The reply governor should really apply site-wide per # recipient (I think). -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'can_acknowledge', ] diff --git a/src/mailman/app/subscriptions.py b/src/mailman/app/subscriptions.py index 99c6ab2de..e3239e97e 100644 --- a/src/mailman/app/subscriptions.py +++ b/src/mailman/app/subscriptions.py @@ -15,11 +15,8 @@ # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see <http://www.gnu.org/licenses/>. -"""Module stuff.""" +"""Handle subscriptions.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'SubscriptionService', 'handle_ListDeletingEvent', @@ -108,7 +105,7 @@ class SubscriptionService: # the parameter can either be an email address or a user id. query = [] if subscriber is not None: - if isinstance(subscriber, basestring): + if isinstance(subscriber, str): # subscriber is an email address. address = user_manager.get_address(subscriber) user = user_manager.get_user(subscriber) @@ -148,7 +145,7 @@ class SubscriptionService: if mlist is None: raise NoSuchListError(list_id) # Is the subscriber an email address or user id? - if isinstance(subscriber, basestring): + if isinstance(subscriber, str): if display_name is None: display_name, at, domain = subscriber.partition('@') # Because we want to keep the REST API simple, there is no diff --git a/src/mailman/app/templates.py b/src/mailman/app/templates.py index 742584b49..a5f9fc1b5 100644 --- a/src/mailman/app/templates.py +++ b/src/mailman/app/templates.py @@ -17,30 +17,27 @@ """Template loader.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TemplateLoader', ] -import urllib2 - from contextlib import closing -from urllib import addinfourl -from urlparse import urlparse -from zope.component import getUtility -from zope.interface import implementer - -from mailman.utilities.i18n import TemplateNotFoundError, find from mailman.interfaces.languages import ILanguageManager from mailman.interfaces.listmanager import IListManager from mailman.interfaces.templates import ITemplateLoader +from mailman.utilities.i18n import TemplateNotFoundError, find +from six.moves.urllib_error import URLError +from six.moves.urllib_parse import urlparse +from six.moves.urllib_request import ( + BaseHandler, build_opener, install_opener, urlopen) +from six.moves.urllib_response import addinfourl +from zope.component import getUtility +from zope.interface import implementer -class MailmanHandler(urllib2.BaseHandler): +class MailmanHandler(BaseHandler): # Handle internal mailman: URLs. def mailman_open(self, req): # Parse urls of the form: @@ -55,9 +52,9 @@ class MailmanHandler(urllib2.BaseHandler): assert parsed.scheme == 'mailman' # The path can contain one, two, or three components. Since no empty # path components are legal, filter them out. - parts = filter(None, parsed.path.split('/')) + parts = [p for p in parsed.path.split('/') if p] if len(parts) == 0: - raise urllib2.URLError('No template specified') + raise URLError('No template specified') elif len(parts) == 1: template = parts[0] elif len(parts) == 2: @@ -69,25 +66,25 @@ class MailmanHandler(urllib2.BaseHandler): language = getUtility(ILanguageManager).get(part0) mlist = getUtility(IListManager).get(part0) if language is None and mlist is None: - raise urllib2.URLError('Bad language or list name') + raise URLError('Bad language or list name') elif mlist is None: code = language.code elif len(parts) == 3: fqdn_listname, code, template = parts mlist = getUtility(IListManager).get(fqdn_listname) if mlist is None: - raise urllib2.URLError('Missing list') + raise URLError('Missing list') language = getUtility(ILanguageManager).get(code) if language is None: - raise urllib2.URLError('No such language') + raise URLError('No such language') code = language.code else: - raise urllib2.URLError('No such file') + raise URLError('No such file') # Find the template, mutating any missing template exception. try: path, fp = find(template, mlist, code) except TemplateNotFoundError: - raise urllib2.URLError('No such file') + raise URLError('No such file') return addinfourl(fp, {}, original_url) @@ -97,10 +94,10 @@ class TemplateLoader: """Loader of templates, with caching and support for mailman:// URIs.""" def __init__(self): - opener = urllib2.build_opener(MailmanHandler()) - urllib2.install_opener(opener) + opener = build_opener(MailmanHandler()) + install_opener(opener) def get(self, uri): """See `ITemplateLoader`.""" - with closing(urllib2.urlopen(uri)) as fp: - return fp.read().decode('utf-8') + with closing(urlopen(uri)) as fp: + return fp.read() diff --git a/src/mailman/app/tests/test_bounces.py b/src/mailman/app/tests/test_bounces.py index 5eb518786..b89664209 100644 --- a/src/mailman/app/tests/test_bounces.py +++ b/src/mailman/app/tests/test_bounces.py @@ -17,9 +17,6 @@ """Testing app.bounces functions.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestBounceMessage', 'TestMaybeForward', @@ -36,8 +33,6 @@ import shutil import tempfile import unittest -from zope.component import getUtility - from mailman.app.bounces import ( ProbeVERP, StandardVERP, bounce_message, maybe_forward, send_probe) from mailman.app.lifecycle import create_list @@ -49,10 +44,9 @@ from mailman.interfaces.member import DeliveryMode, MemberRole from mailman.interfaces.pending import IPendings from mailman.interfaces.usermanager import IUserManager from mailman.testing.helpers import ( - LogFileMark, - get_queue_messages, - specialized_message_from_string as mfs) + LogFileMark, get_queue_messages, specialized_message_from_string as mfs) from mailman.testing.layers import ConfigLayer +from zope.component import getUtility @@ -334,7 +328,7 @@ $owneraddr send_probe(self._member, self._msg) message = get_queue_messages('virgin')[0].msg self.assertEqual( - message['Subject'], + message['subject'].encode(), '=?utf-8?q?ailing-may_ist-lay_Test_obe-pray_essage-may?=') def test_probe_notice_with_member_nonenglish(self): @@ -533,7 +527,7 @@ Subject: Ignore def test_no_sender(self): # The message won't be bounced if it has no discernible sender. - self._msg.sender = None + del self._msg['from'] bounce_message(self._mlist, self._msg) items = get_queue_messages('virgin') # Nothing in the virgin queue means nothing's been bounced. diff --git a/src/mailman/app/tests/test_inject.py b/src/mailman/app/tests/test_inject.py index f7f750662..196c32182 100644 --- a/src/mailman/app/tests/test_inject.py +++ b/src/mailman/app/tests/test_inject.py @@ -17,10 +17,9 @@ """Testing app.inject functions.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ + 'TestInjectMessage', + 'TestInjectText', ] @@ -64,7 +63,7 @@ Nothing. self.assertEqual(len(items), 1) self.assertMultiLineEqual(items[0].msg.as_string(), self.msg.as_string()) - self.assertEqual(items[0].msgdata['listname'], 'test@example.com') + self.assertEqual(items[0].msgdata['listid'], 'test.example.com') self.assertEqual(items[0].msgdata['original_size'], len(self.msg.as_string())) @@ -84,7 +83,7 @@ Nothing. self.assertEqual(len(items), 1) self.assertMultiLineEqual(items[0].msg.as_string(), self.msg.as_string()) - self.assertEqual(items[0].msgdata['listname'], 'test@example.com') + self.assertEqual(items[0].msgdata['listid'], 'test.example.com') self.assertEqual(items[0].msgdata['original_size'], len(self.msg.as_string())) @@ -144,7 +143,7 @@ class TestInjectText(unittest.TestCase): def setUp(self): self.mlist = create_list('test@example.com') - self.text = b"""\ + self.text = """\ From: bart@example.com To: test@example.com Subject: A test message @@ -171,7 +170,7 @@ Nothing. # Delete that header because it is not in the original text. del items[0].msg['x-message-id-hash'] self.assertMultiLineEqual(items[0].msg.as_string(), self.text) - self.assertEqual(items[0].msgdata['listname'], 'test@example.com') + self.assertEqual(items[0].msgdata['listid'], 'test.example.com') self.assertEqual(items[0].msgdata['original_size'], # Add back the X-Message-ID-Header which was in the # message contributing to the original_size, but @@ -196,7 +195,7 @@ Nothing. # Remove the X-Message-ID-Hash header which isn't in the original text. del items[0].msg['x-message-id-hash'] self.assertMultiLineEqual(items[0].msg.as_string(), self.text) - self.assertEqual(items[0].msgdata['listname'], 'test@example.com') + self.assertEqual(items[0].msgdata['listid'], 'test.example.com') self.assertEqual(items[0].msgdata['original_size'], # Add back the X-Message-ID-Header which was in the # message contributing to the original_size, but diff --git a/src/mailman/app/tests/test_lifecycle.py b/src/mailman/app/tests/test_lifecycle.py index 0fb54f193..75386b870 100644 --- a/src/mailman/app/tests/test_lifecycle.py +++ b/src/mailman/app/tests/test_lifecycle.py @@ -17,9 +17,6 @@ """Test the high level list lifecycle API.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestLifecycle', ] diff --git a/src/mailman/app/tests/test_membership.py b/src/mailman/app/tests/test_membership.py index 95e8de1d0..5b2caf103 100644 --- a/src/mailman/app/tests/test_membership.py +++ b/src/mailman/app/tests/test_membership.py @@ -17,9 +17,6 @@ """Tests of application level membership functions.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestAddMember', 'TestAddMemberPassword', @@ -29,8 +26,6 @@ __all__ = [ import unittest -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.app.membership import add_member, delete_member from mailman.core.constants import system_preferences @@ -40,6 +35,7 @@ from mailman.interfaces.member import ( NotAMemberError) from mailman.interfaces.usermanager import IUserManager from mailman.testing.layers import ConfigLayer +from zope.component import getUtility diff --git a/src/mailman/app/tests/test_moderation.py b/src/mailman/app/tests/test_moderation.py index edb6b8c28..190b670d8 100644 --- a/src/mailman/app/tests/test_moderation.py +++ b/src/mailman/app/tests/test_moderation.py @@ -17,9 +17,6 @@ """Moderation tests.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestModeration', ] @@ -27,8 +24,6 @@ __all__ = [ import unittest -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.app.moderator import handle_message, hold_message from mailman.interfaces.action import Action @@ -41,6 +36,7 @@ from mailman.testing.helpers import ( get_queue_messages, make_testable_runner, specialized_message_from_string) from mailman.testing.layers import SMTPLayer from mailman.utilities.datetime import now +from zope.component import getUtility diff --git a/src/mailman/app/tests/test_notifications.py b/src/mailman/app/tests/test_notifications.py index 4cdc1c01c..fda4aaa0b 100644 --- a/src/mailman/app/tests/test_notifications.py +++ b/src/mailman/app/tests/test_notifications.py @@ -17,10 +17,8 @@ """Test notifications.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ + 'TestNotifications', ] @@ -29,8 +27,6 @@ import shutil import tempfile import unittest -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.app.membership import add_member from mailman.config import config @@ -38,6 +34,7 @@ from mailman.interfaces.languages import ILanguageManager from mailman.interfaces.member import DeliveryMode, MemberRole from mailman.testing.helpers import get_queue_messages from mailman.testing.layers import ConfigLayer +from zope.component import getUtility diff --git a/src/mailman/app/tests/test_registration.py b/src/mailman/app/tests/test_registration.py index ff128ae6f..fa34005c8 100644 --- a/src/mailman/app/tests/test_registration.py +++ b/src/mailman/app/tests/test_registration.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 by the Free Software Foundation, Inc. +# Copyright (C) 2012-2014 by the Free Software Foundation, Inc. # # This file is part of GNU Mailman. # @@ -17,9 +17,6 @@ """Test email address registration.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestEmailValidation', 'TestRegistration', @@ -28,14 +25,13 @@ __all__ = [ import unittest -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.interfaces.address import InvalidEmailAddressError from mailman.interfaces.pending import IPendings from mailman.interfaces.registrar import ConfirmationNeededEvent, IRegistrar from mailman.testing.helpers import event_subscribers from mailman.testing.layers import ConfigLayer +from zope.component import getUtility diff --git a/src/mailman/app/tests/test_subscriptions.py b/src/mailman/app/tests/test_subscriptions.py index e5aad18bc..1ba3cc24b 100644 --- a/src/mailman/app/tests/test_subscriptions.py +++ b/src/mailman/app/tests/test_subscriptions.py @@ -17,9 +17,6 @@ """Tests for the subscription service.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestJoin' ] @@ -28,13 +25,13 @@ __all__ = [ import uuid import unittest -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.interfaces.address import InvalidEmailAddressError +from mailman.interfaces.member import MemberRole, MissingPreferredAddressError from mailman.interfaces.subscriptions import ( MissingUserError, ISubscriptionService) from mailman.testing.layers import ConfigLayer +from zope.component import getUtility @@ -57,3 +54,14 @@ class TestJoin(unittest.TestCase): with self.assertRaises(InvalidEmailAddressError) as cm: self._service.join('test.example.com', 'bogus') self.assertEqual(cm.exception.email, 'bogus') + + def test_missing_preferred_address(self): + # A user cannot join a mailing list if they have no preferred address. + anne = self._service.join( + 'test.example.com', 'anne@example.com', 'Anne Person') + # Try to join Anne as a user with a different role. Her user has no + # preferred address, so this will fail. + self.assertRaises(MissingPreferredAddressError, + self._service.join, + 'test.example.com', anne.user.user_id, + role=MemberRole.owner) diff --git a/src/mailman/app/tests/test_templates.py b/src/mailman/app/tests/test_templates.py index afde68647..68bab9f49 100644 --- a/src/mailman/app/tests/test_templates.py +++ b/src/mailman/app/tests/test_templates.py @@ -17,27 +17,24 @@ """Test the template downloader API.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestTemplateLoader', ] import os +import six import shutil -import urllib2 import tempfile import unittest -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.config import config from mailman.interfaces.languages import ILanguageManager from mailman.interfaces.templates import ITemplateLoader from mailman.testing.layers import ConfigLayer +from six.moves.urllib_error import URLError +from zope.component import getUtility @@ -98,32 +95,32 @@ class TestTemplateLoader(unittest.TestCase): self.assertEqual(content, 'Test content') def test_uri_not_found(self): - with self.assertRaises(urllib2.URLError) as cm: + with self.assertRaises(URLError) as cm: self._loader.get('mailman:///missing.txt') self.assertEqual(cm.exception.reason, 'No such file') def test_shorter_url_error(self): - with self.assertRaises(urllib2.URLError) as cm: + with self.assertRaises(URLError) as cm: self._loader.get('mailman:///') self.assertEqual(cm.exception.reason, 'No template specified') def test_short_url_error(self): - with self.assertRaises(urllib2.URLError) as cm: + with self.assertRaises(URLError) as cm: self._loader.get('mailman://') self.assertEqual(cm.exception.reason, 'No template specified') def test_bad_language(self): - with self.assertRaises(urllib2.URLError) as cm: + with self.assertRaises(URLError) as cm: self._loader.get('mailman:///xx/demo.txt') self.assertEqual(cm.exception.reason, 'Bad language or list name') def test_bad_mailing_list(self): - with self.assertRaises(urllib2.URLError) as cm: + with self.assertRaises(URLError) as cm: self._loader.get('mailman:///missing@example.com/demo.txt') self.assertEqual(cm.exception.reason, 'Bad language or list name') def test_too_many_path_components(self): - with self.assertRaises(urllib2.URLError) as cm: + with self.assertRaises(URLError) as cm: self._loader.get('mailman:///missing@example.com/en/foo/demo.txt') self.assertEqual(cm.exception.reason, 'No such file') @@ -132,8 +129,8 @@ class TestTemplateLoader(unittest.TestCase): test_text = b'\xe4\xb8\xad' path = os.path.join(self.var_dir, 'templates', 'site', 'it') os.makedirs(path) - with open(os.path.join(path, 'demo.txt'), 'w') as fp: - print(test_text, end='', file=fp) + with open(os.path.join(path, 'demo.txt'), 'wb') as fp: + fp.write(test_text) content = self._loader.get('mailman:///it/demo.txt') - self.assertTrue(isinstance(content, unicode)) + self.assertIsInstance(content, six.text_type) self.assertEqual(content, test_text.decode('utf-8')) diff --git a/src/mailman/archiving/mailarchive.py b/src/mailman/archiving/mailarchive.py index c5fe5d0cb..a712e4052 100644 --- a/src/mailman/archiving/mailarchive.py +++ b/src/mailman/archiving/mailarchive.py @@ -17,21 +17,16 @@ """The Mail-Archive.com archiver.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'MailArchive', ] -from urllib import quote -from urlparse import urljoin -from zope.interface import implementer - from mailman.config import config from mailman.config.config import external_configuration from mailman.interfaces.archiver import ArchivePolicy, IArchiver +from six.moves.urllib_parse import quote, urljoin +from zope.interface import implementer @@ -77,5 +72,5 @@ class MailArchive: if mlist.archive_policy is ArchivePolicy.public: config.switchboards['out'].enqueue( msg, - listname=mlist.fqdn_listname, + listid=mlist.list_id, recipients=[self.recipient]) diff --git a/src/mailman/archiving/mhonarc.py b/src/mailman/archiving/mhonarc.py index f2d1f77fe..31853183f 100644 --- a/src/mailman/archiving/mhonarc.py +++ b/src/mailman/archiving/mhonarc.py @@ -17,9 +17,6 @@ """MHonArc archiver.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'MHonArc', ] @@ -28,13 +25,12 @@ __all__ = [ import logging import subprocess -from urlparse import urljoin -from zope.interface import implementer - from mailman.config import config from mailman.config.config import external_configuration from mailman.interfaces.archiver import IArchiver from mailman.utilities.string import expand +from six.moves.urllib_parse import urljoin +from zope.interface import implementer log = logging.getLogger('mailman.archiver') @@ -84,7 +80,7 @@ class MHonArc: command = expand(self.command, substitutions) proc = subprocess.Popen( command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - shell=True) + universal_newlines=True, shell=True) stdout, stderr = proc.communicate(msg.as_string()) if proc.returncode != 0: log.error('%s: mhonarc subprocess had non-zero exit code: %s' % diff --git a/src/mailman/archiving/prototype.py b/src/mailman/archiving/prototype.py index 77b2294ed..a27a2e57f 100644 --- a/src/mailman/archiving/prototype.py +++ b/src/mailman/archiving/prototype.py @@ -17,9 +17,6 @@ """Prototypical permalinking archiver.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Prototype', ] @@ -30,14 +27,13 @@ import errno import logging from datetime import timedelta -from mailbox import Maildir -from urlparse import urljoin - from flufl.lock import Lock, TimeOutError -from zope.interface import implementer - +from mailbox import Maildir from mailman.config import config from mailman.interfaces.archiver import IArchiver +from six.moves.urllib_parse import urljoin +from zope.interface import implementer + log = logging.getLogger('mailman.error') diff --git a/src/mailman/archiving/tests/test_prototype.py b/src/mailman/archiving/tests/test_prototype.py index fba46ea4b..4cd33d431 100644 --- a/src/mailman/archiving/tests/test_prototype.py +++ b/src/mailman/archiving/tests/test_prototype.py @@ -17,9 +17,6 @@ """Test the prototype archiver.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestPrototypeArchiver', ] @@ -33,7 +30,6 @@ import threading from email import message_from_file from flufl.lock import Lock - from mailman.app.lifecycle import create_list from mailman.archiving.prototype import Prototype from mailman.config import config @@ -89,13 +85,13 @@ but the water deserves to be swum. def _find(self, path): all_filenames = set() for dirpath, dirnames, filenames in os.walk(path): - if not isinstance(dirpath, unicode): - dirpath = unicode(dirpath) + if isinstance(dirpath, bytes): + dirpath = dirpath.decode('utf-8') all_filenames.add(dirpath) for filename in filenames: new_filename = filename - if not isinstance(filename, unicode): - new_filename = unicode(filename) + if isinstance(filename, bytes): + new_filename = filename.decode('utf-8') all_filenames.add(os.path.join(dirpath, new_filename)) return all_filenames diff --git a/src/mailman/bin/export.py b/src/mailman/bin/export.py index a5400a9bc..1ee9f31e1 100644 --- a/src/mailman/bin/export.py +++ b/src/mailman/bin/export.py @@ -134,7 +134,7 @@ class XMLDumper(object): print >> self._fp, '<%s%s/>' % (_name, attrs) else: # The value might contain angle brackets. - value = escape(unicode(_value)) + value = escape(_value.decode('utf-8')) print >> self._fp, '<%s%s>%s</%s>' % (_name, attrs, value, _name) def _do_list_categories(self, mlist, k, subcat=None): diff --git a/src/mailman/bin/gate_news.py b/src/mailman/bin/gate_news.py index 9bb1e5f61..275956fc1 100644 --- a/src/mailman/bin/gate_news.py +++ b/src/mailman/bin/gate_news.py @@ -149,7 +149,7 @@ def poll_newsgroup(mlist, conn, first, last, glock): # Post the message to the locked list inq = Switchboard(config.INQUEUE_DIR) inq.enqueue(msg, - listname=mlist.internal_name(), + listid=mlist.list_id, fromusenet=True) log.info('posted to list %s: %7d', listname, num) except nntplib.NNTPError as e: diff --git a/src/mailman/bin/mailman.py b/src/mailman/bin/mailman.py index 67f4d0910..ad8de144f 100644 --- a/src/mailman/bin/mailman.py +++ b/src/mailman/bin/mailman.py @@ -17,9 +17,6 @@ """The 'mailman' command dispatcher.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'main', ] @@ -28,13 +25,13 @@ __all__ = [ import os import argparse -from zope.interface.verify import verifyObject - +from functools import cmp_to_key from mailman.core.i18n import _ from mailman.core.initialize import initialize from mailman.interfaces.command import ICLISubCommand from mailman.utilities.modules import find_components from mailman.version import MAILMAN_VERSION_FULL +from zope.interface.verify import verifyObject @@ -77,9 +74,14 @@ def main(): return -1 elif other.name == 'help': return 1 + elif command.name < other.name: + return -1 + elif command.name == other.name: + return 0 else: - return cmp(command.name, other.name) - subcommands.sort(cmp=sort_function) + assert command.name > other.name + return 1 + subcommands.sort(key=cmp_to_key(sort_function)) for command in subcommands: command_parser = subparser.add_parser( command.name, help=_(command.__doc__)) diff --git a/src/mailman/bin/master.py b/src/mailman/bin/master.py index 50e8cb5bf..e7efdc537 100644 --- a/src/mailman/bin/master.py +++ b/src/mailman/bin/master.py @@ -17,9 +17,6 @@ """Master subprocess watcher.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Loop', 'main', @@ -37,7 +34,6 @@ from datetime import timedelta from enum import Enum from flufl.lock import Lock, NotLockedError, TimeOutError from lazr.config import as_boolean - from mailman.config import config from mailman.core.i18n import _ from mailman.core.logging import reopen @@ -357,7 +353,7 @@ class Loop: # Set the environment variable which tells the runner that it's # running under bin/master control. This subtly changes the error # behavior of bin/runner. - os.environ['MAILMAN_UNDER_MASTER_CONTROL'] = '1' + env = {'MAILMAN_UNDER_MASTER_CONTROL': '1'} # Craft the command line arguments for the exec() call. rswitch = '--runner=' + spec # Wherever master lives, so too must live the runner script. @@ -365,15 +361,21 @@ class Loop: # config.PYTHON, which is the absolute path to the Python interpreter, # must be given as argv[0] due to Python's library search algorithm. args = [sys.executable, sys.executable, exe, rswitch] + # Always pass the explicit path to the configuration file to the + # sub-runners. This avoids any debate about which cfg file is used. config_file = (config.filename if self._config_file is None else self._config_file) args.extend(['-C', config_file]) log = logging.getLogger('mailman.runner') log.debug('starting: %s', args) + # We must pass this environment variable through if it's set, + # otherwise runner processes will not have the correct VAR_DIR. + var_dir = os.environ.get('MAILMAN_VAR_DIR') + if var_dir is not None: + env['MAILMAN_VAR_DIR'] = var_dir # For the testing framework, if this environment variable is set, pass # it on to the subprocess. coverage_env = os.environ.get('COVERAGE_PROCESS_START') - env = dict() if coverage_env is not None: env['COVERAGE_PROCESS_START'] = coverage_env args.append(env) diff --git a/src/mailman/bin/onebounce.py b/src/mailman/bin/onebounce.py index 1c23fc42a..b504b4c00 100644 --- a/src/mailman/bin/onebounce.py +++ b/src/mailman/bin/onebounce.py @@ -18,9 +18,6 @@ """Test bounce detection on message files.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'main', ] diff --git a/src/mailman/bin/runner.py b/src/mailman/bin/runner.py index 7648ed961..88e02254f 100644 --- a/src/mailman/bin/runner.py +++ b/src/mailman/bin/runner.py @@ -17,9 +17,6 @@ """The runner process.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'main', ] diff --git a/src/mailman/bin/tests/test_master.py b/src/mailman/bin/tests/test_master.py index d6e301e58..c65777e5e 100644 --- a/src/mailman/bin/tests/test_master.py +++ b/src/mailman/bin/tests/test_master.py @@ -17,9 +17,6 @@ """Test master watcher utilities.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestMasterLock', ] diff --git a/src/mailman/chains/accept.py b/src/mailman/chains/accept.py index f5dd5a73d..89995b5a1 100644 --- a/src/mailman/chains/accept.py +++ b/src/mailman/chains/accept.py @@ -17,9 +17,6 @@ """The terminal 'accept' chain.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'AcceptChain', ] @@ -27,12 +24,11 @@ __all__ = [ import logging -from zope.event import notify - from mailman.chains.base import TerminalChainBase from mailman.config import config from mailman.core.i18n import _ from mailman.interfaces.chain import AcceptEvent +from zope.event import notify log = logging.getLogger('mailman.vette') diff --git a/src/mailman/chains/base.py b/src/mailman/chains/base.py index 37d8e76f3..7db31de73 100644 --- a/src/mailman/chains/base.py +++ b/src/mailman/chains/base.py @@ -17,9 +17,6 @@ """Base class for terminal chains.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Chain', 'Link', @@ -27,11 +24,10 @@ __all__ = [ ] -from zope.interface import implementer - from mailman.config import config from mailman.interfaces.chain import ( IChain, IChainIterator, IChainLink, IMutableChain, LinkAction) +from zope.interface import implementer diff --git a/src/mailman/chains/builtin.py b/src/mailman/chains/builtin.py index bce9349a1..b26b31550 100644 --- a/src/mailman/chains/builtin.py +++ b/src/mailman/chains/builtin.py @@ -17,9 +17,6 @@ """The default built-in starting chain.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'BuiltInChain', ] @@ -27,12 +24,11 @@ __all__ = [ import logging -from zope.interface import implementer - from mailman.chains.base import Link from mailman.config import config from mailman.core.i18n import _ from mailman.interfaces.chain import IChain, LinkAction +from zope.interface import implementer log = logging.getLogger('mailman.vette') diff --git a/src/mailman/chains/discard.py b/src/mailman/chains/discard.py index 001b243ac..9eb419201 100644 --- a/src/mailman/chains/discard.py +++ b/src/mailman/chains/discard.py @@ -17,20 +17,17 @@ """The terminal 'discard' chain.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'DiscardChain', ] import logging -from zope.event import notify from mailman.chains.base import TerminalChainBase from mailman.core.i18n import _ from mailman.interfaces.chain import DiscardEvent +from zope.event import notify log = logging.getLogger('mailman.vette') diff --git a/src/mailman/chains/headers.py b/src/mailman/chains/headers.py index 7628c8b7c..5738336e8 100644 --- a/src/mailman/chains/headers.py +++ b/src/mailman/chains/headers.py @@ -17,9 +17,6 @@ """The header-matching chain.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'HeaderMatchChain', ] @@ -28,13 +25,12 @@ __all__ = [ import re import logging -from zope.interface import implementer - from mailman.chains.base import Chain, Link from mailman.config import config from mailman.core.i18n import _ from mailman.interfaces.chain import LinkAction from mailman.interfaces.rules import IRule +from zope.interface import implementer log = logging.getLogger('mailman.error') @@ -122,7 +118,7 @@ class HeaderMatchChain(Chain): """See `IMutableChain`.""" # Remove all dynamically created rules. Use the keys so we can mutate # the dictionary inside the loop. - for rule_name in config.rules.keys(): + for rule_name in list(config.rules): if rule_name.startswith('header-match-'): del config.rules[rule_name] self._extended_links = [] diff --git a/src/mailman/chains/hold.py b/src/mailman/chains/hold.py index 1293ea266..7a516dc0d 100644 --- a/src/mailman/chains/hold.py +++ b/src/mailman/chains/hold.py @@ -17,9 +17,6 @@ """The terminal 'hold' chain.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'HoldChain', ] @@ -30,10 +27,6 @@ import logging from email.mime.message import MIMEMessage from email.mime.text import MIMEText from email.utils import formatdate, make_msgid -from zope.component import getUtility -from zope.event import notify -from zope.interface import implementer - from mailman.app.moderator import hold_message from mailman.app.replybot import can_acknowledge from mailman.chains.base import TerminalChainBase @@ -47,6 +40,9 @@ from mailman.interfaces.pending import IPendable, IPendings from mailman.interfaces.usermanager import IUserManager from mailman.utilities.i18n import make from mailman.utilities.string import oneline, wrap +from zope.component import getUtility +from zope.event import notify +from zope.interface import implementer log = logging.getLogger('mailman.vette') @@ -157,7 +153,7 @@ class HoldChain(TerminalChainBase): if original_subject is None: original_subject = _('(no subject)') else: - original_subject = oneline(original_subject, charset) + original_subject = oneline(original_subject, in_unicode=True) substitutions = dict( listname = mlist.fqdn_listname, subject = original_subject, diff --git a/src/mailman/chains/moderation.py b/src/mailman/chains/moderation.py index 9b34f6389..944a66089 100644 --- a/src/mailman/chains/moderation.py +++ b/src/mailman/chains/moderation.py @@ -34,21 +34,17 @@ made as to the disposition of the message. `defer` is the default for members, while `hold` is the default for nonmembers. """ -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ModerationChain', ] -from zope.interface import implementer - from mailman.chains.base import Link from mailman.config import config from mailman.core.i18n import _ from mailman.interfaces.action import Action from mailman.interfaces.chain import IChain, LinkAction +from zope.interface import implementer diff --git a/src/mailman/chains/owner.py b/src/mailman/chains/owner.py index 8e9aac154..9b0670ac9 100644 --- a/src/mailman/chains/owner.py +++ b/src/mailman/chains/owner.py @@ -17,9 +17,6 @@ """The standard -owner posting chain.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'BuiltInOwnerChain', ] @@ -27,12 +24,11 @@ __all__ = [ import logging -from zope.event import notify - from mailman.chains.base import TerminalChainBase from mailman.config import config from mailman.core.i18n import _ from mailman.interfaces.chain import AcceptOwnerEvent +from zope.event import notify log = logging.getLogger('mailman.vette') diff --git a/src/mailman/chains/reject.py b/src/mailman/chains/reject.py index e24cedb85..2f358afe1 100644 --- a/src/mailman/chains/reject.py +++ b/src/mailman/chains/reject.py @@ -17,9 +17,6 @@ """The terminal 'reject' chain.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'RejectChain', ] @@ -27,12 +24,11 @@ __all__ = [ import logging -from zope.event import notify - from mailman.app.bounces import bounce_message from mailman.chains.base import TerminalChainBase from mailman.core.i18n import _ from mailman.interfaces.chain import RejectEvent +from zope.event import notify log = logging.getLogger('mailman.vette') diff --git a/src/mailman/chains/tests/test_base.py b/src/mailman/chains/tests/test_base.py index 8d0d70449..784309395 100644 --- a/src/mailman/chains/tests/test_base.py +++ b/src/mailman/chains/tests/test_base.py @@ -17,9 +17,6 @@ """Test the base chain stuff.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestMiscellaneous', ] diff --git a/src/mailman/chains/tests/test_headers.py b/src/mailman/chains/tests/test_headers.py index adfc0ecb6..55bed3af0 100644 --- a/src/mailman/chains/tests/test_headers.py +++ b/src/mailman/chains/tests/test_headers.py @@ -17,9 +17,6 @@ """Test the header chain.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestHeaderChain', ] diff --git a/src/mailman/chains/tests/test_hold.py b/src/mailman/chains/tests/test_hold.py index a1fddd558..2a49b0ff0 100644 --- a/src/mailman/chains/tests/test_hold.py +++ b/src/mailman/chains/tests/test_hold.py @@ -17,9 +17,6 @@ """Additional tests for the hold chain.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestAutorespond', ] @@ -27,14 +24,13 @@ __all__ = [ import unittest -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.chains.hold import autorespond_to_sender from mailman.interfaces.autorespond import IAutoResponseSet, Response from mailman.interfaces.usermanager import IUserManager from mailman.testing.helpers import configuration, get_queue_messages from mailman.testing.layers import ConfigLayer +from zope.component import getUtility diff --git a/src/mailman/chains/tests/test_owner.py b/src/mailman/chains/tests/test_owner.py index 96b858317..0766ba630 100644 --- a/src/mailman/chains/tests/test_owner.py +++ b/src/mailman/chains/tests/test_owner.py @@ -17,9 +17,6 @@ """Test the owner chain.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestOwnerChain', ] @@ -32,8 +29,7 @@ from mailman.chains.owner import BuiltInOwnerChain from mailman.core.chains import process from mailman.interfaces.chain import AcceptOwnerEvent from mailman.testing.helpers import ( - event_subscribers, - get_queue_messages, + event_subscribers, get_queue_messages, specialized_message_from_string as mfs) from mailman.testing.layers import ConfigLayer diff --git a/src/mailman/commands/cli_aliases.py b/src/mailman/commands/cli_aliases.py index 7c85ad9e0..2e1dc88ec 100644 --- a/src/mailman/commands/cli_aliases.py +++ b/src/mailman/commands/cli_aliases.py @@ -17,20 +17,16 @@ """Generate Mailman alias files for your MTA.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Aliases', ] -from zope.interface import implementer - from mailman.config import config from mailman.core.i18n import _ from mailman.interfaces.command import ICLISubCommand from mailman.utilities.modules import call_name +from zope.interface import implementer diff --git a/src/mailman/commands/cli_conf.py b/src/mailman/commands/cli_conf.py index 7fe9fce7d..d0b7f7d2f 100644 --- a/src/mailman/commands/cli_conf.py +++ b/src/mailman/commands/cli_conf.py @@ -17,9 +17,6 @@ """Print the mailman configuration.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Conf' ] @@ -29,11 +26,10 @@ import sys from contextlib import closing from lazr.config._config import Section -from zope.interface import implementer - from mailman.config import config from mailman.core.i18n import _ from mailman.interfaces.command import ICLISubCommand +from zope.interface import implementer diff --git a/src/mailman/commands/cli_control.py b/src/mailman/commands/cli_control.py index b0afc1337..de3542106 100644 --- a/src/mailman/commands/cli_control.py +++ b/src/mailman/commands/cli_control.py @@ -15,11 +15,8 @@ # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see <http://www.gnu.org/licenses/>. -"""Module stuff.""" +"""Start/stop/reopen/restart commands.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Reopen', 'Restart', @@ -34,12 +31,11 @@ import errno import signal import logging -from zope.interface import implementer - from mailman.bin.master import WatcherState, master_state from mailman.config import config from mailman.core.i18n import _ from mailman.interfaces.command import ICLISubCommand +from zope.interface import implementer qlog = logging.getLogger('mailman.runner') @@ -124,8 +120,8 @@ class Start: # subprocesses to calculate their path to the $VAR_DIR. Before we # chdir() though, calculate the absolute path to the configuration # file. - config_path = (os.path.abspath(args.config) - if args.config else None) + config_path = (config.filename if args.config is None + else os.path.abspath(args.config)) os.environ['MAILMAN_VAR_DIR'] = config.VAR_DIR os.chdir(config.VAR_DIR) # Exec the master watcher. @@ -135,8 +131,9 @@ class Start: ] if args.force: execl_args.append('--force') - if config_path: - execl_args.extend(['-C', config_path]) + # Always pass the config file path to the master projects, so there's + # no confusion about which cfg is being used. + execl_args.extend(['-C', config_path]) qlog.debug('starting: %s', execl_args) os.execl(*execl_args) # We should never get here. diff --git a/src/mailman/commands/cli_help.py b/src/mailman/commands/cli_help.py index ce39eeda5..721c8936e 100644 --- a/src/mailman/commands/cli_help.py +++ b/src/mailman/commands/cli_help.py @@ -17,17 +17,13 @@ """The 'help' subcommand.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Help', ] -from zope.interface import implementer - from mailman.interfaces.command import ICLISubCommand +from zope.interface import implementer diff --git a/src/mailman/commands/cli_import.py b/src/mailman/commands/cli_import.py index 5e25cd4fe..38b6fcef4 100644 --- a/src/mailman/commands/cli_import.py +++ b/src/mailman/commands/cli_import.py @@ -17,25 +17,21 @@ """Importing list data into Mailman 3.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Import21', ] import sys -import cPickle - -from zope.component import getUtility -from zope.interface import implementer from mailman.core.i18n import _ from mailman.database.transaction import transactional from mailman.interfaces.command import ICLISubCommand from mailman.interfaces.listmanager import IListManager from mailman.utilities.importer import import_config_pck, Import21Error +from six.moves import cPickle +from zope.component import getUtility +from zope.interface import implementer @@ -78,7 +74,7 @@ class Import21: assert len(args.pickle_file) == 1, ( 'Unexpected positional arguments: %s' % args.pickle_file) filename = args.pickle_file[0] - with open(filename) as fp: + with open(filename, 'rb') as fp: while True: try: config_dict = cPickle.load(fp) diff --git a/src/mailman/commands/cli_info.py b/src/mailman/commands/cli_info.py index 4304e0ddb..6dd938127 100644 --- a/src/mailman/commands/cli_info.py +++ b/src/mailman/commands/cli_info.py @@ -17,9 +17,6 @@ """Information about this Mailman instance.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Info' ] @@ -28,13 +25,12 @@ __all__ = [ import sys from lazr.config import as_boolean -from zope.interface import implementer - from mailman.config import config from mailman.core.i18n import _ from mailman.interfaces.command import ICLISubCommand from mailman.rest.helpers import path_to from mailman.version import MAILMAN_VERSION_FULL +from zope.interface import implementer diff --git a/src/mailman/commands/cli_inject.py b/src/mailman/commands/cli_inject.py index 07ef0ec6c..ad4b53291 100644 --- a/src/mailman/commands/cli_inject.py +++ b/src/mailman/commands/cli_inject.py @@ -17,9 +17,6 @@ """bin/mailman inject""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Inject', ] @@ -27,14 +24,13 @@ __all__ = [ import sys -from zope.component import getUtility -from zope.interface import implementer - from mailman.app.inject import inject_text from mailman.config import config from mailman.core.i18n import _ from mailman.interfaces.command import ICLISubCommand from mailman.interfaces.listmanager import IListManager +from zope.component import getUtility +from zope.interface import implementer @@ -49,7 +45,7 @@ class Inject: self.parser = parser command_parser.add_argument( '-q', '--queue', - type=unicode, help=_(""" + help=_(""" The name of the queue to inject the message to. QUEUE must be one of the directories inside the qfiles directory. If omitted, the incoming queue is used.""")) @@ -59,7 +55,7 @@ class Inject: help=_('Show a list of all available queue names and exit.')) command_parser.add_argument( '-f', '--filename', - type=unicode, help=_(""" + help=_(""" Name of file containing the message to inject. If not given, or '-' (without the quotes) standard input is used.""")) # Required positional argument. diff --git a/src/mailman/commands/cli_lists.py b/src/mailman/commands/cli_lists.py index cf1bd2ead..fac1dcd1d 100644 --- a/src/mailman/commands/cli_lists.py +++ b/src/mailman/commands/cli_lists.py @@ -17,9 +17,6 @@ """The 'lists' subcommand.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Create', 'Lists', @@ -27,9 +24,6 @@ __all__ = [ ] -from zope.component import getUtility -from zope.interface import implementer - from mailman.app.lifecycle import create_list, remove_list from mailman.core.constants import system_preferences from mailman.core.i18n import _ @@ -43,6 +37,8 @@ from mailman.interfaces.domain import ( from mailman.interfaces.languages import ILanguageManager from mailman.interfaces.listmanager import IListManager, ListAlreadyExistsError from mailman.utilities.i18n import make +from zope.component import getUtility +from zope.interface import implementer COMMASPACE = ', ' @@ -135,12 +131,12 @@ class Create: self.parser = parser command_parser.add_argument( '--language', - type=unicode, metavar='CODE', help=_("""\ + metavar='CODE', help=_("""\ Set the list's preferred language to CODE, which must be a registered two letter language code.""")) command_parser.add_argument( '-o', '--owner', - type=unicode, action='append', default=[], + action='append', default=[], dest='owners', metavar='OWNER', help=_("""\ Specify a listowner email address. If the address is not currently registered with Mailman, the address is registered and diff --git a/src/mailman/commands/cli_members.py b/src/mailman/commands/cli_members.py index 291fda3b7..21d78ec54 100644 --- a/src/mailman/commands/cli_members.py +++ b/src/mailman/commands/cli_members.py @@ -17,9 +17,6 @@ """The 'members' subcommand.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Members', ] @@ -29,11 +26,6 @@ import sys import codecs from email.utils import formataddr, parseaddr -from operator import attrgetter -from passlib.utils import generate_password as generate -from zope.component import getUtility -from zope.interface import implementer - from mailman.app.membership import add_member from mailman.config import config from mailman.core.i18n import _ @@ -42,6 +34,10 @@ from mailman.interfaces.command import ICLISubCommand from mailman.interfaces.listmanager import IListManager from mailman.interfaces.member import ( AlreadySubscribedError, DeliveryMode, DeliveryStatus) +from operator import attrgetter +from passlib.utils import generate_password as generate +from zope.component import getUtility +from zope.interface import implementer @@ -197,8 +193,6 @@ class Members: continue # Parse the line and ensure that the values are unicodes. display_name, email = parseaddr(line) - display_name = display_name.decode(fp.encoding) - email = email.decode(fp.encoding) # Give the user a default, user-friendly password. password = generate(int(config.passwords.password_length)) try: diff --git a/src/mailman/commands/cli_qfile.py b/src/mailman/commands/cli_qfile.py index 986898bee..e502deac8 100644 --- a/src/mailman/commands/cli_qfile.py +++ b/src/mailman/commands/cli_qfile.py @@ -17,24 +17,22 @@ """Getting information out of a qfile.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'QFile', ] -import cPickle - -from pprint import PrettyPrinter -from zope.interface import implementer +import six from mailman.core.i18n import _ from mailman.interfaces.command import ICLISubCommand from mailman.utilities.interact import interact +from pprint import PrettyPrinter +from six.moves import cPickle +from zope.interface import implementer +# This is deliberately called 'm' for use with --interactive. m = [] @@ -71,7 +69,7 @@ class QFile: """See `ICLISubCommand`.""" printer = PrettyPrinter(indent=4) assert len(args.qfile) == 1, 'Wrong number of positional arguments' - with open(args.qfile[0]) as fp: + with open(args.qfile[0], 'rb') as fp: while True: try: m.append(cPickle.load(fp)) @@ -82,7 +80,7 @@ class QFile: for i, obj in enumerate(m): count = i + 1 print(_('<----- start object $count ----->')) - if isinstance(obj, basestring): + if isinstance(obj, six.string_types): print(obj) else: printer.pprint(obj) diff --git a/src/mailman/commands/cli_status.py b/src/mailman/commands/cli_status.py index 207b44e04..2bef9d73c 100644 --- a/src/mailman/commands/cli_status.py +++ b/src/mailman/commands/cli_status.py @@ -17,9 +17,6 @@ """bin/mailman status.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Status', ] @@ -27,11 +24,10 @@ __all__ = [ import socket -from zope.interface import implementer - from mailman.bin.master import WatcherState, master_state from mailman.core.i18n import _ from mailman.interfaces.command import ICLISubCommand +from zope.interface import implementer diff --git a/src/mailman/commands/cli_unshunt.py b/src/mailman/commands/cli_unshunt.py index 77196565b..7cfa9e4ed 100644 --- a/src/mailman/commands/cli_unshunt.py +++ b/src/mailman/commands/cli_unshunt.py @@ -17,9 +17,6 @@ """The 'unshunt' command.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Unshunt', ] @@ -27,11 +24,10 @@ __all__ = [ import sys -from zope.interface import implementer - from mailman.config import config from mailman.core.i18n import _ from mailman.interfaces.command import ICLISubCommand +from zope.interface import implementer @@ -62,7 +58,7 @@ class Unshunt: which_queue = msgdata.get('whichq', 'in') if not args.discard: config.switchboards[which_queue].enqueue(msg, msgdata) - except Exception as error: + except Exception: print(_('Cannot unshunt message $filebase, skipping:\n$error'), file=sys.stderr) else: diff --git a/src/mailman/commands/cli_version.py b/src/mailman/commands/cli_version.py index 86ce9ab68..bc0f34a34 100644 --- a/src/mailman/commands/cli_version.py +++ b/src/mailman/commands/cli_version.py @@ -17,18 +17,14 @@ """The Mailman version.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Version', ] -from zope.interface import implementer - from mailman.interfaces.command import ICLISubCommand from mailman.version import MAILMAN_VERSION_FULL +from zope.interface import implementer diff --git a/src/mailman/commands/cli_withlist.py b/src/mailman/commands/cli_withlist.py index fc2363816..7cf8c0451 100644 --- a/src/mailman/commands/cli_withlist.py +++ b/src/mailman/commands/cli_withlist.py @@ -17,9 +17,6 @@ """bin/mailman withlist""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Shell', 'Withlist', @@ -30,15 +27,15 @@ import re import sys from lazr.config import as_boolean -from zope.component import getUtility -from zope.interface import implementer - from mailman.config import config from mailman.core.i18n import _ from mailman.interfaces.command import ICLISubCommand from mailman.interfaces.listmanager import IListManager from mailman.utilities.interact import DEFAULT_BANNER, interact from mailman.utilities.modules import call_name +from zope.component import getUtility +from zope.interface import implementer + # Global holding onto the open mailing list. m = None diff --git a/src/mailman/commands/docs/echo.rst b/src/mailman/commands/docs/echo.rst index 32399ebfc..6412a4afe 100644 --- a/src/mailman/commands/docs/echo.rst +++ b/src/mailman/commands/docs/echo.rst @@ -24,7 +24,7 @@ The original message is ignored, but the results receive the echoed command. >>> from mailman.email.message import Message >>> print(command.process(mlist, Message(), {}, ('foo', 'bar'), results)) ContinueProcessing.yes - >>> print(unicode(results)) + >>> print(str(results)) The results of your email command are provided below. <BLANKLINE> echo foo bar diff --git a/src/mailman/commands/docs/help.rst b/src/mailman/commands/docs/help.rst index bbd6c8c09..5330a0b79 100644 --- a/src/mailman/commands/docs/help.rst +++ b/src/mailman/commands/docs/help.rst @@ -25,7 +25,7 @@ short description of each of them. >>> from mailman.email.message import Message >>> print(help.process(mlist, Message(), {}, (), results)) ContinueProcessing.yes - >>> print(unicode(results)) + >>> print(results) The results of your email command are provided below. <BLANKLINE> confirm - Confirm a subscription request. @@ -44,19 +44,19 @@ With an argument, you can get more detailed help about a specific command. >>> results = Results() >>> print(help.process(mlist, Message(), {}, ('help',), results)) ContinueProcessing.yes - >>> print(unicode(results)) + >>> print(results) The results of your email command are provided below. <BLANKLINE> help [command] Get help about available email commands. <BLANKLINE> - + Some commands have even more detailed help. >>> results = Results() >>> print(help.process(mlist, Message(), {}, ('join',), results)) ContinueProcessing.yes - >>> print(unicode(results)) + >>> print(results) The results of your email command are provided below. <BLANKLINE> join [digest=<no|mime|plain>] diff --git a/src/mailman/commands/docs/info.rst b/src/mailman/commands/docs/info.rst index 8bc7579e6..6ce223403 100644 --- a/src/mailman/commands/docs/info.rst +++ b/src/mailman/commands/docs/info.rst @@ -62,20 +62,21 @@ definition. Python ... ... File system paths: - ARCHIVE_DIR = /var/lib/mailman/archives - BIN_DIR = /sbin - DATA_DIR = /var/lib/mailman/data - ETC_DIR = /etc - EXT_DIR = /etc/mailman.d - LIST_DATA_DIR = /var/lib/mailman/lists - LOCK_DIR = /var/lock/mailman - LOCK_FILE = /var/lock/mailman/master.lck - LOG_DIR = /var/log/mailman - MESSAGES_DIR = /var/lib/mailman/messages - PID_FILE = /var/run/mailman/master.pid - QUEUE_DIR = /var/spool/mailman - TEMPLATE_DIR = .../mailman/templates - VAR_DIR = /var/lib/mailman + ARCHIVE_DIR = /var/lib/mailman/archives + BIN_DIR = /sbin + CFG_FILE = .../test.cfg + DATA_DIR = /var/lib/mailman/data + ETC_DIR = /etc + EXT_DIR = /etc/mailman.d + LIST_DATA_DIR = /var/lib/mailman/lists + LOCK_DIR = /var/lock/mailman + LOCK_FILE = /var/lock/mailman/master.lck + LOG_DIR = /var/log/mailman + MESSAGES_DIR = /var/lib/mailman/messages + PID_FILE = /var/run/mailman/master.pid + QUEUE_DIR = /var/spool/mailman + TEMPLATE_DIR = .../mailman/templates + VAR_DIR = /var/lib/mailman .. _`Filesystem Hierarchy Standard`: http://www.pathname.com/fhs/ diff --git a/src/mailman/commands/docs/inject.rst b/src/mailman/commands/docs/inject.rst index 63e7b0366..de295b8f6 100644 --- a/src/mailman/commands/docs/inject.rst +++ b/src/mailman/commands/docs/inject.rst @@ -94,7 +94,7 @@ By default, the incoming queue is used. >>> dump_msgdata(items[0].msgdata) _parsemsg : False - listname : test@example.com + listid : test.example.com original_size: 203 version : 3 @@ -122,7 +122,7 @@ But a different queue can be specified on the command line. >>> dump_msgdata(items[0].msgdata) _parsemsg : False - listname : test@example.com + listid : test.example.com original_size: 203 version : 3 @@ -133,7 +133,7 @@ Standard input The message text can also be provided on standard input. :: - >>> from StringIO import StringIO + >>> from six import StringIO # Remember: we've got unicode literals turned on. >>> standard_in = StringIO(str("""\ @@ -167,7 +167,7 @@ The message text can also be provided on standard input. >>> dump_msgdata(items[0].msgdata) _parsemsg : False - listname : test@example.com + listid : test.example.com original_size: 211 version : 3 @@ -195,7 +195,7 @@ injected. _parsemsg : False bar : two foo : one - listname : test@example.com + listid : test.example.com original_size: 203 version : 3 diff --git a/src/mailman/commands/docs/members.rst b/src/mailman/commands/docs/members.rst index 7b99e92f9..28f238f31 100644 --- a/src/mailman/commands/docs/members.rst +++ b/src/mailman/commands/docs/members.rst @@ -229,15 +229,14 @@ You can also specify ``-`` as the filename, in which case the addresses are taken from standard input. :: - >>> from StringIO import StringIO + >>> from six import StringIO >>> fp = StringIO() - >>> fp.encoding = 'us-ascii' >>> for address in ('dperson@example.com', ... 'Elly Person <eperson@example.com>', ... 'fperson@example.com (Fred Person)', ... ): ... print(address, file=fp) - >>> fp.seek(0) + >>> filepos = fp.seek(0) >>> import sys >>> sys.stdin = fp diff --git a/src/mailman/commands/docs/membership.rst b/src/mailman/commands/docs/membership.rst index aa3ab97e6..a260e930a 100644 --- a/src/mailman/commands/docs/membership.rst +++ b/src/mailman/commands/docs/membership.rst @@ -45,7 +45,7 @@ If that's missing though, then an error is returned. >>> from mailman.email.message import Message >>> print(join.process(mlist, Message(), {}, (), results)) ContinueProcessing.no - >>> print(unicode(results)) + >>> print(results) The results of your email command are provided below. <BLANKLINE> join: No valid address found to subscribe @@ -60,7 +60,7 @@ The ``subscribe`` command is an alias. >>> results = Results() >>> print(subscribe.process(mlist, Message(), {}, (), results)) ContinueProcessing.no - >>> print(unicode(results)) + >>> print(results) The results of your email command are provided below. <BLANKLINE> subscribe: No valid address found to subscribe @@ -79,7 +79,7 @@ When the message has a From field, that address will be subscribed. >>> results = Results() >>> print(join.process(mlist, msg, {}, (), results)) ContinueProcessing.yes - >>> print(unicode(results)) + >>> print(results) The results of your email command are provided below. <BLANKLINE> Confirmation email sent to Anne Person <anne@example.com> @@ -150,7 +150,7 @@ list. >>> results = Results() >>> print(confirm.process(mlist, msg, {}, (token,), results)) ContinueProcessing.yes - >>> print(unicode(results)) + >>> print(results) The results of your email command are provided below. <BLANKLINE> Confirmed @@ -208,7 +208,7 @@ list. >>> results = Results() >>> print(confirm.process(mlist_2, msg, {}, (token,), results)) ContinueProcessing.yes - >>> print(unicode(results)) + >>> print(results) The results of your email command are provided below. <BLANKLINE> Confirmed @@ -241,7 +241,7 @@ is sent a confirmation message for her request. >>> results = Results() >>> print(leave.process(mlist_2, msg, {}, (), results)) ContinueProcessing.yes - >>> print(unicode(results)) + >>> print(results) The results of your email command are provided below. <BLANKLINE> Anne Person <anne@example.com> left baker@example.com @@ -278,7 +278,7 @@ to unsubscribe Anne from the alpha mailing list. >>> print(leave.process(mlist, msg, {}, (), results)) ContinueProcessing.no - >>> print(unicode(results)) + >>> print(results) The results of your email command are provided below. <BLANKLINE> Invalid or unverified email address: anne.person@example.org @@ -299,7 +299,7 @@ unsubscribe her from the list. >>> print(leave.process(mlist, msg, {}, (), results)) ContinueProcessing.yes - >>> print(unicode(results)) + >>> print(results) The results of your email command are provided below. <BLANKLINE> Anne Person <anne.person@example.org> left alpha@example.com @@ -354,7 +354,7 @@ a user of the system. >>> print(confirm.process(mlist, msg, {}, (token,), results)) ContinueProcessing.yes - >>> print(unicode(results)) + >>> print(results) The results of your email command are provided below. <BLANKLINE> Confirmed diff --git a/src/mailman/commands/docs/qfile.rst b/src/mailman/commands/docs/qfile.rst index 8ec0a3952..e097ebf97 100644 --- a/src/mailman/commands/docs/qfile.rst +++ b/src/mailman/commands/docs/qfile.rst @@ -47,7 +47,6 @@ Once we've figured out the file name of the shunted message, we can print it. >>> command.process(FakeArgs) [----- start pickle -----] <----- start object 1 -----> - From nobody ... From: aperson@example.com To: test@example.com Subject: Uh oh @@ -55,11 +54,7 @@ Once we've figured out the file name of the shunted message, we can print it. I borkeded Mailman. <BLANKLINE> <----- start object 2 -----> - { u'_parsemsg': False, - 'bad': u'yes', - 'bar': u'baz', - 'foo': 7, - u'version': 3} + {'_parsemsg': False, 'bad': 'yes', 'bar': 'baz', 'foo': 7, 'version': 3} [----- end pickle -----] Maybe we don't want to print the contents of the file though, in case we want diff --git a/src/mailman/commands/docs/withlist.rst b/src/mailman/commands/docs/withlist.rst index e915eb04c..321b6e68a 100644 --- a/src/mailman/commands/docs/withlist.rst +++ b/src/mailman/commands/docs/withlist.rst @@ -52,10 +52,10 @@ single argument, the mailing list. >>> with open(os.path.join(config.VAR_DIR, 'showme.py'), 'w') as fp: ... print("""\ ... def showme(mailing_list): - ... print "The list's name is", mailing_list.fqdn_listname + ... print("The list's name is", mailing_list.fqdn_listname) ... ... def displayname(mailing_list): - ... print "The list's display name is", mailing_list.display_name + ... print("The list's display name is", mailing_list.display_name) ... """, file=fp) If the name of the function is the same as the module, then you only need to diff --git a/src/mailman/commands/eml_confirm.py b/src/mailman/commands/eml_confirm.py index 0239e0f25..2cef7cbad 100644 --- a/src/mailman/commands/eml_confirm.py +++ b/src/mailman/commands/eml_confirm.py @@ -15,22 +15,18 @@ # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see <http://www.gnu.org/licenses/>. -"""Module stuff.""" +"""The 'confirm' email command.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Confirm', ] -from zope.component import getUtility -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.interfaces.command import ContinueProcessing, IEmailCommand from mailman.interfaces.registrar import IRegistrar +from zope.component import getUtility +from zope.interface import implementer diff --git a/src/mailman/commands/eml_echo.py b/src/mailman/commands/eml_echo.py index eb476dc7d..2bd55edbc 100644 --- a/src/mailman/commands/eml_echo.py +++ b/src/mailman/commands/eml_echo.py @@ -17,18 +17,14 @@ """The email command 'echo'.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Echo', ] -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.interfaces.command import ContinueProcessing, IEmailCommand +from zope.interface import implementer SPACE = ' ' diff --git a/src/mailman/commands/eml_end.py b/src/mailman/commands/eml_end.py index 447d4066b..d25c19fcb 100644 --- a/src/mailman/commands/eml_end.py +++ b/src/mailman/commands/eml_end.py @@ -17,19 +17,15 @@ """The email commands 'end' and 'stop'.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'End', 'Stop', ] -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.interfaces.command import ContinueProcessing, IEmailCommand +from zope.interface import implementer diff --git a/src/mailman/commands/eml_help.py b/src/mailman/commands/eml_help.py index 139d484fb..8b93b272a 100644 --- a/src/mailman/commands/eml_help.py +++ b/src/mailman/commands/eml_help.py @@ -17,20 +17,16 @@ """The email command 'help'.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Help', ] -from zope.interface import implementer - from mailman.config import config from mailman.core.i18n import _ from mailman.interfaces.command import ContinueProcessing, IEmailCommand from mailman.utilities.string import wrap +from zope.interface import implementer SPACE = ' ' diff --git a/src/mailman/commands/eml_membership.py b/src/mailman/commands/eml_membership.py index c56b14041..e6a6825ed 100644 --- a/src/mailman/commands/eml_membership.py +++ b/src/mailman/commands/eml_membership.py @@ -17,9 +17,6 @@ """The email commands 'join' and 'subscribe'.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Join', 'Subscribe', @@ -29,15 +26,14 @@ __all__ = [ from email.utils import formataddr, parseaddr -from zope.component import getUtility -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.interfaces.command import ContinueProcessing, IEmailCommand from mailman.interfaces.member import DeliveryMode, MemberRole from mailman.interfaces.registrar import IRegistrar from mailman.interfaces.subscriptions import ISubscriptionService from mailman.interfaces.usermanager import IUserManager +from zope.component import getUtility +from zope.interface import implementer @@ -182,6 +178,7 @@ You may be asked to confirm your request.""") return ContinueProcessing.yes + class Unsubscribe(Leave): """The email 'unsubscribe' command (an alias for 'leave').""" diff --git a/src/mailman/commands/tests/test_conf.py b/src/mailman/commands/tests/test_conf.py index 12ed5c537..07036df3a 100644 --- a/src/mailman/commands/tests/test_conf.py +++ b/src/mailman/commands/tests/test_conf.py @@ -17,9 +17,6 @@ """Test the conf subcommand.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestConf', ] @@ -31,9 +28,9 @@ import mock import tempfile import unittest -from StringIO import StringIO from mailman.commands.cli_conf import Conf from mailman.testing.layers import ConfigLayer +from six import StringIO diff --git a/src/mailman/commands/tests/test_confirm.py b/src/mailman/commands/tests/test_confirm.py index 19a9068bc..f067a2a0a 100644 --- a/src/mailman/commands/tests/test_confirm.py +++ b/src/mailman/commands/tests/test_confirm.py @@ -17,9 +17,6 @@ """Test the `confirm` command.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestConfirm', ] @@ -27,8 +24,6 @@ __all__ = [ import unittest -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.commands.eml_confirm import Confirm from mailman.email.message import Message @@ -37,6 +32,7 @@ from mailman.interfaces.registrar import IRegistrar from mailman.runners.command import Results from mailman.testing.helpers import get_queue_messages, reset_the_world from mailman.testing.layers import ConfigLayer +from zope.component import getUtility diff --git a/src/mailman/commands/tests/test_control.py b/src/mailman/commands/tests/test_control.py index 0847d86b1..299f0da25 100644 --- a/src/mailman/commands/tests/test_control.py +++ b/src/mailman/commands/tests/test_control.py @@ -17,9 +17,6 @@ """Test some additional corner cases for starting/stopping.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestStart', 'find_master', @@ -37,11 +34,11 @@ import socket import unittest from datetime import timedelta, datetime - from mailman.commands.cli_control import Start, kill_watcher from mailman.config import config from mailman.testing.layers import ConfigLayer + SEP = '|' diff --git a/src/mailman/commands/tests/test_create.py b/src/mailman/commands/tests/test_create.py index c2dffb929..47808c997 100644 --- a/src/mailman/commands/tests/test_create.py +++ b/src/mailman/commands/tests/test_create.py @@ -17,9 +17,6 @@ """Test `bin/mailman create`.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestCreate', ] diff --git a/src/mailman/commands/tests/test_help.py b/src/mailman/commands/tests/test_help.py index 3c7d1ae9f..b2de0297d 100644 --- a/src/mailman/commands/tests/test_help.py +++ b/src/mailman/commands/tests/test_help.py @@ -17,10 +17,8 @@ """Additional tests for the `help` email command.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ + 'TestHelp', ] @@ -47,11 +45,11 @@ class TestHelp(unittest.TestCase): def test_too_many_arguments(self): # Error message when too many help arguments are given. results = Results() - status = self._help.process(self._mlist, Message(), {}, + status = self._help.process(self._mlist, Message(), {}, ('more', 'than', 'one'), results) self.assertEqual(status, ContinueProcessing.no) - self.assertEqual(unicode(results), """\ + self.assertEqual(str(results), """\ The results of your email command are provided below. help: too many arguments: more than one @@ -60,10 +58,10 @@ help: too many arguments: more than one def test_no_such_command(self): # Error message when asking for help on an existent command. results = Results() - status = self._help.process(self._mlist, Message(), {}, + status = self._help.process(self._mlist, Message(), {}, ('doesnotexist',), results) self.assertEqual(status, ContinueProcessing.no) - self.assertEqual(unicode(results), """\ + self.assertEqual(str(results), """\ The results of your email command are provided below. help: no such command: doesnotexist diff --git a/src/mailman/config/__init__.py b/src/mailman/config/__init__.py index fb240ad76..4b9b1d07a 100644 --- a/src/mailman/config/__init__.py +++ b/src/mailman/config/__init__.py @@ -17,9 +17,6 @@ """Mailman configuration package.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'config', ] diff --git a/src/mailman/config/config.py b/src/mailman/config/config.py index 7181e23e9..779fa27e5 100644 --- a/src/mailman/config/config.py +++ b/src/mailman/config/config.py @@ -17,9 +17,6 @@ """Configuration file loading and management.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Configuration', 'external_configuration', @@ -29,27 +26,28 @@ __all__ = [ import os import sys +import mailman.templates -from ConfigParser import SafeConfigParser from flufl.lock import Lock from lazr.config import ConfigSchema, as_boolean -from pkg_resources import resource_stream, resource_string -from string import Template -from zope.component import getUtility -from zope.event import notify -from zope.interface import implementer - -import mailman.templates - from mailman import version from mailman.interfaces.configuration import ( ConfigurationUpdatedEvent, IConfiguration, MissingConfigurationFileError) from mailman.interfaces.languages import ILanguageManager from mailman.utilities.filesystem import makedirs from mailman.utilities.modules import call_name, expand_path +from pkg_resources import resource_filename, resource_string as resource_bytes +from six.moves.configparser import ConfigParser, RawConfigParser +from string import Template +from unittest.mock import patch +from zope.component import getUtility +from zope.event import notify +from zope.interface import implementer SPACE = ' ' +SPACERS = '\n' + MAILMAN_CFG_TEMPLATE = """\ # AUTOMATICALLY GENERATED BY MAILMAN ON {} @@ -66,6 +64,11 @@ MAILMAN_CFG_TEMPLATE = """\ # enabled: yes # recipient: your.address@your.domain""" +class _NonStrictRawConfigParser(RawConfigParser): + def __init__(self, *args, **kws): + kws['strict'] = False + super().__init__(*args, **kws) + @implementer(IConfiguration) @@ -102,30 +105,29 @@ class Configuration: def load(self, filename=None): """Load the configuration from the schema and config files.""" - schema_file = config_file = None - try: - schema_file = resource_stream('mailman.config', 'schema.cfg') - schema = ConfigSchema('schema.cfg', schema_file) - # If a configuration file was given, load it now too. First, load - # the absolute minimum default configuration, then if a - # configuration filename was given by the user, push it. - config_file = resource_stream('mailman.config', 'mailman.cfg') - self._config = schema.loadFile(config_file, 'mailman.cfg') - if filename is not None: - self.filename = filename - with open(filename) as user_config: - self._config.push(filename, user_config.read()) - finally: - if schema_file: - schema_file.close() - if config_file: - config_file.close() - self._post_process() + schema_file = resource_filename('mailman.config', 'schema.cfg') + schema = ConfigSchema(schema_file) + # If a configuration file was given, load it now too. First, load + # the absolute minimum default configuration, then if a + # configuration filename was given by the user, push it. + config_file = resource_filename('mailman.config', 'mailman.cfg') + self._config = schema.load(config_file) + if filename is None: + self._post_process() + else: + self.filename = filename + with open(filename, 'r', encoding='utf-8') as user_config: + self.push(filename, user_config.read()) def push(self, config_name, config_string): """Push a new configuration onto the stack.""" self._clear() - self._config.push(config_name, config_string) + # In Python 3, the RawConfigParser() must be created with + # strict=False, otherwise we'll get a DuplicateSectionError. + # See https://bugs.launchpad.net/lazr.config/+bug/1397779 + with patch('lazr.config._config.RawConfigParser', + _NonStrictRawConfigParser): + self._config.push(config_name, config_string) self._post_process() def pop(self, config_name): @@ -164,6 +166,7 @@ class Configuration: # path is relative. var_dir = os.environ.get('MAILMAN_VAR_DIR', category.var_dir) substitutions = dict( + cwd = os.getcwd(), argv = bin_dir, # Directories. bin_dir = category.bin_dir, @@ -185,26 +188,32 @@ class Configuration: lock_file = category.lock_file, pid_file = category.pid_file, ) + # Add the path to the .cfg file, if one was given on the command line. + if self.filename is not None: + substitutions['cfg_file'] = self.filename # Now, perform substitutions recursively until there are no more # variables with $-vars in them, or until substitutions are not # helping any more. last_dollar_count = 0 while True: + expandables = [] # Mutate the dictionary during iteration. - dollar_count = 0 - for key in substitutions.keys(): + for key in substitutions: raw_value = substitutions[key] value = Template(raw_value).safe_substitute(substitutions) if '$' in value: # Still more work to do. - dollar_count += 1 + expandables.append((key, value)) substitutions[key] = value - if dollar_count == 0: + if len(expandables) == 0: break - if dollar_count == last_dollar_count: - print('Path expansion infloop detected', file=sys.stderr) + if len(expandables) == last_dollar_count: + print('Path expansion infloop detected:\n', + SPACERS.join('\t{}: {}'.format(key, value) + for key, value in sorted(expandables)), + file=sys.stderr) sys.exit(1) - last_dollar_count = dollar_count + last_dollar_count = len(expandables) # Ensure that all paths are normalized and made absolute. Handle the # few special cases first. Most of these are due to backward # compatibility. @@ -269,7 +278,7 @@ class Configuration: -def load_external(path, encoding=None): +def load_external(path): """Load the configuration file named by path. :param path: A string naming the location of the external configuration @@ -278,21 +287,16 @@ def load_external(path, encoding=None): value must name a ``.cfg`` file located within Python's import path, however the trailing ``.cfg`` suffix is implied (don't provide it here). - :param encoding: The encoding to apply to the data read from path. If - None, then bytes will be returned. - :return: A unicode string or bytes, depending on ``encoding``. + :return: The contents of the configuration file. + :rtype: str """ # Is the context coming from a file system or Python path? if path.startswith('python:'): resource_path = path[7:] package, dot, resource = resource_path.rpartition('.') - config_string = resource_string(package, resource + '.cfg') - else: - with open(path, 'rb') as fp: - config_string = fp.read() - if encoding is None: - return config_string - return config_string.decode(encoding) + return resource_bytes(package, resource + '.cfg').decode('utf-8') + with open(path, 'r', encoding='utf-8') as fp: + return fp.read() def external_configuration(path): @@ -308,7 +312,7 @@ def external_configuration(path): """ # Is the context coming from a file system or Python path? cfg_path = expand_path(path) - parser = SafeConfigParser() + parser = ConfigParser() files = parser.read(cfg_path) if files != [cfg_path]: raise MissingConfigurationFileError(path) diff --git a/src/mailman/config/mailman.cfg b/src/mailman/config/mailman.cfg index 24e81ec91..aea420280 100644 --- a/src/mailman/config/mailman.cfg +++ b/src/mailman/config/mailman.cfg @@ -23,9 +23,13 @@ # /var/tmp/mailman [paths.dev] -# Convenient development layout where everything is put in the current -# directory. -var_dir: var +# Convenient development layout where everything is put in a directory above +# where the mailman.cfg file lives. +var_dir: $cfg_file/../.. + +[paths.here] +# Layout where the var directory is put in the current working directory. +var_dir: $cwd/var [paths.fhs] # Filesystem Hiearchy Standard 2.3 diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg index c7a63e794..4a896eec5 100644 --- a/src/mailman/config/schema.cfg +++ b/src/mailman/config/schema.cfg @@ -59,7 +59,7 @@ pre_hook: post_hook: # Which paths.* file system layout to use. -layout: dev +layout: here # Can MIME filtered messages be preserved by list owners? filtered_messages_are_preservable: no diff --git a/src/mailman/config/tests/test_archivers.py b/src/mailman/config/tests/test_archivers.py index 08e466878..b74f680d9 100644 --- a/src/mailman/config/tests/test_archivers.py +++ b/src/mailman/config/tests/test_archivers.py @@ -17,9 +17,6 @@ """Site-wide archiver configuration tests.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestArchivers', ] diff --git a/src/mailman/config/tests/test_configuration.py b/src/mailman/config/tests/test_configuration.py index f3a49d64f..253b63239 100644 --- a/src/mailman/config/tests/test_configuration.py +++ b/src/mailman/config/tests/test_configuration.py @@ -17,9 +17,6 @@ """Test the system-wide global configuration.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestConfiguration', 'TestConfigurationErrors', @@ -32,6 +29,7 @@ import mock import tempfile import unittest +from contextlib import ExitStack from mailman.config.config import ( Configuration, external_configuration, load_external) from mailman.interfaces.configuration import ( @@ -65,26 +63,13 @@ class TestConfiguration(unittest.TestCase): class TestExternal(unittest.TestCase): """Test external configuration file loading APIs.""" - def test_load_external_by_filename_as_bytes(self): + def test_load_external_by_filename(self): filename = resource_filename('mailman.config', 'postfix.cfg') contents = load_external(filename) - self.assertIsInstance(contents, bytes) - self.assertEqual(contents[:9], b'[postfix]') - - def test_load_external_by_path_as_bytes(self): - contents = load_external('python:mailman.config.postfix') - self.assertIsInstance(contents, bytes) - self.assertEqual(contents[:9], b'[postfix]') - - def test_load_external_by_filename_as_string(self): - filename = resource_filename('mailman.config', 'postfix.cfg') - contents = load_external(filename, encoding='utf-8') - self.assertIsInstance(contents, unicode) self.assertEqual(contents[:9], '[postfix]') - def test_load_external_by_path_as_string(self): - contents = load_external('python:mailman.config.postfix', 'utf-8') - self.assertIsInstance(contents, unicode) + def test_load_external_by_path(self): + contents = load_external('python:mailman.config.postfix') self.assertEqual(contents[:9], '[postfix]') def test_external_configuration_by_filename(self): @@ -121,24 +106,32 @@ layout: nonesuch # Use a fake sys.exit() function that records that it was called, and # that prevents further processing. config = Configuration() - # Suppress warning messages in the test output. - with self.assertRaises(SystemExit) as cm, mock.patch('sys.stderr'): + # Suppress warning messages in the test output. Also, make sure that + # the config.load() call doesn't break global state. + with ExitStack() as resources: + resources.enter_context(mock.patch('sys.stderr')) + resources.enter_context(mock.patch.object(config, '_clear')) + cm = resources.enter_context(self.assertRaises(SystemExit)) config.load(filename) self.assertEqual(cm.exception.args, (1,)) def test_path_expansion_infloop(self): - # A path expansion never completes because it references a - # non-existent substitution variable. + # A path expansion never completes because it references a non-existent + # substitution variable. fd, filename = tempfile.mkstemp() self.addCleanup(os.remove, filename) os.close(fd) with open(filename, 'w') as fp: print("""\ -[paths.dev] +[paths.here] log_dir: $nopath/log_dir """, file=fp) config = Configuration() - # Suppress warning messages in the test output. - with self.assertRaises(SystemExit) as cm, mock.patch('sys.stderr'): + # Suppress warning messages in the test output. Also, make sure that + # the config.load() call doesn't break global state. + with ExitStack() as resources: + resources.enter_context(mock.patch('sys.stderr')) + resources.enter_context(mock.patch.object(config, '_clear')) + cm = resources.enter_context(self.assertRaises(SystemExit)) config.load(filename) self.assertEqual(cm.exception.args, (1,)) diff --git a/src/mailman/core/chains.py b/src/mailman/core/chains.py index df4c199d5..610c396b0 100644 --- a/src/mailman/core/chains.py +++ b/src/mailman/core/chains.py @@ -17,21 +17,17 @@ """Application support for chain processing.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'initialize', 'process', ] -from zope.interface.verify import verifyObject - from mailman.chains.base import Chain, TerminalChainBase from mailman.config import config from mailman.interfaces.chain import LinkAction, IChain from mailman.utilities.modules import find_components +from zope.interface.verify import verifyObject diff --git a/src/mailman/core/constants.py b/src/mailman/core/constants.py index f8e354199..63fa0d0d8 100644 --- a/src/mailman/core/constants.py +++ b/src/mailman/core/constants.py @@ -17,21 +17,17 @@ """Various constants and enumerations.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'system_preferences', ] -from zope.component import getUtility -from zope.interface import implementer - from mailman.config import config from mailman.interfaces.languages import ILanguageManager from mailman.interfaces.member import DeliveryMode, DeliveryStatus from mailman.interfaces.preferences import IPreferences +from zope.component import getUtility +from zope.interface import implementer diff --git a/src/mailman/core/docs/runner.rst b/src/mailman/core/docs/runner.rst index e9fd21c57..11a771fe8 100644 --- a/src/mailman/core/docs/runner.rst +++ b/src/mailman/core/docs/runner.rst @@ -55,7 +55,7 @@ on instance variables. ... A test message. ... """) >>> switchboard = config.switchboards['test'] - >>> filebase = switchboard.enqueue(msg, listname=mlist.fqdn_listname, + >>> filebase = switchboard.enqueue(msg, listid=mlist.list_id, ... foo='yes', bar='no') >>> runner.run() >>> print(runner.msg.as_string()) @@ -69,7 +69,7 @@ on instance variables. bar : no foo : yes lang : en - listname : test@example.com + listid : test.example.com version : 3 XXX More of the Runner API should be tested. diff --git a/src/mailman/core/errors.py b/src/mailman/core/errors.py index b8f5a1f64..95b1ae821 100644 --- a/src/mailman/core/errors.py +++ b/src/mailman/core/errors.py @@ -26,9 +26,6 @@ interfaces. """ -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'AlreadyReceivingDigests', 'AlreadyReceivingRegularDeliveries', diff --git a/src/mailman/core/i18n.py b/src/mailman/core/i18n.py index b078a985f..ae9dcc8b8 100644 --- a/src/mailman/core/i18n.py +++ b/src/mailman/core/i18n.py @@ -17,9 +17,6 @@ """Internationalization.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ '_', 'ctime', @@ -28,11 +25,12 @@ __all__ = [ import time -from flufl.i18n import PackageStrategy, registry - import mailman.messages + +from flufl.i18n import PackageStrategy, registry from mailman.interfaces.configuration import ConfigurationUpdatedEvent + _ = None diff --git a/src/mailman/core/initialize.py b/src/mailman/core/initialize.py index 3a0e27024..47d7106e2 100644 --- a/src/mailman/core/initialize.py +++ b/src/mailman/core/initialize.py @@ -24,9 +24,6 @@ line argument parsing, since some of the initialization behavior is controlled by the command line arguments. """ -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'initialize', 'initialize_1', @@ -38,16 +35,15 @@ __all__ = [ import os import sys - -from pkg_resources import resource_string -from zope.component import getUtility -from zope.configuration import xmlconfig - import mailman.config.config import mailman.core.logging from mailman.interfaces.database import IDatabaseFactory from mailman.utilities.modules import call_name +from pkg_resources import resource_string as resource_bytes +from zope.component import getUtility +from zope.configuration import xmlconfig + # The test infrastructure uses this to prevent the search and loading of any # existing configuration file. Otherwise the existence of say a @@ -109,8 +105,8 @@ def initialize_1(config_path=None): :param config_path: The path to the configuration file. :type config_path: string """ - zcml = resource_string('mailman.config', 'configure.zcml') - xmlconfig.string(zcml) + zcml = resource_bytes('mailman.config', 'configure.zcml') + xmlconfig.string(zcml.decode('utf-8')) # By default, set the umask so that only owner and group can read and # write our files. Specifically we must have g+rw and we probably want # o-rwx although I think in most cases it doesn't hurt if other can read diff --git a/src/mailman/core/logging.py b/src/mailman/core/logging.py index c5ce1a538..7529cc1d7 100644 --- a/src/mailman/core/logging.py +++ b/src/mailman/core/logging.py @@ -17,9 +17,6 @@ """Logging initialization, using Python's standard logging package.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'initialize', 'reopen', diff --git a/src/mailman/core/pipelines.py b/src/mailman/core/pipelines.py index e164169a4..b7773736c 100644 --- a/src/mailman/core/pipelines.py +++ b/src/mailman/core/pipelines.py @@ -17,9 +17,6 @@ """Built-in pipelines.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'BasePipeline', 'OwnerPipeline', @@ -32,9 +29,6 @@ __all__ = [ import logging -from zope.interface import implementer -from zope.interface.verify import verifyObject - from mailman.app.bounces import bounce_message from mailman.config import config from mailman.core import errors @@ -42,6 +36,8 @@ from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler from mailman.interfaces.pipeline import IPipeline from mailman.utilities.modules import find_components +from zope.interface import implementer +from zope.interface.verify import verifyObject dlog = logging.getLogger('mailman.debug') @@ -120,6 +116,7 @@ class PostingPipeline(BasePipeline): 'cleanse', 'cleanse-dkim', 'cook-headers', + 'subject-prefix', 'rfc-2369', 'to-archive', 'to-digest', diff --git a/src/mailman/core/rules.py b/src/mailman/core/rules.py index 1a2b9f56d..0110c07f7 100644 --- a/src/mailman/core/rules.py +++ b/src/mailman/core/rules.py @@ -17,19 +17,15 @@ """Various rule helpers""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'initialize', ] -from zope.interface.verify import verifyObject - from mailman.config import config from mailman.interfaces.rules import IRule from mailman.utilities.modules import find_components +from zope.interface.verify import verifyObject diff --git a/src/mailman/core/runner.py b/src/mailman/core/runner.py index 81a2ea3d1..5ffc3f57d 100644 --- a/src/mailman/core/runner.py +++ b/src/mailman/core/runner.py @@ -17,9 +17,6 @@ """The process runner base class.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Runner', ] @@ -30,12 +27,7 @@ import signal import logging import traceback -from cStringIO import StringIO from lazr.config import as_boolean, as_timedelta -from zope.component import getUtility -from zope.event import notify -from zope.interface import implementer - from mailman.config import config from mailman.core.i18n import _ from mailman.core.logging import reopen @@ -44,6 +36,10 @@ from mailman.interfaces.languages import ILanguageManager from mailman.interfaces.listmanager import IListManager from mailman.interfaces.runner import IRunner, RunnerCrashEvent from mailman.utilities.string import expand +from six.moves import cStringIO as StringIO +from zope.component import getUtility +from zope.event import notify +from zope.interface import implementer dlog = logging.getLogger('mailman.debug') @@ -218,16 +214,26 @@ class Runner: # them out of our sight. # # Find out which mailing list this message is destined for. + mlist = None missing = object() - listname = msgdata.get('listname', missing) - mlist = (None - if listname is missing - else getUtility(IListManager).get(unicode(listname))) + # First try to dig out the target list by id. If there's no list-id + # in the metadata, fall back to the fqdn list name for backward + # compatibility. + list_manager = getUtility(IListManager) + list_id = msgdata.get('listid', missing) + fqdn_listname = None + if list_id is missing: + fqdn_listname = msgdata.get('listname', missing) + # XXX Deprecate. + if fqdn_listname is not missing: + mlist = list_manager.get(fqdn_listname) + else: + mlist = list_manager.get_by_list_id(list_id) if mlist is None: + identifier = (list_id if list_id is not None else fqdn_listname) elog.error( '%s runner "%s" shunting message for missing list: %s', - msg['message-id'], self.name, - ('n/a' if listname is missing else listname)) + msg['message-id'], self.name, identifier) config.switchboards['shunt'].enqueue(msg, msgdata) return # Now process this message. We also want to set up the language diff --git a/src/mailman/core/switchboard.py b/src/mailman/core/switchboard.py index 2e8ef24a7..f54bc243a 100644 --- a/src/mailman/core/switchboard.py +++ b/src/mailman/core/switchboard.py @@ -24,9 +24,6 @@ written. First, the message is written to the pickle, then the metadata dictionary is written. """ -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Switchboard', 'handle_ConfigurationUpdatedEvent', @@ -37,22 +34,22 @@ import os import time import email import pickle -import cPickle import hashlib import logging -from zope.interface import implementer - from mailman.config import config from mailman.email.message import Message from mailman.interfaces.configuration import ConfigurationUpdatedEvent from mailman.interfaces.switchboard import ISwitchboard from mailman.utilities.filesystem import makedirs from mailman.utilities.string import expand +from six.moves import cPickle +from zope.interface import implementer -# 20 bytes of all bits set, maximum hashlib.sha.digest() value. -shamax = 0xffffffffffffffffffffffffffffffffffffffffL +# 20 bytes of all bits set, maximum hashlib.sha.digest() value. We do it this +# way for Python 2/3 compatibility. +shamax = int('0xffffffffffffffffffffffffffffffffffffffff', 16) # Small increment to add to time in case two entries have the same time. This # prevents skipping one of two entries with the same time until the next pass. DELTA = .0001 @@ -92,7 +89,7 @@ class Switchboard: self.queue_directory = queue_directory # If configured to, create the directory if it doesn't yet exist. if config.create_paths: - makedirs(self.queue_directory, 0770) + makedirs(self.queue_directory, 0o770) # Fast track for no slices self._lower = None self._upper = None @@ -112,37 +109,37 @@ class Switchboard: # of parallel runner processes. data = _metadata.copy() data.update(_kws) - listname = data.get('listname', '--nolist--') + list_id = data.get('listid', '--nolist--') # Get some data for the input to the sha hash. - now = time.time() + now = repr(time.time()) if data.get('_plaintext'): protocol = 0 msgsave = cPickle.dumps(str(_msg), protocol) else: protocol = pickle.HIGHEST_PROTOCOL msgsave = cPickle.dumps(_msg, protocol) - # listname is unicode but the input to the hash function must be an - # 8-bit string (eventually, a bytes object). - hashfood = msgsave + listname.encode('utf-8') + repr(now) + # The list-id field is a string but the input to the hash function must + # be bytes. + hashfood = msgsave + list_id.encode('utf-8') + now.encode('utf-8') # Encode the current time into the file name for FIFO sorting. The # file name consists of two parts separated by a '+': the received # time for this message (i.e. when it first showed up on this system) # and the sha hex digest. - filebase = repr(now) + '+' + hashlib.sha1(hashfood).hexdigest() + filebase = now + '+' + hashlib.sha1(hashfood).hexdigest() filename = os.path.join(self.queue_directory, filebase + '.pck') tmpfile = filename + '.tmp' # Always add the metadata schema version number data['version'] = config.QFILE_SCHEMA_VERSION # Filter out volatile entries. Use .keys() so that we can mutate the # dictionary during the iteration. - for k in data.keys(): + for k in list(data): if k.startswith('_'): del data[k] # We have to tell the dequeue() method whether to parse the message # object or not. data['_parsemsg'] = (protocol == 0) # Write to the pickle file the message object and metadata. - with open(tmpfile, 'w') as fp: + with open(tmpfile, 'wb') as fp: fp.write(msgsave) cPickle.dump(data, fp, protocol) fp.flush() @@ -156,7 +153,7 @@ class Switchboard: filename = os.path.join(self.queue_directory, filebase + '.pck') backfile = os.path.join(self.queue_directory, filebase + '.bak') # Read the message object and metadata. - with open(filename) as fp: + with open(filename, 'rb') as fp: # Move the file to the backup file name for processing. If this # process crashes uncleanly the .bak file will be used to # re-instate the .pck file in order to try again. @@ -207,13 +204,13 @@ class Switchboard: # Throw out any files which don't match our bitrange. BAW: test # performance and end-cases of this algorithm. MAS: both # comparisons need to be <= to get complete range. - if lower is None or (lower <= long(digest, 16) <= upper): + if lower is None or (lower <= int(digest, 16) <= upper): key = float(when) while key in times: key += DELTA times[key] = filebase # FIFO sort - return [times[key] for key in sorted(times)] + return [times[k] for k in sorted(times)] def recover_backup_files(self): """See `ISwitchboard`.""" @@ -228,7 +225,8 @@ class Switchboard: dst = os.path.join(self.queue_directory, filebase + '.pck') with open(src, 'rb+') as fp: try: - msg = cPickle.load(fp) + # Throw away the message object. + cPickle.load(fp) data_pos = fp.tell() data = cPickle.load(fp) except Exception as error: diff --git a/src/mailman/core/system.py b/src/mailman/core/system.py index 495cc37ee..0c01d94aa 100644 --- a/src/mailman/core/system.py +++ b/src/mailman/core/system.py @@ -17,9 +17,6 @@ """System information.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'system', ] @@ -27,10 +24,9 @@ __all__ = [ import sys -from zope.interface import implementer - from mailman import version from mailman.interfaces.system import ISystem +from zope.interface import implementer diff --git a/src/mailman/core/tests/test_pipelines.py b/src/mailman/core/tests/test_pipelines.py index 67e6af36e..91be1f79f 100644 --- a/src/mailman/core/tests/test_pipelines.py +++ b/src/mailman/core/tests/test_pipelines.py @@ -17,9 +17,6 @@ """Test the core modification pipelines.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestOwnerPipeline', 'TestPostingPipeline', @@ -28,9 +25,6 @@ __all__ = [ import unittest -from zope.component import getUtility -from zope.interface import implementer - from mailman.app.lifecycle import create_list from mailman.config import config from mailman.core.errors import DiscardMessage, RejectMessage @@ -40,11 +34,11 @@ from mailman.interfaces.member import MemberRole from mailman.interfaces.pipeline import IPipeline from mailman.interfaces.usermanager import IUserManager from mailman.testing.helpers import ( - LogFileMark, - get_queue_messages, - reset_the_world, + LogFileMark, get_queue_messages, reset_the_world, specialized_message_from_string as mfs) from mailman.testing.layers import ConfigLayer +from zope.component import getUtility +from zope.interface import implementer @@ -175,5 +169,5 @@ To: test-owner@example.com pipeline_name='default-owner-pipeline') messages = get_queue_messages('out', sort_on='to') self.assertEqual(len(messages), 1) - self.assertEqual(messages[0].msgdata['recipients'], + self.assertEqual(messages[0].msgdata['recipients'], set(('anne@example.com', 'bart@example.com'))) diff --git a/src/mailman/core/tests/test_runner.py b/src/mailman/core/tests/test_runner.py index 1fb8f0b7b..3d2e76096 100644 --- a/src/mailman/core/tests/test_runner.py +++ b/src/mailman/core/tests/test_runner.py @@ -17,9 +17,6 @@ """Test some Runner base class behavior.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestRunner', ] @@ -70,7 +67,7 @@ To: test@example.com Message-ID: <ant> """) - config.switchboards['in'].enqueue(msg, listname='test@example.com') + config.switchboards['in'].enqueue(msg, listid='test.example.com') with event_subscribers(self._got_event): runner.run() # We should now have exactly one event, which will contain the @@ -81,7 +78,7 @@ Message-ID: <ant> self.assertTrue(isinstance(event, RunnerCrashEvent)) self.assertEqual(event.mailing_list, self._mlist) self.assertEqual(event.message['message-id'], '<ant>') - self.assertEqual(event.metadata['listname'], 'test@example.com') + self.assertEqual(event.metadata['listid'], 'test.example.com') self.assertTrue(isinstance(event.error, RuntimeError)) self.assertEqual(str(event.error), 'borked') self.assertTrue(isinstance(event.runner, CrashingRunner)) diff --git a/src/mailman/database/alembic/__init__.py b/src/mailman/database/alembic/__init__.py index ffd3af6df..4dbbc31d9 100644 --- a/src/mailman/database/alembic/__init__.py +++ b/src/mailman/database/alembic/__init__.py @@ -17,9 +17,6 @@ """Alembic configuration initization.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'alembic_cfg', ] diff --git a/src/mailman/database/alembic/env.py b/src/mailman/database/alembic/env.py index 125868566..261782d29 100644 --- a/src/mailman/database/alembic/env.py +++ b/src/mailman/database/alembic/env.py @@ -17,9 +17,6 @@ """Alembic migration environment.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'run_migrations_offline', 'run_migrations_online', @@ -28,11 +25,10 @@ __all__ = [ from alembic import context from contextlib import closing -from sqlalchemy import create_engine - from mailman.config import config from mailman.database.model import Model from mailman.utilities.string import expand +from sqlalchemy import create_engine diff --git a/src/mailman/database/alembic/versions/51b7f92bd06c_initial.py b/src/mailman/database/alembic/versions/51b7f92bd06c_initial.py index 3feb24fff..5e3527abe 100644 --- a/src/mailman/database/alembic/versions/51b7f92bd06c_initial.py +++ b/src/mailman/database/alembic/versions/51b7f92bd06c_initial.py @@ -29,9 +29,6 @@ Revises: None Create Date: 2014-10-10 09:53:35.624472 """ -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'downgrade', 'upgrade', diff --git a/src/mailman/database/base.py b/src/mailman/database/base.py index 55edf6005..09fd47b80 100644 --- a/src/mailman/database/base.py +++ b/src/mailman/database/base.py @@ -15,9 +15,8 @@ # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see <http://www.gnu.org/licenses/>. -from __future__ import absolute_import, print_function, unicode_literals +"""Common database support.""" -__metaclass__ = type __all__ = [ 'SABaseDatabase', ] @@ -25,17 +24,15 @@ __all__ = [ import logging -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from zope.interface import implementer - from mailman.config import config from mailman.interfaces.database import IDatabase from mailman.utilities.string import expand +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from zope.interface import implementer log = logging.getLogger('mailman.database') -NL = '\n' diff --git a/src/mailman/database/factory.py b/src/mailman/database/factory.py index 64174449d..9fffd4545 100644 --- a/src/mailman/database/factory.py +++ b/src/mailman/database/factory.py @@ -17,9 +17,6 @@ """Database factory.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'DatabaseFactory', 'DatabaseTestingFactory', @@ -33,16 +30,15 @@ import alembic.command from alembic.migration import MigrationContext from alembic.script import ScriptDirectory from flufl.lock import Lock -from sqlalchemy import MetaData -from zope.interface import implementer -from zope.interface.verify import verifyObject - from mailman.config import config from mailman.database.alembic import alembic_cfg from mailman.database.model import Model from mailman.interfaces.database import ( DatabaseError, IDatabase, IDatabaseFactory) from mailman.utilities.modules import call_name +from sqlalchemy import MetaData +from zope.interface import implementer +from zope.interface.verify import verifyObject LAST_STORM_SCHEMA_VERSION = '20130406000000' diff --git a/src/mailman/database/model.py b/src/mailman/database/model.py index a6056bf63..8dad6f0cf 100644 --- a/src/mailman/database/model.py +++ b/src/mailman/database/model.py @@ -17,9 +17,6 @@ """Base class for all database classes.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Model', ] diff --git a/src/mailman/database/postgresql.py b/src/mailman/database/postgresql.py index 717b69dd1..4a6f02da6 100644 --- a/src/mailman/database/postgresql.py +++ b/src/mailman/database/postgresql.py @@ -17,9 +17,6 @@ """PostgreSQL database support.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'PostgreSQLDatabase', ] diff --git a/src/mailman/database/sqlite.py b/src/mailman/database/sqlite.py index db7860390..95dba460e 100644 --- a/src/mailman/database/sqlite.py +++ b/src/mailman/database/sqlite.py @@ -17,9 +17,6 @@ """SQLite database support.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'SQLiteDatabase', ] @@ -28,7 +25,7 @@ __all__ = [ import os from mailman.database.base import SABaseDatabase -from urlparse import urlparse +from six.moves.urllib_parse import urlparse diff --git a/src/mailman/database/tests/test_factory.py b/src/mailman/database/tests/test_factory.py index 29cca41ba..71f810a56 100644 --- a/src/mailman/database/tests/test_factory.py +++ b/src/mailman/database/tests/test_factory.py @@ -17,9 +17,6 @@ """Test database schema migrations""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestSchemaManager', ] @@ -28,17 +25,16 @@ __all__ = [ import unittest import alembic.command -from mock import patch -from sqlalchemy import MetaData, Table, Column, Integer, Unicode -from sqlalchemy.exc import ProgrammingError, OperationalError -from sqlalchemy.schema import Index - from mailman.config import config from mailman.database.alembic import alembic_cfg from mailman.database.factory import LAST_STORM_SCHEMA_VERSION, SchemaManager from mailman.database.model import Model from mailman.interfaces.database import DatabaseError from mailman.testing.layers import ConfigLayer +from mock import patch +from sqlalchemy import MetaData, Table, Column, Integer, Unicode +from sqlalchemy.exc import ProgrammingError, OperationalError +from sqlalchemy.schema import Index diff --git a/src/mailman/database/transaction.py b/src/mailman/database/transaction.py index 3e156cfb8..dc468aaab 100644 --- a/src/mailman/database/transaction.py +++ b/src/mailman/database/transaction.py @@ -17,9 +17,6 @@ """Transactional support.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'dbconnection', 'transaction', @@ -28,7 +25,6 @@ __all__ = [ from contextlib import contextmanager - from mailman.config import config diff --git a/src/mailman/database/types.py b/src/mailman/database/types.py index 1984b08b5..463d271f0 100644 --- a/src/mailman/database/types.py +++ b/src/mailman/database/types.py @@ -15,17 +15,14 @@ # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see <http://www.gnu.org/licenses/>. -"""Storm type conversions.""" +"""Database type conversions.""" - -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Enum', 'UUID', ] + import uuid from sqlalchemy import Integer diff --git a/src/mailman/docs/DEVELOP.rst b/src/mailman/docs/DEVELOP.rst index c9e1bd596..f1225658e 100644 --- a/src/mailman/docs/DEVELOP.rst +++ b/src/mailman/docs/DEVELOP.rst @@ -3,7 +3,9 @@ Developing Mailman ================== The following documentation is generated from the internal developer -documentation. This documentation is also used by the test suite. +documentation. This documentation is also used by the test suite. Another +good source of architectural information is available in the chapter written +by Barry Warsaw for the `Architecture of Open Source Applications`_. For now, this will have to suffice as an overview of the Mailman system. @@ -154,3 +156,4 @@ extensive set of command line commands, and email commands. .. _`Python pickles`: http://docs.python.org/2/library/pickle.html +.. _`Architecture of Open Source Applications`: http://www.aosabook.org/en/mailman.html diff --git a/src/mailman/docs/INTRODUCTION.rst b/src/mailman/docs/INTRODUCTION.rst index ac77d0e72..b4f016c46 100644 --- a/src/mailman/docs/INTRODUCTION.rst +++ b/src/mailman/docs/INTRODUCTION.rst @@ -82,7 +82,7 @@ lists and archives, etc., are available at: Requirements ============ -Mailman 3.0 requires `Python 2.7`_. +Mailman 3 requires `Python 3.4`_ or newer. .. _`GNU Mailman`: http://www.list.org @@ -90,4 +90,4 @@ Mailman 3.0 requires `Python 2.7`_. .. _`Getting Started`: START.html .. _Python: http://www.python.org .. _FAQ: http://wiki.list.org/display/DOC/Frequently+Asked+Questions -.. _`Python 2.7`: http://www.python.org/download/releases/2.7.3/ +.. _`Python 3.4`: https://www.python.org/downloads/release/python-342/ diff --git a/src/mailman/docs/NEWS.rst b/src/mailman/docs/NEWS.rst index 651e4b98f..9115c9bdb 100644 --- a/src/mailman/docs/NEWS.rst +++ b/src/mailman/docs/NEWS.rst @@ -12,6 +12,22 @@ Here is a history of user visible changes to Mailman. =============================== (2015-XX-XX) +Configuration +------------- + * When specifying a file system path in the [paths.*] section, $cfg_file can + be used to expand into the path of the ``-C`` option if given. In the + default ``[paths.dev]`` section, ``$var_dir`` is now specified relative to + ``$cfg_file`` so that it won't accidentally be relative to the current + working directory, if ``-C`` is given. + * ``$cwd`` is now an additional substitution variable for the ``mailman.cfg`` + file's ``[paths.*]`` sections. A new ``[paths.here]`` section is added, + which puts the ``var_dir`` in ``$cwd``. It is made the default layout. + +REST +---- + * You can now view the contents of, inject messages into, and delete messages + from the various queue directories via the ``<api>/queues`` resource. + 3.0 beta 5 -- "Carve Away The Stone" ==================================== @@ -49,6 +65,7 @@ Database Development ----------- + * Python 3.4 is now the minimum requirement. * You no longer have to create a virtual environment separately when running the test suite. Just use `tox`. * You no longer have to edit `src/mailman/testing/testing.cfg` to run the diff --git a/src/mailman/docs/START.rst b/src/mailman/docs/START.rst index 794740c64..454f6a387 100644 --- a/src/mailman/docs/START.rst +++ b/src/mailman/docs/START.rst @@ -39,12 +39,11 @@ list, or ask on IRC channel ``#mailman`` on Freenode. Requirements ============ -Python 2.7 is required. It can either be the default 'python' on your -``$PATH`` or it can be accessible via the ``python2.7`` binary. If -your operating system does not include Python, see http://www.python.org -for information about downloading installers (where available) and -installing it from source (when necessary or preferred). Python 3 is -not yet supported. +Python 3.4 or newer is required. It can either be the default 'python3' on +your ``$PATH`` or it can be accessible via the ``python3.4`` binary. If your +operating system does not include Python, see http://www.python.org for +information about downloading installers (where available) and installing it +from source (when necessary or preferred). Python 2 is not supported. You may need some additional dependencies, which are either available from your OS vendor, or can be downloaded automatically from the `Python @@ -80,9 +79,9 @@ downloads everything from the Cheeseshop. You do have access to the virtualenv, and you can use this to run individual tests, e.g.:: - $ .tox/py27/bin/python -m nose2 -vv -P user + $ .tox/py34/bin/python -m nose2 -vv -P user -Use `.tox/py27/bin/python -m nose2 --help` for more options. +Use `.tox/py34/bin/python -m nose2 --help` for more options. If you want to run the full test suite against the PostgreSQL database, set the database up as described in :doc:`DATABASE`, then create a `postgres.cfg` @@ -112,23 +111,23 @@ installed. First, create a virtual environment. By default ``virtualenv`` uses the ``python`` executable it finds first on your ``$PATH``. Make sure this is -Python 2.7 (just start the interactive interpreter and check the version in +Python 3.4 (just start the interactive interpreter and check the version in the startup banner). The directory you install the virtualenv into is up to -you, but for purposes of this document, we'll install it into ``/tmp/py27``:: +you, but for purposes of this document, we'll install it into ``/tmp/mm3``:: - % virtualenv --system-site-packages /tmp/py27 + % virtualenv -p python3 --system-site-packages /tmp/mm3 -If your default Python is not version 2.7, use the ``--python`` option to +If your default Python is not version 3.4, use the ``--python`` option to specify the Python executable. You can use the command name if this version is on your ``PATH``:: - % virtualenv --system-site-packages --python=python2.7 /tmp/py27 + % virtualenv --system-site-packages --python=python3.4 /tmp/mm3 -or you may specify the full path to any Python 2.7 executable. +or you may specify the full path to any Python 3.4 executable. Now, activate the virtual environment and set it up for development:: - % source /tmp/py27/bin/activate + % source /tmp/mm3/bin/activate % python setup.py develop Sit back and have some Kombucha while you wait for everything to download and diff --git a/src/mailman/docs/STYLEGUIDE.rst b/src/mailman/docs/STYLEGUIDE.rst index 13fb0cdf1..1d63d2b46 100644 --- a/src/mailman/docs/STYLEGUIDE.rst +++ b/src/mailman/docs/STYLEGUIDE.rst @@ -15,33 +15,25 @@ http://barry.warsaw.us/software/STYLEGUIDE.txt This document contains a style guide for Python programming, as used in GNU Mailman. `PEP 8`_ is the basis for this style guide so it's recommendations should be followed except for the differences outlined here. This document -assumes the use of Python 2.7, but not (yet) Python 3. +assumes the use of Python 3. -* After file comments (e.g. license block), add a ``__metaclass__`` definition - so that all classes will be new-style. Following that, add an ``__all__`` - section that names, one-per-line, all the public names exported by this - module. You should enable absolute imports and unicode literals. See the +* After file comments (e.g. license block), add an ``__all__`` section that + names, one-per-line, all the public names exported by this module. See the `GNU Mailman Python template`_ as an example. * Imports are always put at the top of the file, just after any module comments and docstrings, and before module globals and constants, but after - any ``__future__`` imports, or ``__metaclass__`` and ``__all__`` - definitions. + any ``__all__`` definitions. Imports should be grouped, with the order being: - 1. non-from imports for standard and third party libraries - 2. non-from imports from the application - 3. from-imports from the standard and third party libraries - 4. from-imports from the application + 1. non-from imports, grouped from shorted module name to longest module + name, with ties being broken by alphabetical order. + 3. from-imports grouped alphabetically. - From-imports should follow non-from imports. Dotted imports should follow - non-dotted imports. Non-dotted imports should be grouped by increasing - length, while dotted imports should be grouped alphabetically. - -* In general, there should be one class per module. Keep files small, but - it's okay to group related code together. List everything exported from the - module in the ``__all__``. +* In general, there should be one class per module. This is not a + hard-and-fast rule. Keep files small, but it's okay to group related code + together. List everything exported from the module in the ``__all__``. * Right hanging comments are discouraged, in favor of preceding comments. E.g. bad:: diff --git a/src/mailman/docs/__init__.py b/src/mailman/docs/__init__.py index f588eb14d..fa09dde76 100644 --- a/src/mailman/docs/__init__.py +++ b/src/mailman/docs/__init__.py @@ -17,9 +17,6 @@ """General Mailman doc tests.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'layer', ] diff --git a/src/mailman/email/message.py b/src/mailman/email/message.py index e653133ba..d4b373bea 100644 --- a/src/mailman/email/message.py +++ b/src/mailman/email/message.py @@ -23,9 +23,6 @@ safe pickle deserialization, even if the email package adds additional Message attributes. """ -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Message', 'MultipartDigestMessage', @@ -40,7 +37,6 @@ import email.utils from email.header import Header from email.mime.multipart import MIMEMultipart - from mailman.config import config @@ -149,8 +145,8 @@ class UserNotification(Message): subject = ('(no subject)' if subject is None else subject) if text is not None: self.set_payload(text.encode(charset), charset) - self['Subject'] = Header(subject.encode(charset), charset, - header_name='Subject', errors='replace') + self['Subject'] = Header( + subject, charset, header_name='Subject', errors='replace') self['From'] = sender if isinstance(recipients, (list, set, tuple)): self['To'] = COMMASPACE.join(recipients) @@ -198,7 +194,7 @@ class UserNotification(Message): reduced_list_headers=True, ) if mlist is not None: - enqueue_kws['listname'] = mlist.fqdn_listname + enqueue_kws['listid'] = mlist.list_id enqueue_kws.update(_kws) virginq.enqueue(self, **enqueue_kws) @@ -227,7 +223,7 @@ class OwnerNotification(UserNotification): virginq = config.switchboards['virgin'] # The message metadata better have a `recip' attribute virginq.enqueue(self, - listname=mlist.fqdn_listname, + listid=mlist.list_id, recipients=self.recipients, nodecorate=True, reduced_list_headers=True, diff --git a/src/mailman/email/tests/test_message.py b/src/mailman/email/tests/test_message.py index 1fdef5e86..59335b890 100644 --- a/src/mailman/email/tests/test_message.py +++ b/src/mailman/email/tests/test_message.py @@ -17,9 +17,6 @@ """Test the message API.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestMessage', 'TestMessageSubclass', @@ -27,8 +24,8 @@ __all__ = [ import unittest -from email.parser import FeedParser +from email.parser import FeedParser from mailman.app.lifecycle import create_list from mailman.email.message import Message, UserNotification from mailman.testing.helpers import get_queue_messages @@ -66,7 +63,7 @@ class TestMessage(unittest.TestCase): class TestMessageSubclass(unittest.TestCase): def test_i18n_filenames(self): parser = FeedParser(_factory=Message) - parser.feed(b"""\ + parser.feed("""\ Message-ID: <blah@example.com> Content-Type: multipart/mixed; boundary="------------050607040206050605060208" @@ -88,6 +85,6 @@ Test content attachment = msg.get_payload(1) try: filename = attachment.get_filename() - except TypeError as e: - self.fail(e) + except TypeError as error: + self.fail(error) self.assertEqual(filename, u'd\xe9jeuner.txt') diff --git a/src/mailman/email/validate.py b/src/mailman/email/validate.py index b4cf8b5e2..d6f664b01 100644 --- a/src/mailman/email/validate.py +++ b/src/mailman/email/validate.py @@ -17,9 +17,6 @@ """Email address validation.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Validator', ] @@ -27,11 +24,10 @@ __all__ = [ import re -from zope.interface import implementer - from mailman.interfaces.address import ( IEmailValidator, InvalidEmailAddressError) from mailman.utilities.email import split_email +from zope.interface import implementer # What other characters should be disallowed? diff --git a/src/mailman/handlers/acknowledge.py b/src/mailman/handlers/acknowledge.py index c3af9ab27..c10043981 100644 --- a/src/mailman/handlers/acknowledge.py +++ b/src/mailman/handlers/acknowledge.py @@ -20,23 +20,19 @@ This only happens if the sender has set their AcknowledgePosts attribute. """ -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Acknowledge', ] -from zope.component import getUtility -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.email.message import UserNotification from mailman.interfaces.handler import IHandler from mailman.interfaces.languages import ILanguageManager from mailman.utilities.i18n import make from mailman.utilities.string import oneline +from zope.component import getUtility +from zope.interface import implementer @@ -67,14 +63,13 @@ class Acknowledge: language = (language_manager[msgdata['lang']] if 'lang' in msgdata else member.preferred_language) - charset = language_manager[language.code].charset # Now get the acknowledgement template. display_name = mlist.display_name text = make('postack.txt', mailing_list=mlist, language=language.code, wrap=False, - subject=oneline(original_subject, charset), + subject=oneline(original_subject, in_unicode=True), list_name=mlist.list_name, display_name=display_name, listinfo_url=mlist.script_url('listinfo'), diff --git a/src/mailman/handlers/after_delivery.py b/src/mailman/handlers/after_delivery.py index 7fa7a4554..464fafd8c 100644 --- a/src/mailman/handlers/after_delivery.py +++ b/src/mailman/handlers/after_delivery.py @@ -17,19 +17,15 @@ """Perform some bookkeeping after a successful post.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'AfterDelivery', ] -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler from mailman.utilities.datetime import now +from zope.interface import implementer diff --git a/src/mailman/handlers/avoid_duplicates.py b/src/mailman/handlers/avoid_duplicates.py index 529a99f68..636a9f24d 100644 --- a/src/mailman/handlers/avoid_duplicates.py +++ b/src/mailman/handlers/avoid_duplicates.py @@ -23,19 +23,15 @@ has already received a copy, we either drop the message, add a duplicate warning header, or pass it through, depending on the user's preferences. """ -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'AvoidDuplicates', ] from email.utils import getaddresses, formataddr -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler +from zope.interface import implementer COMMASPACE = ', ' diff --git a/src/mailman/handlers/cleanse.py b/src/mailman/handlers/cleanse.py index 6b653bb34..0dad3077e 100644 --- a/src/mailman/handlers/cleanse.py +++ b/src/mailman/handlers/cleanse.py @@ -17,9 +17,6 @@ """Cleanse certain headers from all messages.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Cleanse', ] @@ -28,11 +25,10 @@ __all__ = [ import logging from email.utils import formataddr -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.handlers.cook_headers import uheader from mailman.interfaces.handler import IHandler +from zope.interface import implementer log = logging.getLogger('mailman.post') diff --git a/src/mailman/handlers/cleanse_dkim.py b/src/mailman/handlers/cleanse_dkim.py index 225666bf1..a4c16d31e 100644 --- a/src/mailman/handlers/cleanse_dkim.py +++ b/src/mailman/handlers/cleanse_dkim.py @@ -25,20 +25,16 @@ and it will also give the MTA the opportunity to regenerate valid keys originating at the Mailman server for the outgoing message. """ -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'CleanseDKIM', ] from lazr.config import as_boolean -from zope.interface import implementer - from mailman.config import config from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler +from zope.interface import implementer diff --git a/src/mailman/handlers/cook_headers.py b/src/mailman/handlers/cook_headers.py index d5d096448..44ef02e36 100644 --- a/src/mailman/handlers/cook_headers.py +++ b/src/mailman/handlers/cook_headers.py @@ -17,9 +17,6 @@ """Cook a message's headers.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'CookHeaders', ] @@ -27,21 +24,18 @@ __all__ = [ import re -from email.errors import HeaderParseError -from email.header import Header, decode_header, make_header +from email.header import Header from email.utils import parseaddr, formataddr, getaddresses -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler from mailman.interfaces.mailinglist import Personalization, ReplyToMunging from mailman.version import VERSION +from zope.interface import implementer COMMASPACE = ', ' MAXLINELEN = 78 - -nonascii = re.compile('[^\s!-~]') +NONASCII = re.compile('[^\s!-~]') @@ -54,12 +48,12 @@ def uheader(mlist, s, header_name=None, continuation_ws='\t', maxlinelen=None): specified. """ charset = mlist.preferred_language.charset - if nonascii.search(s): + if NONASCII.search(s): # use list charset but ... if charset == 'us-ascii': charset = 'iso-8859-1' else: - # there is no nonascii so ... + # there is no non-ascii so ... charset = 'us-ascii' return Header(s, charset, maxlinelen, header_name, continuation_ws) @@ -78,13 +72,6 @@ def process(mlist, msg, msgdata): msgdata['original_sender'] = msg.sender # VirginRunner sets _fasttrack for internally crafted messages. fasttrack = msgdata.get('_fasttrack') - if not msgdata.get('isdigest') and not fasttrack: - try: - prefix_subject(mlist, msg, msgdata) - except (UnicodeError, ValueError): - # TK: Sometimes subject header is not MIME encoded for 8bit - # simply abort prefixing. - pass # Add Precedence: and other useful headers. None of these are standard # and finding information on some of them are fairly difficult. Some are # just common practice, and we'll add more here as they become necessary. @@ -171,114 +158,6 @@ def process(mlist, msg, msgdata): -def prefix_subject(mlist, msg, msgdata): - """Maybe add a subject prefix. - - Add the subject prefix unless the message is a digest or is being fast - tracked (e.g. internally crafted, delivered to a single user such as the - list admin). - """ - if not mlist.subject_prefix.strip(): - return - prefix = mlist.subject_prefix - subject = msg.get('subject', '') - # Try to figure out what the continuation_ws is for the header - if isinstance(subject, Header): - lines = str(subject).splitlines() - else: - lines = subject.splitlines() - ws = '\t' - if len(lines) > 1 and lines[1] and lines[1][0] in ' \t': - ws = lines[1][0] - msgdata['original_subject'] = subject - # The subject may be multilingual but we take the first charset as major - # one and try to decode. If it is decodable, returned subject is in one - # line and cset is properly set. If fail, subject is mime-encoded and - # cset is set as us-ascii. See detail for ch_oneline() (CookHeaders one - # line function). - subject, cset = ch_oneline(subject) - # TK: Python interpreter has evolved to be strict on ascii charset code - # range. It is safe to use unicode string when manupilating header - # contents with re module. It would be best to return unicode in - # ch_oneline() but here is temporary solution. - subject = unicode(subject, cset) - # If the subject_prefix contains '%d', it is replaced with the - # mailing list sequential number. Sequential number format allows - # '%d' or '%05d' like pattern. - prefix_pattern = re.escape(prefix) - # unescape '%' :-< - prefix_pattern = '%'.join(prefix_pattern.split(r'\%')) - p = re.compile('%\d*d') - if p.search(prefix, 1): - # prefix have number, so we should search prefix w/number in subject. - # Also, force new style. - prefix_pattern = p.sub(r'\s*\d+\s*', prefix_pattern) - subject = re.sub(prefix_pattern, '', subject) - rematch = re.match('((RE|AW|SV|VS)(\[\d+\])?:\s*)+', subject, re.I) - if rematch: - subject = subject[rematch.end():] - recolon = 'Re:' - else: - recolon = '' - # At this point, subject may become null if someone post mail with - # subject: [subject prefix] - if subject.strip() == '': - subject = _('(no subject)') - cset = mlist.preferred_language.charset - # and substitute %d in prefix with post_id - try: - prefix = prefix % mlist.post_id - except TypeError: - pass - # Get the header as a Header instance, with proper unicode conversion - if not recolon: - h = uheader(mlist, prefix, 'Subject', continuation_ws=ws) - else: - h = uheader(mlist, prefix, 'Subject', continuation_ws=ws) - h.append(recolon) - # TK: Subject is concatenated and unicode string. - subject = subject.encode(cset, 'replace') - h.append(subject, cset) - del msg['subject'] - msg['Subject'] = h - ss = uheader(mlist, recolon, 'Subject', continuation_ws=ws) - ss.append(subject, cset) - msgdata['stripped_subject'] = ss - - - -def ch_oneline(headerstr): - # Decode header string in one line and convert into single charset. - # Return (string, cset) tuple as check for failure. - try: - d = decode_header(headerstr) - # At this point, we should rstrip() every string because some - # MUA deliberately add trailing spaces when composing return - # message. - d = [(s.rstrip(), c) for (s, c) in d] - # Find all charsets in the original header. We use 'utf-8' rather - # than using the first charset (in mailman 2.1.x) if multiple - # charsets are used. - csets = [] - for (s, c) in d: - if c and c not in csets: - csets.append(c) - if len(csets) == 0: - cset = 'us-ascii' - elif len(csets) == 1: - cset = csets[0] - else: - cset = 'utf-8' - h = make_header(d) - ustr = unicode(h) - oneline = ''.join(ustr.splitlines()) - return oneline.encode(cset, 'replace'), cset - except (LookupError, UnicodeError, ValueError, HeaderParseError): - # possibly charset problem. return with undecoded string in one line. - return ''.join(headerstr.splitlines()), 'us-ascii' - - - @implementer(IHandler) class CookHeaders: """Modify message headers.""" diff --git a/src/mailman/handlers/decorate.py b/src/mailman/handlers/decorate.py index bf8454232..78fafb3ca 100644 --- a/src/mailman/handlers/decorate.py +++ b/src/mailman/handlers/decorate.py @@ -17,9 +17,6 @@ """Decorate a message by sticking the header and footer around it.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Decorate', 'decorate', @@ -31,15 +28,14 @@ import re import logging from email.mime.text import MIMEText -from urllib2 import URLError -from zope.component import getUtility -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.email.message import Message from mailman.interfaces.handler import IHandler from mailman.interfaces.templates import ITemplateLoader from mailman.utilities.string import expand +from six.moves.urllib_error import URLError +from zope.component import getUtility +from zope.interface import implementer log = logging.getLogger('mailman.error') diff --git a/src/mailman/handlers/docs/acknowledge.rst b/src/mailman/handlers/docs/acknowledge.rst index e91f94f62..42cab04a0 100644 --- a/src/mailman/handlers/docs/acknowledge.rst +++ b/src/mailman/handlers/docs/acknowledge.rst @@ -113,9 +113,9 @@ The receipt will include the original message's subject in the response body, 1 >>> dump_msgdata(messages[0].msgdata) _parsemsg : False - listname : test@example.com + listid : test.example.com nodecorate : True - recipients : set([u'aperson@example.com']) + recipients : {'aperson@example.com'} reduced_list_headers: True ... >>> print(messages[0].msg.as_string()) @@ -150,9 +150,9 @@ If there is no subject, then the receipt will use a generic message. 1 >>> dump_msgdata(messages[0].msgdata) _parsemsg : False - listname : test@example.com + listid : test.example.com nodecorate : True - recipients : set([u'aperson@example.com']) + recipients : {'aperson@example.com'} reduced_list_headers: True ... >>> print(messages[0].msg.as_string()) diff --git a/src/mailman/handlers/docs/avoid-duplicates.rst b/src/mailman/handlers/docs/avoid-duplicates.rst index 612634941..19a41bf85 100644 --- a/src/mailman/handlers/docs/avoid-duplicates.rst +++ b/src/mailman/handlers/docs/avoid-duplicates.rst @@ -71,7 +71,7 @@ or ``Resent-CC``), then they will get a list copy. >>> msgdata = recips.copy() >>> handler.process(mlist, msg, msgdata) >>> sorted(msgdata['recipients']) - [u'aperson@example.com', u'bperson@example.com'] + ['aperson@example.com', 'bperson@example.com'] >>> print(msg.as_string()) From: Claire Person <cperson@example.com> <BLANKLINE> @@ -89,7 +89,7 @@ If they're mentioned on the ``CC`` line, they won't get a list copy. >>> msgdata = recips.copy() >>> handler.process(mlist, msg, msgdata) >>> sorted(msgdata['recipients']) - [u'bperson@example.com'] + ['bperson@example.com'] >>> print(msg.as_string()) From: Claire Person <cperson@example.com> CC: aperson@example.com @@ -109,7 +109,7 @@ to ``True`` (the default), then they still get a list copy. >>> msgdata = recips.copy() >>> handler.process(mlist, msg, msgdata) >>> sorted(msgdata['recipients']) - [u'aperson@example.com', u'bperson@example.com'] + ['aperson@example.com', 'bperson@example.com'] >>> print(msg.as_string()) From: Claire Person <cperson@example.com> CC: bperson@example.com @@ -128,7 +128,7 @@ Other headers checked for recipients include the ``To``... >>> msgdata = recips.copy() >>> handler.process(mlist, msg, msgdata) >>> sorted(msgdata['recipients']) - [u'bperson@example.com'] + ['bperson@example.com'] >>> print(msg.as_string()) From: Claire Person <cperson@example.com> To: aperson@example.com @@ -147,7 +147,7 @@ Other headers checked for recipients include the ``To``... >>> msgdata = recips.copy() >>> handler.process(mlist, msg, msgdata) >>> sorted(msgdata['recipients']) - [u'bperson@example.com'] + ['bperson@example.com'] >>> print(msg.as_string()) From: Claire Person <cperson@example.com> Resent-To: aperson@example.com @@ -166,7 +166,7 @@ Other headers checked for recipients include the ``To``... >>> msgdata = recips.copy() >>> handler.process(mlist, msg, msgdata) >>> sorted(msgdata['recipients']) - [u'bperson@example.com'] + ['bperson@example.com'] >>> print(msg.as_string()) From: Claire Person <cperson@example.com> Resent-Cc: aperson@example.com diff --git a/src/mailman/handlers/docs/digests.rst b/src/mailman/handlers/docs/digests.rst index ac6ea33d6..c3fc62ebf 100644 --- a/src/mailman/handlers/docs/digests.rst +++ b/src/mailman/handlers/docs/digests.rst @@ -82,11 +82,13 @@ actually crafted by the handler. >>> mlist.digest_size_threshold = 1 >>> mlist.volume = 2 >>> mlist.next_digest_number = 10 + >>> digest_path = os.path.join(mlist.data_path, 'digest.mmdf') >>> size = 0 >>> for msg in message_factory: ... process(mlist, msg, {}) - ... size += len(str(msg)) - ... if size >= mlist.digest_size_threshold * 1024: + ... # When the digest reaches the proper size, it is renamed. So we + ... # can break out of this list when the file disappears. + ... if not os.path.exists(digest_path): ... break >>> sum(1 for msg in digest_mbox(mlist)) diff --git a/src/mailman/handlers/docs/file-recips.rst b/src/mailman/handlers/docs/file-recips.rst index 58af6f480..73b47adb1 100644 --- a/src/mailman/handlers/docs/file-recips.rst +++ b/src/mailman/handlers/docs/file-recips.rst @@ -34,26 +34,6 @@ returns. recipients: 7 -Missing file -============ - -The include file must live inside the list's data directory, under the name -``members.txt``. If the file doesn't exist, the list of recipients will be -empty. - - >>> import os - >>> file_path = os.path.join(mlist.data_path, 'members.txt') - >>> open(file_path) - Traceback (most recent call last): - ... - IOError: [Errno ...] - No such file or directory: u'.../_xtest@example.com/members.txt' - >>> msgdata = {} - >>> handler.process(mlist, msg, msgdata) - >>> dump_list(msgdata['recipients']) - *Empty* - - Existing file ============= @@ -61,16 +41,15 @@ If the file exists, it contains a list of addresses, one per line. These addresses are returned as the set of recipients. :: - >>> fp = open(file_path, 'w') - >>> try: + >>> import os + >>> file_path = os.path.join(mlist.data_path, 'members.txt') + >>> with open(file_path, 'w', encoding='utf-8') as fp: ... print('bperson@example.com', file=fp) ... print('cperson@example.com', file=fp) ... print('dperson@example.com', file=fp) ... print('eperson@example.com', file=fp) ... print('fperson@example.com', file=fp) ... print('gperson@example.com', file=fp) - ... finally: - ... fp.close() >>> msgdata = {} >>> handler.process(mlist, msg, msgdata) diff --git a/src/mailman/handlers/docs/filtering.rst b/src/mailman/handlers/docs/filtering.rst index 6c3735f1b..582211d54 100644 --- a/src/mailman/handlers/docs/filtering.rst +++ b/src/mailman/handlers/docs/filtering.rst @@ -26,6 +26,8 @@ Filtering the outer content type A simple filtering setting will just search the content types of the messages parts, discarding all parts with a matching MIME type. If the message's outer content type matches the filter, the entire message will be discarded. +However, if we turn off content filtering altogether, then the handler +short-circuits. :: >>> from mailman.interfaces.mime import FilterAction @@ -42,14 +44,6 @@ content type matches the filter, the entire message will be discarded. ... """) >>> process = config.handlers['mime-delete'].process - >>> process(mlist, msg, {}) - Traceback (most recent call last): - ... - DiscardMessage: The message's content type was explicitly disallowed - -However, if we turn off content filtering altogether, then the handler -short-circuits. - >>> mlist.filter_content = False >>> msgdata = {} >>> process(mlist, msg, msgdata) @@ -74,15 +68,15 @@ crafted internally by Mailman. MIME-Version: 1.0 <BLANKLINE> xxxxx - >>> msgdata - {u'isdigest': True} + >>> dump_msgdata(msgdata) + isdigest: True Simple multipart filtering ========================== -If one of the subparts in a multipart message matches the filter type, then -just that subpart will be stripped. +If one of the subparts in a ``multipart`` message matches the filter type, +then just that subpart will be stripped. :: >>> msg = message_from_string("""\ @@ -241,8 +235,8 @@ name of the file containing the message payload to filter. >>> try: ... print("""\ ... import sys - ... print 'Converted text/html to text/plain' - ... print 'Filename:', sys.argv[1] + ... print('Converted text/html to text/plain') + ... print('Filename:', sys.argv[1]) ... """, file=fp) ... finally: ... fp.close() diff --git a/src/mailman/handlers/docs/nntp.rst b/src/mailman/handlers/docs/nntp.rst index 2dfc95ce1..72bcb35f0 100644 --- a/src/mailman/handlers/docs/nntp.rst +++ b/src/mailman/handlers/docs/nntp.rst @@ -63,5 +63,5 @@ messages are gated to. >>> dump_msgdata(messages[0].msgdata) _parsemsg: False - listname : test@example.com + listid : test.example.com version : 3 diff --git a/src/mailman/handlers/docs/replybot.rst b/src/mailman/handlers/docs/replybot.rst index 638c2fdc8..9e18ce911 100644 --- a/src/mailman/handlers/docs/replybot.rst +++ b/src/mailman/handlers/docs/replybot.rst @@ -49,9 +49,9 @@ response. >>> dump_msgdata(messages[0].msgdata) _parsemsg : False - listname : _xtest@example.com + listid : _xtest.example.com nodecorate : True - recipients : set([u'aperson@example.com']) + recipients : {'aperson@example.com'} reduced_list_headers: True version : 3 @@ -141,9 +141,9 @@ Unless the ``X-Ack:`` header has a value of ``yes``, in which case, the >>> dump_msgdata(messages[0].msgdata) _parsemsg : False - listname : _xtest@example.com + listid : _xtest.example.com nodecorate : True - recipients : set([u'asystem@example.com']) + recipients : {'asystem@example.com'} reduced_list_headers: True version : 3 diff --git a/src/mailman/handlers/docs/rfc-2369.rst b/src/mailman/handlers/docs/rfc-2369.rst index 8180b0635..b5a783edc 100644 --- a/src/mailman/handlers/docs/rfc-2369.rst +++ b/src/mailman/handlers/docs/rfc-2369.rst @@ -13,7 +13,7 @@ headers generally start with the `List-` prefix. .. This is a helper function for the following section. >>> def list_headers(msg, only=None): - ... if isinstance(only, basestring): + ... if isinstance(only, str): ... only = (only.lower(),) ... elif only is None: ... only = set(header.lower() for header in msg.keys() diff --git a/src/mailman/handlers/docs/subject-munging.rst b/src/mailman/handlers/docs/subject-munging.rst index 538ad99c7..de22a928c 100644 --- a/src/mailman/handlers/docs/subject-munging.rst +++ b/src/mailman/handlers/docs/subject-munging.rst @@ -1,44 +1,42 @@ -=============== -Subject munging -=============== +================ +Subject prefixes +================ -Messages that flow through the global pipeline get their headers *cooked*, -which basically means that their headers go through several mostly unrelated -transformations. Some headers get added, others get changed. Some of these -changes depend on mailing list settings and others depend on how the message -is getting sent through the system. We'll take things one-by-one. +Mailing lists can define a *subject prefix* which gets added to the front of +any ``Subject`` text. This can be used to quickly identify which mailing list +the message was posted to. >>> mlist = create_list('test@example.com') +The default list style gives the mailing list a default prefix. -Inserting a prefix -================== + >>> print(mlist.subject_prefix) + [Test] -Another thing header cooking does is *munge* the ``Subject`` header by -inserting the subject prefix for the list at the front. If there's no subject -header in the original message, Mailman uses a canned default. In order to do -subject munging, a mailing list must have a preferred language. -:: +This can be changed to anything, but typically ends with a trailing space. >>> mlist.subject_prefix = '[XTest] ' - >>> mlist.preferred_language = 'en' + >>> process = config.handlers['subject-prefix'].process + + +No Subject +========== + +If the original message has no ``Subject``, then a canned one is used. + >>> msg = message_from_string("""\ ... From: aperson@example.com ... ... A message of great import. ... """) - >>> msgdata = {} - - >>> from mailman.handlers.cook_headers import process - >>> process(mlist, msg, msgdata) - -The original subject header is stored in the message metadata. - - >>> msgdata['original_subject'] - u'' + >>> process(mlist, msg, {}) >>> print(msg['subject']) [XTest] (no subject) + +Inserting a prefix +================== + If the original message had a ``Subject`` header, then the prefix is inserted at the beginning of the header's value. @@ -50,34 +48,12 @@ at the beginning of the header's value. ... """) >>> msgdata = {} >>> process(mlist, msg, msgdata) - >>> print(msgdata['original_subject']) - Something important >>> print(msg['subject']) [XTest] Something important -``Subject`` headers are not munged for digest messages. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Subject: Something important - ... - ... A message of great import. - ... """) - >>> process(mlist, msg, dict(isdigest=True)) - >>> print(msg['subject']) - Something important - -Nor are they munged for *fast tracked* messages, which are generally defined -as messages that Mailman crafts internally. +The original ``Subject`` is available in the metadata. - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Subject: Something important - ... - ... A message of great import. - ... """) - >>> process(mlist, msg, dict(_fasttrack=True)) - >>> print(msg['subject']) + >>> print(msgdata['original_subject']) Something important If a ``Subject`` header already has a prefix, usually following a ``Re:`` @@ -95,8 +71,7 @@ front of the header text. [XTest] Re: Something important If the ``Subject`` header has a prefix at the front of the header text, that's -where it will stay. This is called *new style* prefixing and is the only -option available in Mailman 3. +where it will stay. >>> msg = message_from_string("""\ ... From: aperson@example.com @@ -122,10 +97,10 @@ set than the encoded header. ... ... """) >>> process(mlist, msg, {}) - >>> print(msg['subject']) + >>> print(msg['subject'].encode()) [XTest] =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= - >>> unicode(msg['subject']) - u'[XTest] \u30e1\u30fc\u30eb\u30de\u30f3' + >>> print(str(msg['subject'])) + [XTest] メールマン Prefix numbers @@ -178,10 +153,10 @@ in the subject prefix, and the subject is encoded non-ASCII. ... ... """) >>> process(mlist, msg, {}) - >>> print(msg['subject']) + >>> print(msg['subject'].encode()) [XTest 456] =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= - >>> unicode(msg['subject']) - u'[XTest 456] \u30e1\u30fc\u30eb\u30de\u30f3' + >>> print(msg['subject']) + [XTest 456] メールマン Even more fun is when the internationalized ``Subject`` header already has a prefix, possibly with a different posting number. @@ -191,13 +166,10 @@ prefix, possibly with a different posting number. ... ... """) >>> process(mlist, msg, {}) - >>> print(msg['subject']) + >>> print(msg['subject'].encode()) [XTest 456] Re: =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= - -.. - # XXX This requires Python email patch #1681333 to succeed. - # >>> unicode(msg['subject']) - # u'[XTest 456] Re: \u30e1\u30fc\u30eb\u30de\u30f3' + >>> print(msg['subject']) + [XTest 456] Re: メールマン As before, old style subject prefixes are re-ordered. @@ -206,14 +178,11 @@ As before, old style subject prefixes are re-ordered. ... ... """) >>> process(mlist, msg, {}) - >>> print(msg['subject']) + >>> print(msg['subject'].encode()) [XTest 456] Re: =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= - -.. - # XXX This requires Python email patch #1681333 to succeed. - # >>> unicode(msg['subject']) - # u'[XTest 456] Re: \u30e1\u30fc\u30eb\u30de\u30f3' + >>> print(msg['subject']) + [XTest 456] Re: メールマン In this test case, we get an extra space between the prefix and the original diff --git a/src/mailman/handlers/docs/tagger.rst b/src/mailman/handlers/docs/tagger.rst index f3303b7ef..fcefdb01c 100644 --- a/src/mailman/handlers/docs/tagger.rst +++ b/src/mailman/handlers/docs/tagger.rst @@ -55,7 +55,7 @@ and the message metadata gets a key with a list of matching topic names. <BLANKLINE> <BLANKLINE> >>> msgdata['topichits'] - [u'bar fight'] + ['bar fight'] Scanning body lines @@ -114,7 +114,7 @@ found. Keywords: barbaz <BLANKLINE> >>> msgdata['topichits'] - [u'bar fight'] + ['bar fight'] However, scanning stops at the first body line that doesn't look like a header. @@ -161,7 +161,7 @@ When set to a negative number, all body lines will be scanned. >>> print(msg['x-topics']) bar fight >>> msgdata['topichits'] - [u'bar fight'] + ['bar fight'] Scanning sub-parts @@ -175,14 +175,14 @@ text payload. ... Subject: Was ... Keywords: Raw ... Content-Type: multipart/alternative; boundary="BOUNDARY" - ... + ... ... --BOUNDARY ... From: sabo ... To: obas - ... + ... ... Subject: farbaw ... Keywords: barbaz - ... + ... ... --BOUNDARY-- ... """) >>> msgdata = {} @@ -203,7 +203,7 @@ text payload. --BOUNDARY-- <BLANKLINE> >>> msgdata['topichits'] - [u'bar fight'] + ['bar fight'] But the tagger will not descend into non-text parts. @@ -211,23 +211,23 @@ But the tagger will not descend into non-text parts. ... Subject: Was ... Keywords: Raw ... Content-Type: multipart/alternative; boundary=BOUNDARY - ... + ... ... --BOUNDARY ... From: sabo ... To: obas ... Content-Type: message/rfc822 - ... + ... ... Subject: farbaw ... Keywords: barbaz - ... + ... ... --BOUNDARY ... From: sabo ... To: obas ... Content-Type: message/rfc822 - ... + ... ... Subject: farbaw ... Keywords: barbaz - ... + ... ... --BOUNDARY-- ... """) >>> msgdata = {} diff --git a/src/mailman/handlers/docs/to-outgoing.rst b/src/mailman/handlers/docs/to-outgoing.rst index e87fd4f26..90ea137a5 100644 --- a/src/mailman/handlers/docs/to-outgoing.rst +++ b/src/mailman/handlers/docs/to-outgoing.rst @@ -37,6 +37,6 @@ additional key set: the mailing list name. _parsemsg: False bar : 2 foo : 1 - listname : test@example.com + listid : test.example.com verp : True version : 3 diff --git a/src/mailman/handlers/file_recipients.py b/src/mailman/handlers/file_recipients.py index ec8868649..4b115bb53 100644 --- a/src/mailman/handlers/file_recipients.py +++ b/src/mailman/handlers/file_recipients.py @@ -17,9 +17,6 @@ """Get the normal delivery recipients from a Sendmail style :include: file.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'FileRecipients', ] @@ -28,10 +25,9 @@ __all__ = [ import os import errno -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler +from zope.interface import implementer diff --git a/src/mailman/handlers/member_recipients.py b/src/mailman/handlers/member_recipients.py index 0f99bf709..7497746eb 100644 --- a/src/mailman/handlers/member_recipients.py +++ b/src/mailman/handlers/member_recipients.py @@ -23,22 +23,18 @@ on the `recipients' attribute of the message. This attribute is used by the SendmailDeliver and BulkDeliver modules. """ -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'MemberRecipients', ] -from zope.interface import implementer - from mailman.config import config from mailman.core import errors from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler from mailman.interfaces.member import DeliveryStatus from mailman.utilities.string import wrap +from zope.interface import implementer diff --git a/src/mailman/handlers/mime_delete.py b/src/mailman/handlers/mime_delete.py index 98c1de3f9..1d107522d 100644 --- a/src/mailman/handlers/mime_delete.py +++ b/src/mailman/handlers/mime_delete.py @@ -24,9 +24,6 @@ wrapping only single sections after other processing are replaced by their contents. """ -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'MIMEDelete', ] @@ -41,9 +38,6 @@ from email.iterators import typed_subpart_iterator from email.mime.message import MIMEMessage from email.mime.text import MIMEText from lazr.config import as_boolean -from os.path import splitext -from zope.interface import implementer - from mailman.config import config from mailman.core import errors from mailman.core.i18n import _ @@ -52,6 +46,8 @@ from mailman.interfaces.action import FilterAction from mailman.interfaces.handler import IHandler from mailman.utilities.string import oneline from mailman.version import VERSION +from os.path import splitext +from zope.interface import implementer log = logging.getLogger('mailman.error') @@ -245,7 +241,7 @@ def to_plaintext(msg): filename = tempfile.mktemp('.html') fp = open(filename, 'w') try: - fp.write(subpart.get_payload(decode=True)) + fp.write(subpart.get_payload()) fp.close() cmd = os.popen(config.HTML_TO_PLAIN_TEXT_COMMAND % {'filename': filename}) diff --git a/src/mailman/handlers/owner_recipients.py b/src/mailman/handlers/owner_recipients.py index 5a1d0bd2e..dbb203728 100644 --- a/src/mailman/handlers/owner_recipients.py +++ b/src/mailman/handlers/owner_recipients.py @@ -17,20 +17,16 @@ """Calculate the list owner recipients (includes moderators).""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'OwnerRecipients', ] -from zope.interface import implementer - from mailman.config import config from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler from mailman.interfaces.member import DeliveryStatus +from zope.interface import implementer diff --git a/src/mailman/handlers/replybot.py b/src/mailman/handlers/replybot.py index 63f3ca4cf..44df2344e 100644 --- a/src/mailman/handlers/replybot.py +++ b/src/mailman/handlers/replybot.py @@ -17,9 +17,6 @@ """Handler for automatic responses.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Replybot', ] @@ -27,9 +24,6 @@ __all__ = [ import logging -from zope.component import getUtility -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.email.message import UserNotification from mailman.interfaces.autorespond import ( @@ -38,6 +32,8 @@ from mailman.interfaces.handler import IHandler from mailman.interfaces.usermanager import IUserManager from mailman.utilities.datetime import today from mailman.utilities.string import expand, wrap +from zope.component import getUtility +from zope.interface import implementer log = logging.getLogger('mailman.error') diff --git a/src/mailman/handlers/rfc_2369.py b/src/mailman/handlers/rfc_2369.py index ea909f41b..c835f2a67 100644 --- a/src/mailman/handlers/rfc_2369.py +++ b/src/mailman/handlers/rfc_2369.py @@ -17,22 +17,18 @@ """RFC 2369 List-* and related headers.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'RFC2369', ] from email.utils import formataddr -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.handlers.cook_headers import uheader from mailman.interfaces.archiver import ArchivePolicy from mailman.interfaces.mailinglist import IListArchiverSet from mailman.interfaces.handler import IHandler +from zope.interface import implementer CONTINUATION = ',\n\t' diff --git a/src/mailman/handlers/subject_prefix.py b/src/mailman/handlers/subject_prefix.py new file mode 100644 index 000000000..20abd1036 --- /dev/null +++ b/src/mailman/handlers/subject_prefix.py @@ -0,0 +1,184 @@ +# Copyright (C) 2014 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Subject header prefix munging.""" + +__all__ = [ + 'SubjectPrefix', + ] + + +import re + +from email.header import Header, make_header, decode_header +from mailman.core.i18n import _ +from mailman.interfaces.handler import IHandler +from zope.interface import implementer + + +RE_PATTERN = '((RE|AW|SV|VS)(\[\d+\])?:\s*)+' +ASCII_CHARSETS = (None, 'ascii', 'us-ascii') +EMPTYSTRING = '' + + + +def ascii_header(mlist, msgdata, subject, prefix, prefix_pattern, ws): + if mlist.preferred_language.charset not in ASCII_CHARSETS: + return None + for chunk, charset in decode_header(subject.encode()): + if charset not in ASCII_CHARSETS: + return None + subject_text = EMPTYSTRING.join(str(subject).splitlines()) + rematch = re.match(RE_PATTERN, subject_text, re.I) + if rematch: + subject_text = subject_text[rematch.end():] + recolon = 'Re: ' + else: + recolon = '' + # At this point, the subject may become null if someone posted mail + # with "Subject: [subject prefix]". + if subject_text.strip() == '': + with _.using(mlist.preferred_language.code): + subject_text = _('(no subject)') + else: + subject_text = re.sub(prefix_pattern, '', subject_text) + msgdata['stripped_subject'] = subject_text + lines = subject_text.splitlines() + first_line = [lines[0]] + if recolon: + first_line.insert(0, recolon) + if prefix: + first_line.insert(0, prefix) + subject_text = EMPTYSTRING.join(first_line) + return Header(subject_text, continuation_ws=ws) + + +def all_same_charset(mlist, msgdata, subject, prefix, prefix_pattern, ws): + list_charset = mlist.preferred_language.charset + chunks = [] + for chunk, charset in decode_header(subject.encode()): + if charset is None: + charset = 'us-ascii' + chunks.append(chunk.decode(charset)) + if charset != list_charset: + return None + subject_text = EMPTYSTRING.join(chunks) + rematch = re.match(RE_PATTERN, subject_text, re.I) + if rematch: + subject_text = subject_text[rematch.end():] + recolon = 'Re: ' + else: + recolon = '' + # At this point, the subject may become null if someone posted mail + # with "Subject: [subject prefix]". + if subject_text.strip() == '': + with _.push(mlist.preferred_language.code): + subject_text = _('(no subject)') + else: + subject_text = re.sub(prefix_pattern, '', subject_text) + msgdata['stripped_subject'] = subject_text + lines = subject_text.splitlines() + first_line = [lines[0]] + if recolon: + first_line.insert(0, recolon) + if prefix: + first_line.insert(0, prefix) + subject_text = EMPTYSTRING.join(first_line) + return Header(subject_text, charset=list_charset, continuation_ws=ws) + + +def mixed_charsets(mlist, msgdata, subject, prefix, prefix_pattern, ws): + list_charset = mlist.preferred_language.charset + chunks = decode_header(subject.encode()) + if len(chunks) == 0: + with _.push(mlist.preferred_language.code): + subject_text = _('(no subject)') + chunks = [(prefix, list_charset), + (subject_text, list_charset), + ] + return make_header(chunks, continuation_ws=ws) + # Only search the first chunk for Re and existing prefix. + chunk_text, chunk_charset = chunks[0] + if chunk_charset is None: + chunk_charset = 'us-ascii' + first_text = chunk_text.decode(chunk_charset) + first_text = re.sub(prefix_pattern, '', first_text).lstrip() + rematch = re.match(RE_PATTERN, first_text, re.I) + if rematch: + first_text = 'Re: ' + first_text[rematch.end():] + chunks[0] = (first_text, chunk_charset) + # The subject text stripped of the prefix, for use in the NNTP gateway. + msgdata['stripped_subject'] = str(make_header(chunks, continuation_ws=ws)) + chunks.insert(0, (prefix, list_charset)) + return make_header(chunks, continuation_ws=ws) + + + +@implementer(IHandler) +class SubjectPrefix: + """Add a list-specific prefix to the Subject header value.""" + + name = 'subject-prefix' + description = _('Add a list-specific prefix to the Subject header value.') + + def process(self, mlist, msg, msgdata): + """See `IHandler`.""" + if msgdata.get('isdigest') or msgdata.get('_fasttrack'): + return + prefix = mlist.subject_prefix + if not prefix.strip(): + return + subject = msg.get('subject', '') + # Turn the value into a Header instance and try to figure out what + # continuation whitespace is being used. + # Save the original Subject. + msgdata['original_subject'] = subject + if isinstance(subject, Header): + subject_text = str(subject) + else: + subject = make_header(decode_header(subject)) + subject_text = str(subject) + lines = subject_text.splitlines() + ws = '\t' + if len(lines) > 1 and lines[1] and lines[1][0] in ' \t': + ws = lines[1][0] + # If the subject_prefix contains '%d', it is replaced with the mailing + # list's sequence number. The sequential number format allows '%d' or + # '%05d' like pattern. + prefix_pattern = re.escape(prefix) + # Unescape '%'. + prefix_pattern = '%'.join(prefix_pattern.split(r'\%')) + p = re.compile('%\d*d') + if p.search(prefix, 1): + # The prefix has number, so we should search prefix w/number in + # subject. Also, force new style. + prefix_pattern = p.sub(r'\s*\d+\s*', prefix_pattern) + # Substitute %d in prefix with post_id + try: + prefix = prefix % mlist.post_id + except TypeError: + pass + for handler in (ascii_header, + all_same_charset, + mixed_charsets, + ): + new_subject = handler( + mlist, msgdata, subject, prefix, prefix_pattern, ws) + if new_subject is not None: + del msg['subject'] + msg['Subject'] = new_subject + return diff --git a/src/mailman/handlers/tagger.py b/src/mailman/handlers/tagger.py index 803cc6d11..199c5907f 100644 --- a/src/mailman/handlers/tagger.py +++ b/src/mailman/handlers/tagger.py @@ -17,9 +17,6 @@ """Extract topics from the original mail message.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Tagger', ] @@ -29,15 +26,14 @@ import re import email.iterators import email.parser -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler +from zope.interface import implementer OR = '|' CRNL = '\r\n' -EMPTYBYTES = b'' +EMPTYSTRING = '' NLTAB = '\n\t' @@ -104,7 +100,7 @@ def scanbody(msg, numlines=None): reader = list(email.iterators.body_line_iterator(msg)) while numlines is None or lineno < numlines: try: - line = bytes(reader.pop(0)) + line = reader.pop(0) except IndexError: break # Blank lines don't count @@ -115,7 +111,7 @@ def scanbody(msg, numlines=None): # Concatenate those body text lines with newlines, and then create a new # message object from those lines. p = _ForgivingParser() - msg = p.parsestr(EMPTYBYTES.join(lines)) + msg = p.parsestr(EMPTYSTRING.join(lines)) return msg.get_all('subject', []) + msg.get_all('keywords', []) diff --git a/src/mailman/handlers/tests/test_cook_headers.py b/src/mailman/handlers/tests/test_cook_headers.py index d83a44f20..385f402c5 100644 --- a/src/mailman/handlers/tests/test_cook_headers.py +++ b/src/mailman/handlers/tests/test_cook_headers.py @@ -17,9 +17,6 @@ """Test the cook_headers handler.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestCookHeaders', ] @@ -50,6 +47,6 @@ class TestCookHeaders(unittest.TestCase): for msg in messages: try: cook_headers.process(self._mlist, msg, {}) - except AttributeError as e: + except AttributeError as error: # LP: #1130696 would raise an AttributeError on .sender - self.fail(e) + self.fail(error) diff --git a/src/mailman/handlers/tests/test_file_recips.py b/src/mailman/handlers/tests/test_file_recips.py new file mode 100644 index 000000000..906530762 --- /dev/null +++ b/src/mailman/handlers/tests/test_file_recips.py @@ -0,0 +1,73 @@ +# Copyright (C) 2014 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test file-recips handler.""" + +__all__ = [ + 'TestFileRecips', + ] + + +import os +import unittest + +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.testing.helpers import specialized_message_from_string as mfs +from mailman.testing.layers import ConfigLayer + + + +class TestFileRecips(unittest.TestCase): + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._handler = config.handlers['file-recipients'].process + self._msg = mfs("""\ +From: aperson@example.com + +A message. +""") + + def test_file_is_missing(self): + # It is not an error for the list's the members.txt file to be + # missing. The missing file is just ignored. + msgdata = {} + self._handler(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set()) + + def test_file_exists(self): + # Like above, but the file exists and contains recipients. + path = os.path.join(self._mlist.data_path, 'members.txt') + with open(path, 'w', encoding='utf-8') as fp: + print('bperson@example.com', file=fp) + print('cperson@example.com', file=fp) + print('dperson@example.com', file=fp) + print('eperson@example.com', file=fp) + print('fperson@example.com', file=fp) + print('gperson@example.com', file=fp) + msgdata = {} + self._handler(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(( + 'bperson@example.com', + 'cperson@example.com', + 'dperson@example.com', + 'eperson@example.com', + 'fperson@example.com', + 'gperson@example.com', + ))) diff --git a/src/mailman/handlers/tests/test_filter.py b/src/mailman/handlers/tests/test_filter.py new file mode 100644 index 000000000..b81744008 --- /dev/null +++ b/src/mailman/handlers/tests/test_filter.py @@ -0,0 +1,57 @@ +# Copyright (C) 2014 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test the filter handler.""" + +__all__ = [ + 'TestFilters', + ] + + +import unittest + +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.core.errors import DiscardMessage +from mailman.interfaces.mime import FilterAction +from mailman.testing.helpers import specialized_message_from_string as mfs +from mailman.testing.layers import ConfigLayer + + + +class TestFilters(unittest.TestCase): + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + + def test_discard_when_outer_type_matches(self): + # When the outer MIME type of the message matches a filter type, the + # entire message is discarded. + self._mlist.filter_content = True + self._mlist.filter_types = ['image/jpeg'] + self._mlist.filter_action = FilterAction.discard + msg = mfs("""\ +From: aperson@example.com +Content-Type: image/jpeg +MIME-Version: 1.0 + +xxxxx +""") + self.assertRaises(DiscardMessage, + config.handlers['mime-delete'].process, + self._mlist, msg, {}) diff --git a/src/mailman/handlers/tests/test_mimedel.py b/src/mailman/handlers/tests/test_mimedel.py index c7c37152f..02cb275e0 100644 --- a/src/mailman/handlers/tests/test_mimedel.py +++ b/src/mailman/handlers/tests/test_mimedel.py @@ -17,9 +17,6 @@ """Test the mime_delete handler.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestDispose', ] @@ -27,8 +24,6 @@ __all__ = [ import unittest -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.config import config from mailman.core import errors @@ -40,6 +35,7 @@ from mailman.testing.helpers import ( LogFileMark, configuration, get_queue_messages, specialized_message_from_string as mfs) from mailman.testing.layers import ConfigLayer +from zope.component import getUtility diff --git a/src/mailman/handlers/tests/test_recipients.py b/src/mailman/handlers/tests/test_recipients.py index afe533a7e..688dcce04 100644 --- a/src/mailman/handlers/tests/test_recipients.py +++ b/src/mailman/handlers/tests/test_recipients.py @@ -17,9 +17,6 @@ """Testing various recipients stuff.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestMemberRecipients', 'TestOwnerRecipients', @@ -28,13 +25,14 @@ __all__ = [ import unittest -from zope.component import getUtility from mailman.app.lifecycle import create_list from mailman.config import config from mailman.interfaces.member import DeliveryMode, DeliveryStatus, MemberRole from mailman.interfaces.usermanager import IUserManager -from mailman.testing.helpers import specialized_message_from_string as mfs +from mailman.testing.helpers import ( + configuration, specialized_message_from_string as mfs) from mailman.testing.layers import ConfigLayer +from zope.component import getUtility @@ -199,23 +197,14 @@ To: test-owner@example.com self._process(self._mlist, self._msg, msgdata) self.assertEqual(msgdata['recipients'], set(('noreply@example.com',))) - def test_site_admin_unicode(self): - # Since the config file is read as bytes, the site_owner is also a - # bytes and must be converted to unicode when used as a fallback. + @configuration('mailman', site_owner='siteadmin@example.com') + def test_no_owners_site_owner_fallback(self): + # The list has no owners or moderators, but there is a non-default + # site owner defined. That owner gets the message. self._cris.unsubscribe() self._dave.unsubscribe() self.assertEqual(self._mlist.administrators.member_count, 0) msgdata = {} - # In order to properly mimic the testing environment, use - # config.push()/config.pop() directly instead of using the - # configuration() context manager. - config.push('test_site_admin_unicode', b"""\ -[mailman] -site_owner: siteadmin@example.com -""") - try: - self._process(self._mlist, self._msg, msgdata) - finally: - config.pop('test_site_admin_unicode') - self.assertEqual(len(msgdata['recipients']), 1) - self.assertIsInstance(list(msgdata['recipients'])[0], unicode) + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], + set(('siteadmin@example.com',))) diff --git a/src/mailman/handlers/tests/test_subject_prefix.py b/src/mailman/handlers/tests/test_subject_prefix.py new file mode 100644 index 000000000..f4fd8c113 --- /dev/null +++ b/src/mailman/handlers/tests/test_subject_prefix.py @@ -0,0 +1,129 @@ +# Copyright (C) 2014 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test the Subject header prefix munging..""" + +__all__ = [ + 'TestSubjectPrefix', + ] + + +import unittest + +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.email.message import Message +from mailman.testing.layers import ConfigLayer + + + +class TestSubjectPrefix(unittest.TestCase): + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._process = config.handlers['subject-prefix'].process + + def test_isdigest(self): + # If the message is destined for the digest, the Subject header does + # not get touched. + msg = Message() + msg['Subject'] = 'A test message' + self._process(self._mlist, msg, dict(isdigest=True)) + self.assertEqual(str(msg['subject']), 'A test message') + + def test_fasttrack(self): + # Messages internally crafted are 'fast tracked' and don't get their + # Subjects prefixed either. + msg = Message() + msg['Subject'] = 'A test message' + self._process(self._mlist, msg, dict(_fasttrack=True)) + self.assertEqual(str(msg['subject']), 'A test message') + + def test_whitespace_only_prefix(self): + # If the Subject prefix only contains whitespace, ignore it. + self._mlist.subject_prefix = ' ' + msg = Message() + msg['Subject'] = 'A test message' + self._process(self._mlist, msg, dict(_fasttrack=True)) + self.assertEqual(str(msg['subject']), 'A test message') + + def test_save_original_subject(self): + # When the Subject gets prefixed, the original is saved in the message + # metadata. + msgdata = {} + msg = Message() + msg['Subject'] = 'A test message' + self._process(self._mlist, msg, msgdata) + self.assertEqual(msgdata['original_subject'], 'A test message') + + def test_prefix(self): + # The Subject gets prefixed. The prefix gets automatically set by the + # list style when the list gets created. + msg = Message() + msg['Subject'] = 'A test message' + self._process(self._mlist, msg, {}) + self.assertEqual(str(msg['subject']), '[Test] A test message') + + def test_no_double_prefix(self): + # Don't add a prefix if the subject already contains one. + msg = Message() + msg['Subject'] = '[Test] A test message' + self._process(self._mlist, msg, {}) + self.assertEqual(str(msg['subject']), '[Test] A test message') + + def test_re_prefix(self): + # The subject has a Re: prefix. Make sure that gets preserved, but + # after the list prefix. + msg = Message() + msg['Subject'] = 'Re: [Test] A test message' + self._process(self._mlist, msg, {}) + self.assertEqual(str(msg['subject']), '[Test] Re: A test message') + + def test_multiline_subject(self): + # The subject appears on multiple lines. + msg = Message() + msg['Subject'] = '\n A test message' + self._process(self._mlist, msg, {}) + self.assertEqual(str(msg['subject']), '[Test] A test message') + + def test_i18n_prefix(self): + # The Subject header is encoded, but the prefix is still added. + msg = Message() + msg['Subject'] = '=?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?=' + self._process(self._mlist, msg, {}) + subject = msg['subject'] + self.assertEqual(subject.encode(), + '[Test] =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?=') + self.assertEqual(str(subject), '[Test] メールマン') + + def test_i18n_subject_with_sequential_prefix_and_re(self): + # The mailing list defines a sequential prefix, and the original + # Subject has a prefix with a different sequence number, *and* it also + # contains a Re: prefix. Make sure the sequence gets updated and all + # the bits get put back together in the right order. + self._mlist.subject_prefix = '[Test %d]' + self._mlist.post_id = 456 + msg = Message() + msg['Subject'] = \ + '[Test 123] Re: =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?=' + self._process(self._mlist, msg, {}) + subject = msg['subject'] + self.assertEqual( + subject.encode(), + '[Test 456] Re: =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?=') + self.assertEqual(str(subject), '[Test 456] Re: メールマン') diff --git a/src/mailman/handlers/tests/test_to_digest.py b/src/mailman/handlers/tests/test_to_digest.py index 451ebf9a5..8562c3fd7 100644 --- a/src/mailman/handlers/tests/test_to_digest.py +++ b/src/mailman/handlers/tests/test_to_digest.py @@ -17,9 +17,6 @@ """Test the to_digest handler.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestToDigest', ] diff --git a/src/mailman/handlers/to_archive.py b/src/mailman/handlers/to_archive.py index d18742f3c..d8c61bc7d 100644 --- a/src/mailman/handlers/to_archive.py +++ b/src/mailman/handlers/to_archive.py @@ -17,20 +17,16 @@ """Add the message to the archives.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ToArchive', ] -from zope.interface import implementer - from mailman.config import config from mailman.core.i18n import _ from mailman.interfaces.archiver import ArchivePolicy from mailman.interfaces.handler import IHandler +from zope.interface import implementer diff --git a/src/mailman/handlers/to_digest.py b/src/mailman/handlers/to_digest.py index e915bbfa3..70aeb0dcc 100644 --- a/src/mailman/handlers/to_digest.py +++ b/src/mailman/handlers/to_digest.py @@ -17,9 +17,6 @@ """Add the message to the list's current digest.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ToDigest', ] @@ -27,8 +24,6 @@ __all__ = [ import os -from zope.interface import implementer - from mailman.config import config from mailman.core.i18n import _ from mailman.email.message import Message @@ -36,6 +31,7 @@ from mailman.interfaces.digests import DigestFrequency from mailman.interfaces.handler import IHandler from mailman.utilities.datetime import now as right_now from mailman.utilities.mailbox import Mailbox +from zope.interface import implementer @@ -55,7 +51,7 @@ class ToDigest: mailbox_path = os.path.join(mlist.data_path, 'digest.mmdf') # Lock the mailbox and append the message. with Mailbox(mailbox_path, create=True) as mbox: - mbox.add(msg.as_string()) + mbox.add(msg) # Calculate the current size of the mailbox file. This will not tell # us exactly how big the resulting MIME and rfc1153 digest will # actually be, but it's the most easily available metric to decide @@ -75,7 +71,7 @@ class ToDigest: os.rename(mailbox_path, mailbox_dest) config.switchboards['digest'].enqueue( Message(), - listname=mlist.fqdn_listname, + listid=mlist.list_id, digest_path=mailbox_dest, volume=volume, digest_number=digest_number) diff --git a/src/mailman/handlers/to_outgoing.py b/src/mailman/handlers/to_outgoing.py index 6dfbe88c0..95686d9c7 100644 --- a/src/mailman/handlers/to_outgoing.py +++ b/src/mailman/handlers/to_outgoing.py @@ -22,19 +22,15 @@ posted to the list membership. Anything else that needs to go out to some recipient should just be placed in the out queue directly. """ -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ToOutgoing', ] -from zope.interface import implementer - from mailman.config import config from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler +from zope.interface import implementer @@ -47,5 +43,4 @@ class ToOutgoing: def process(self, mlist, msg, msgdata): """See `IHandler`.""" - config.switchboards['out'].enqueue( - msg, msgdata, listname=mlist.fqdn_listname) + config.switchboards['out'].enqueue(msg, msgdata, listid=mlist.list_id) diff --git a/src/mailman/handlers/to_usenet.py b/src/mailman/handlers/to_usenet.py index d5a946644..8d86ea86e 100644 --- a/src/mailman/handlers/to_usenet.py +++ b/src/mailman/handlers/to_usenet.py @@ -17,9 +17,6 @@ """Move the message to the mail->news queue.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ToUsenet', ] @@ -27,14 +24,13 @@ __all__ = [ import logging -from zope.interface import implementer - from mailman.config import config from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler +from zope.interface import implementer -COMMASPACE = ', ' +COMMASPACE = ', ' log = logging.getLogger('mailman.error') @@ -65,5 +61,4 @@ class ToUsenet: COMMASPACE.join(error)) return # Put the message in the news runner's queue. - config.switchboards['nntp'].enqueue( - msg, msgdata, listname=mlist.fqdn_listname) + config.switchboards['nntp'].enqueue(msg, msgdata, listid=mlist.list_id) diff --git a/src/mailman/interfaces/action.py b/src/mailman/interfaces/action.py index 5d4b150a3..c4147f57a 100644 --- a/src/mailman/interfaces/action.py +++ b/src/mailman/interfaces/action.py @@ -17,7 +17,6 @@ """Message actions.""" -__metaclass__ = type __all__ = [ 'Action', 'FilterAction', diff --git a/src/mailman/interfaces/address.py b/src/mailman/interfaces/address.py index 28a9e8ef4..24d0899f5 100644 --- a/src/mailman/interfaces/address.py +++ b/src/mailman/interfaces/address.py @@ -17,9 +17,6 @@ """Interface for email address related information.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'AddressAlreadyLinkedError', 'AddressError', @@ -33,9 +30,8 @@ __all__ = [ ] -from zope.interface import Interface, Attribute - from mailman.interfaces.errors import MailmanError +from zope.interface import Interface, Attribute diff --git a/src/mailman/interfaces/archiver.py b/src/mailman/interfaces/archiver.py index 8b843bc60..b2fc4f1af 100644 --- a/src/mailman/interfaces/archiver.py +++ b/src/mailman/interfaces/archiver.py @@ -17,9 +17,6 @@ """Interface for archiving schemes.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ArchivePolicy', 'ClobberDate', diff --git a/src/mailman/interfaces/autorespond.py b/src/mailman/interfaces/autorespond.py index 8da2fc795..d53e181f0 100644 --- a/src/mailman/interfaces/autorespond.py +++ b/src/mailman/interfaces/autorespond.py @@ -17,9 +17,6 @@ """Autoresponder.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ALWAYS_REPLY', 'IAutoResponseRecord', @@ -33,6 +30,7 @@ from datetime import timedelta from enum import Enum from zope.interface import Interface, Attribute + ALWAYS_REPLY = timedelta() diff --git a/src/mailman/interfaces/bans.py b/src/mailman/interfaces/bans.py index 48b3415c8..ea19abc38 100644 --- a/src/mailman/interfaces/bans.py +++ b/src/mailman/interfaces/bans.py @@ -17,9 +17,6 @@ """Manager of email address bans.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'IBan', 'IBanManager', diff --git a/src/mailman/interfaces/bounce.py b/src/mailman/interfaces/bounce.py index 8a0ffd4b2..9556830eb 100644 --- a/src/mailman/interfaces/bounce.py +++ b/src/mailman/interfaces/bounce.py @@ -17,9 +17,6 @@ """Interface to bounce detection components.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'BounceContext', 'IBounceEvent', diff --git a/src/mailman/interfaces/chain.py b/src/mailman/interfaces/chain.py index 85bad22a4..788112f0b 100644 --- a/src/mailman/interfaces/chain.py +++ b/src/mailman/interfaces/chain.py @@ -17,9 +17,6 @@ """Interfaces describing the basics of chains and links.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'AcceptEvent', 'AcceptOwnerEvent', diff --git a/src/mailman/interfaces/command.py b/src/mailman/interfaces/command.py index 720e59ee8..a73d0b1de 100644 --- a/src/mailman/interfaces/command.py +++ b/src/mailman/interfaces/command.py @@ -17,9 +17,6 @@ """Interfaces defining email commands.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ContinueProcessing', 'ICLISubCommand', diff --git a/src/mailman/interfaces/configuration.py b/src/mailman/interfaces/configuration.py index 65547d44d..49d0bb3c6 100644 --- a/src/mailman/interfaces/configuration.py +++ b/src/mailman/interfaces/configuration.py @@ -17,9 +17,6 @@ """Configuration system interface.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ConfigurationUpdatedEvent', 'IConfiguration', @@ -27,9 +24,8 @@ __all__ = [ ] -from zope.interface import Interface - from mailman.core.errors import MailmanError +from zope.interface import Interface diff --git a/src/mailman/interfaces/database.py b/src/mailman/interfaces/database.py index 9ca05b747..37830329a 100644 --- a/src/mailman/interfaces/database.py +++ b/src/mailman/interfaces/database.py @@ -17,9 +17,6 @@ """Interfaces for database interaction.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'DatabaseError', 'IDatabase', diff --git a/src/mailman/interfaces/digests.py b/src/mailman/interfaces/digests.py index c5231e488..c343669f5 100644 --- a/src/mailman/interfaces/digests.py +++ b/src/mailman/interfaces/digests.py @@ -17,9 +17,6 @@ """One last digest.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'IOneLastDigest' ] diff --git a/src/mailman/interfaces/domain.py b/src/mailman/interfaces/domain.py index a4f929ddb..aed76ebe9 100644 --- a/src/mailman/interfaces/domain.py +++ b/src/mailman/interfaces/domain.py @@ -17,9 +17,6 @@ """Interface representing domains.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'BadDomainSpecificationError', 'DomainCreatedEvent', diff --git a/src/mailman/interfaces/errors.py b/src/mailman/interfaces/errors.py index 187c329b3..ecb4270f1 100644 --- a/src/mailman/interfaces/errors.py +++ b/src/mailman/interfaces/errors.py @@ -22,9 +22,6 @@ components. More specific exceptions will be located in the relevant interfaces. """ -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'MailmanError', ] diff --git a/src/mailman/interfaces/handler.py b/src/mailman/interfaces/handler.py index 2e6c3fa20..6c52f017b 100644 --- a/src/mailman/interfaces/handler.py +++ b/src/mailman/interfaces/handler.py @@ -17,9 +17,6 @@ """Interface describing a pipeline handler.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'IHandler', ] diff --git a/src/mailman/interfaces/languages.py b/src/mailman/interfaces/languages.py index 9e88dd78f..810de7af1 100644 --- a/src/mailman/interfaces/languages.py +++ b/src/mailman/interfaces/languages.py @@ -17,9 +17,6 @@ """Interfaces for managing languages.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ILanguage', 'ILanguageManager', diff --git a/src/mailman/interfaces/listmanager.py b/src/mailman/interfaces/listmanager.py index 7fe8ed35a..27b6b5838 100644 --- a/src/mailman/interfaces/listmanager.py +++ b/src/mailman/interfaces/listmanager.py @@ -17,9 +17,6 @@ """Interface for list storage, deleting, and finding.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'IListManager', 'ListAlreadyExistsError', diff --git a/src/mailman/interfaces/mailinglist.py b/src/mailman/interfaces/mailinglist.py index 3900e3349..2d145dc6c 100644 --- a/src/mailman/interfaces/mailinglist.py +++ b/src/mailman/interfaces/mailinglist.py @@ -17,9 +17,6 @@ """Interface for a mailing list.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'IAcceptableAlias', 'IAcceptableAliasSet', @@ -32,9 +29,8 @@ __all__ = [ from enum import Enum -from zope.interface import Interface, Attribute - from mailman.interfaces.member import MemberRole +from zope.interface import Interface, Attribute diff --git a/src/mailman/interfaces/member.py b/src/mailman/interfaces/member.py index e2a5dc4fe..9e3917b86 100644 --- a/src/mailman/interfaces/member.py +++ b/src/mailman/interfaces/member.py @@ -17,9 +17,6 @@ """Interface describing the basics of a member.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'AlreadySubscribedError', 'DeliveryMode', @@ -37,9 +34,8 @@ __all__ = [ from enum import Enum -from zope.interface import Interface, Attribute - from mailman.core.errors import MailmanError +from zope.interface import Interface, Attribute diff --git a/src/mailman/interfaces/messages.py b/src/mailman/interfaces/messages.py index 7b99578c4..c78971dfd 100644 --- a/src/mailman/interfaces/messages.py +++ b/src/mailman/interfaces/messages.py @@ -17,9 +17,6 @@ """The message storage service.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'IMessage', 'IMessageStore', diff --git a/src/mailman/interfaces/mime.py b/src/mailman/interfaces/mime.py index 4729c426c..11feca331 100644 --- a/src/mailman/interfaces/mime.py +++ b/src/mailman/interfaces/mime.py @@ -17,9 +17,6 @@ """MIME content filtering.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'FilterAction', 'FilterType', diff --git a/src/mailman/interfaces/mlistrequest.py b/src/mailman/interfaces/mlistrequest.py index 77451f8bf..2af0f1776 100644 --- a/src/mailman/interfaces/mlistrequest.py +++ b/src/mailman/interfaces/mlistrequest.py @@ -17,9 +17,6 @@ """Interface for a web request accessing a mailing list.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'IMailingListRequest', ] diff --git a/src/mailman/interfaces/mta.py b/src/mailman/interfaces/mta.py index 22c3d121e..44c0aba42 100644 --- a/src/mailman/interfaces/mta.py +++ b/src/mailman/interfaces/mta.py @@ -17,9 +17,6 @@ """Interface for mail transport agent integration.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'IMailTransportAgentAliases', 'IMailTransportAgentDelivery', @@ -27,9 +24,8 @@ __all__ = [ ] -from zope.interface import Interface - from mailman.core.errors import MailmanError +from zope.interface import Interface diff --git a/src/mailman/interfaces/nntp.py b/src/mailman/interfaces/nntp.py index 8e73c2c50..46d705489 100644 --- a/src/mailman/interfaces/nntp.py +++ b/src/mailman/interfaces/nntp.py @@ -17,9 +17,6 @@ """NNTP and newsgroup interfaces.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'NewsgroupModeration', ] diff --git a/src/mailman/interfaces/pending.py b/src/mailman/interfaces/pending.py index a97552306..ff156d95a 100644 --- a/src/mailman/interfaces/pending.py +++ b/src/mailman/interfaces/pending.py @@ -22,9 +22,6 @@ maps these events to a unique hash that can be used as a token for end user confirmation. """ -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'IPendable', 'IPended', diff --git a/src/mailman/interfaces/permissions.py b/src/mailman/interfaces/permissions.py index 8d06e9ffb..cf32936ff 100644 --- a/src/mailman/interfaces/permissions.py +++ b/src/mailman/interfaces/permissions.py @@ -17,9 +17,6 @@ """Interfaces for various permissions.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'IPostingPermission', ] diff --git a/src/mailman/interfaces/pipeline.py b/src/mailman/interfaces/pipeline.py index 817ebfc62..4ce11d8a6 100644 --- a/src/mailman/interfaces/pipeline.py +++ b/src/mailman/interfaces/pipeline.py @@ -17,9 +17,6 @@ """Interface for describing pipelines.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'IPipeline', ] @@ -37,4 +34,3 @@ class IPipeline(Interface): def __iter__(): """Iterate over all the handlers in this pipeline.""" - diff --git a/src/mailman/interfaces/preferences.py b/src/mailman/interfaces/preferences.py index 27ae49faa..b68d7a0f5 100644 --- a/src/mailman/interfaces/preferences.py +++ b/src/mailman/interfaces/preferences.py @@ -17,9 +17,6 @@ """Interface for preferences.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'IPreferences', ] diff --git a/src/mailman/interfaces/registrar.py b/src/mailman/interfaces/registrar.py index 413f3284e..df7c4ed86 100644 --- a/src/mailman/interfaces/registrar.py +++ b/src/mailman/interfaces/registrar.py @@ -22,9 +22,6 @@ etc. than the IUserManager. The latter does no validation, syntax checking, or confirmation, while this interface does. """ -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ConfirmationNeededEvent', 'IRegistrar', diff --git a/src/mailman/interfaces/requests.py b/src/mailman/interfaces/requests.py index 4dcb3cace..ed3540e4c 100644 --- a/src/mailman/interfaces/requests.py +++ b/src/mailman/interfaces/requests.py @@ -21,9 +21,6 @@ The request database handles events that must be approved by the list moderators, such as subscription requests and held messages. """ -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'IListRequests', 'RequestType', diff --git a/src/mailman/interfaces/roster.py b/src/mailman/interfaces/roster.py index c4a7f5567..79c9fd573 100644 --- a/src/mailman/interfaces/roster.py +++ b/src/mailman/interfaces/roster.py @@ -17,9 +17,6 @@ """Interface for a roster of members.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'IRoster', ] diff --git a/src/mailman/interfaces/rules.py b/src/mailman/interfaces/rules.py index feb773fca..2118a0b43 100644 --- a/src/mailman/interfaces/rules.py +++ b/src/mailman/interfaces/rules.py @@ -17,9 +17,6 @@ """Interface describing the basics of rules.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'IRule', ] diff --git a/src/mailman/interfaces/runner.py b/src/mailman/interfaces/runner.py index 9cb554597..74038ab71 100644 --- a/src/mailman/interfaces/runner.py +++ b/src/mailman/interfaces/runner.py @@ -17,9 +17,6 @@ """Interface for runners.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'IRunner', 'RunnerCrashEvent', diff --git a/src/mailman/interfaces/styles.py b/src/mailman/interfaces/styles.py index 33ab8ee84..615cb6abd 100644 --- a/src/mailman/interfaces/styles.py +++ b/src/mailman/interfaces/styles.py @@ -17,9 +17,6 @@ """Interfaces for list styles.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'DuplicateStyleError', 'IStyle', @@ -27,8 +24,8 @@ __all__ = [ ] -from zope.interface import Interface, Attribute from mailman.interfaces.errors import MailmanError +from zope.interface import Interface, Attribute diff --git a/src/mailman/interfaces/subscriptions.py b/src/mailman/interfaces/subscriptions.py index 64d4280d6..036cc4631 100644 --- a/src/mailman/interfaces/subscriptions.py +++ b/src/mailman/interfaces/subscriptions.py @@ -17,18 +17,14 @@ """Membership interface for REST.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ISubscriptionService', ] -from zope.interface import Interface - from mailman.interfaces.errors import MailmanError from mailman.interfaces.member import DeliveryMode, MemberRole +from zope.interface import Interface diff --git a/src/mailman/interfaces/switchboard.py b/src/mailman/interfaces/switchboard.py index ae613700a..c763c142b 100644 --- a/src/mailman/interfaces/switchboard.py +++ b/src/mailman/interfaces/switchboard.py @@ -17,9 +17,6 @@ """Interface for switchboards.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ISwitchboard', ] diff --git a/src/mailman/interfaces/system.py b/src/mailman/interfaces/system.py index 83992629c..36aa3279e 100644 --- a/src/mailman/interfaces/system.py +++ b/src/mailman/interfaces/system.py @@ -17,9 +17,6 @@ """System information.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ISystem', ] diff --git a/src/mailman/interfaces/templates.py b/src/mailman/interfaces/templates.py index de5fa11a9..9e39747a3 100644 --- a/src/mailman/interfaces/templates.py +++ b/src/mailman/interfaces/templates.py @@ -17,9 +17,6 @@ """Template downloader with cache.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ITemplateLoader', ] diff --git a/src/mailman/interfaces/user.py b/src/mailman/interfaces/user.py index e1c1df243..c42bb6c33 100644 --- a/src/mailman/interfaces/user.py +++ b/src/mailman/interfaces/user.py @@ -17,9 +17,6 @@ """Interface describing the basics of a user.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'IUser', 'PasswordChangeEvent', @@ -27,9 +24,8 @@ __all__ = [ ] -from zope.interface import Interface, Attribute - from mailman.interfaces.address import AddressError +from zope.interface import Interface, Attribute diff --git a/src/mailman/interfaces/usermanager.py b/src/mailman/interfaces/usermanager.py index f37d39f6a..ab58347dc 100644 --- a/src/mailman/interfaces/usermanager.py +++ b/src/mailman/interfaces/usermanager.py @@ -17,9 +17,6 @@ """Interface describing the user management service.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'IUserManager', ] diff --git a/src/mailman/languages/language.py b/src/mailman/languages/language.py index 35e142559..de406e10c 100644 --- a/src/mailman/languages/language.py +++ b/src/mailman/languages/language.py @@ -18,17 +18,13 @@ """The representation of a language.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Language', ] -from zope.interface import implementer - from mailman.interfaces.languages import ILanguage +from zope.interface import implementer diff --git a/src/mailman/languages/manager.py b/src/mailman/languages/manager.py index 7e73c11b0..2732d490a 100644 --- a/src/mailman/languages/manager.py +++ b/src/mailman/languages/manager.py @@ -17,20 +17,16 @@ """Language manager.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'LanguageManager', ] -from zope.component import getUtility -from zope.interface import implementer - from mailman.interfaces.configuration import ConfigurationUpdatedEvent from mailman.interfaces.languages import ILanguageManager from mailman.languages.language import Language +from zope.component import getUtility +from zope.interface import implementer diff --git a/src/mailman/model/address.py b/src/mailman/model/address.py index 5d1994567..5ded77dd8 100644 --- a/src/mailman/model/address.py +++ b/src/mailman/model/address.py @@ -17,26 +17,22 @@ """Model for addresses.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Address', ] from email.utils import formataddr +from mailman.database.model import Model +from mailman.interfaces.address import ( + AddressVerificationEvent, IAddress, IEmailValidator) +from mailman.utilities.datetime import now from sqlalchemy import Column, DateTime, ForeignKey, Integer, Unicode from sqlalchemy.orm import relationship, backref from zope.component import getUtility from zope.event import notify from zope.interface import implementer -from mailman.database.model import Model -from mailman.interfaces.address import ( - AddressVerificationEvent, IAddress, IEmailValidator) -from mailman.utilities.datetime import now - @implementer(IAddress) diff --git a/src/mailman/model/autorespond.py b/src/mailman/model/autorespond.py index cfb9e017d..332d04521 100644 --- a/src/mailman/model/autorespond.py +++ b/src/mailman/model/autorespond.py @@ -17,25 +17,21 @@ """Module stuff.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'AutoResponseRecord', 'AutoResponseSet', ] -from sqlalchemy import Column, Date, ForeignKey, Integer, desc -from sqlalchemy.orm import relationship -from zope.interface import implementer - from mailman.database.model import Model from mailman.database.transaction import dbconnection from mailman.database.types import Enum from mailman.interfaces.autorespond import ( IAutoResponseRecord, IAutoResponseSet, Response) from mailman.utilities.datetime import today +from sqlalchemy import Column, Date, ForeignKey, Integer, desc +from sqlalchemy.orm import relationship +from zope.interface import implementer diff --git a/src/mailman/model/bans.py b/src/mailman/model/bans.py index 8678fc1e7..3ad11cbf6 100644 --- a/src/mailman/model/bans.py +++ b/src/mailman/model/bans.py @@ -17,9 +17,6 @@ """Ban manager.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'BanManager', ] @@ -27,12 +24,11 @@ __all__ = [ import re -from sqlalchemy import Column, Integer, Unicode -from zope.interface import implementer - from mailman.database.model import Model from mailman.database.transaction import dbconnection from mailman.interfaces.bans import IBan, IBanManager +from sqlalchemy import Column, Integer, Unicode +from zope.interface import implementer diff --git a/src/mailman/model/bounce.py b/src/mailman/model/bounce.py index 26ebbe0c6..585a92594 100644 --- a/src/mailman/model/bounce.py +++ b/src/mailman/model/bounce.py @@ -17,9 +17,6 @@ """Bounce support.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'BounceEvent', 'BounceProcessor', @@ -27,15 +24,14 @@ __all__ = [ -from sqlalchemy import Boolean, Column, DateTime, Integer, Unicode -from zope.interface import implementer - from mailman.database.model import Model from mailman.database.transaction import dbconnection from mailman.database.types import Enum from mailman.interfaces.bounce import ( BounceContext, IBounceEvent, IBounceProcessor) from mailman.utilities.datetime import now +from sqlalchemy import Boolean, Column, DateTime, Integer, Unicode +from zope.interface import implementer diff --git a/src/mailman/model/digests.py b/src/mailman/model/digests.py index 7bfd512b6..8e8f7dedd 100644 --- a/src/mailman/model/digests.py +++ b/src/mailman/model/digests.py @@ -17,22 +17,18 @@ """One last digest.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'OneLastDigest', ] -from sqlalchemy import Column, Integer, ForeignKey -from sqlalchemy.orm import relationship -from zope.interface import implementer - from mailman.database.model import Model from mailman.database.types import Enum from mailman.interfaces.digests import IOneLastDigest from mailman.interfaces.member import DeliveryMode +from sqlalchemy import Column, Integer, ForeignKey +from sqlalchemy.orm import relationship +from zope.interface import implementer diff --git a/src/mailman/model/docs/addresses.rst b/src/mailman/model/docs/addresses.rst index 795afe43c..9f34efdec 100644 --- a/src/mailman/model/docs/addresses.rst +++ b/src/mailman/model/docs/addresses.rst @@ -205,23 +205,9 @@ case-preserved version are available on attributes of the `IAddress` object. FPERSON@example.com Because addresses are case-insensitive for all other purposes, you cannot -create an address that differs only in case. - - >>> user_manager.create_address('fperson@example.com') - Traceback (most recent call last): - ... - ExistingAddressError: FPERSON@example.com - >>> user_manager.create_address('fperson@EXAMPLE.COM') - Traceback (most recent call last): - ... - ExistingAddressError: FPERSON@example.com - >>> user_manager.create_address('FPERSON@example.com') - Traceback (most recent call last): - ... - ExistingAddressError: FPERSON@example.com - -You can get the address using either the lower cased version or case-preserved -version. In fact, searching for an address is case insensitive. +create an address that differs only in case. You can get the address using +either the lower cased version or case-preserved version. In fact, searching +for an address is case insensitive. >>> print(user_manager.get_address('fperson@example.com').email) fperson@example.com diff --git a/src/mailman/model/docs/domains.rst b/src/mailman/model/docs/domains.rst index 153f6c19d..abb594a62 100644 --- a/src/mailman/model/docs/domains.rst +++ b/src/mailman/model/docs/domains.rst @@ -108,12 +108,7 @@ In the global domain manager, domains are indexed by their email host name. base_url: http://lists.example.net, contact_address: postmaster@example.com> - >>> print(manager['doesnotexist.com']) - Traceback (most recent call last): - ... - KeyError: u'doesnotexist.com' - -As with a dictionary, you can also get the domain. If the domain does not +As with dictionaries, you can also get the domain. If the domain does not exist, ``None`` or a default is returned. :: @@ -128,13 +123,6 @@ exist, ``None`` or a default is returned. >>> print(manager.get('doesnotexist.com', 'blahdeblah')) blahdeblah -Non-existent domains cannot be removed. - - >>> manager.remove('doesnotexist.com') - Traceback (most recent call last): - ... - KeyError: u'doesnotexist.com' - Confirmation tokens =================== diff --git a/src/mailman/model/docs/languages.rst b/src/mailman/model/docs/languages.rst index fedea0e6e..4ed1f8ef2 100644 --- a/src/mailman/model/docs/languages.rst +++ b/src/mailman/model/docs/languages.rst @@ -62,7 +62,7 @@ You can iterate over all the known language codes. >>> mgr.add('pl', 'iso-8859-2', 'Polish') <Language [pl] Polish> >>> sorted(mgr.codes) - [u'en', u'it', u'pl'] + ['en', 'it', 'pl'] You can iterate over all the known languages. @@ -89,7 +89,7 @@ You can get a particular language by its code. >>> print(mgr['xx'].code) Traceback (most recent call last): ... - KeyError: u'xx' + KeyError: 'xx' >>> print(mgr.get('it').description) Italian >>> print(mgr.get('xx')) diff --git a/src/mailman/model/docs/listmanager.rst b/src/mailman/model/docs/listmanager.rst index 151bee1fe..8ff6ad3b0 100644 --- a/src/mailman/model/docs/listmanager.rst +++ b/src/mailman/model/docs/listmanager.rst @@ -34,22 +34,6 @@ the mailing list to the system. >>> print(mlist.list_id) test.example.com -If you try to create a mailing list with the same name as an existing list, -you will get an exception. - - >>> list_manager.create('test@example.com') - Traceback (most recent call last): - ... - ListAlreadyExistsError: test@example.com - -It is an error to create a mailing list that isn't a fully qualified list name -(i.e. posting address). - - >>> list_manager.create('foo') - Traceback (most recent call last): - ... - InvalidEmailAddressError: foo - Deleting a mailing list ======================= diff --git a/src/mailman/model/docs/mailinglist.rst b/src/mailman/model/docs/mailinglist.rst index 3d01710c5..00a01662b 100644 --- a/src/mailman/model/docs/mailinglist.rst +++ b/src/mailman/model/docs/mailinglist.rst @@ -114,7 +114,8 @@ Subscribing users An alternative way of subscribing to a mailing list is as a user with a preferred address. This way the user can change their subscription address just by changing their preferred address. -:: + +The user must have a preferred address. >>> from mailman.utilities.datetime import now >>> user = user_manager.create_user('dperson@example.com', 'Dave Person') @@ -122,6 +123,8 @@ just by changing their preferred address. >>> address.verified_on = now() >>> user.preferred_address = address +The preferred address is used in the subscription. + >>> mlist.subscribe(user) <Member: Dave Person <dperson@example.com> on aardvark@example.com as MemberRole.member> @@ -132,6 +135,10 @@ just by changing their preferred address. <Member: Dave Person <dperson@example.com> on aardvark@example.com as MemberRole.member> +If the user's preferred address changes, their subscribed email address also +changes automatically. +:: + >>> new_address = user.register('dave.person@example.com') >>> new_address.verified_on = now() >>> user.preferred_address = new_address @@ -143,31 +150,12 @@ just by changing their preferred address. <Member: dave.person@example.com on aardvark@example.com as MemberRole.member> -A user is not allowed to subscribe more than once to the mailing list. - - >>> mlist.subscribe(user) - Traceback (most recent call last): - ... - AlreadySubscribedError: <User "Dave Person" (1) at ...> - is already a MemberRole.member of mailing list aardvark@example.com - -However, they are allowed to subscribe again with a specific address, even if -this address is their preferred address. +A user is allowed to explicitly subscribe again with a specific address, even +if this address is their preferred address. >>> mlist.subscribe(user.preferred_address) <Member: dave.person@example.com on aardvark@example.com as MemberRole.member> -A user cannot subscribe to a mailing list without a preferred address. - - >>> user = user_manager.create_user('eperson@example.com', 'Elly Person') - >>> address = list(user.addresses)[0] - >>> address.verified_on = now() - >>> mlist.subscribe(user) - Traceback (most recent call last): - ... - MissingPreferredAddressError: User must have a preferred address: - <User "Elly Person" (2) at ...> - .. _`RFC 2369`: http://www.faqs.org/rfcs/rfc2369.html diff --git a/src/mailman/model/docs/membership.rst b/src/mailman/model/docs/membership.rst index 2a6c99fc0..60ccd1ac1 100644 --- a/src/mailman/model/docs/membership.rst +++ b/src/mailman/model/docs/membership.rst @@ -2,10 +2,10 @@ List memberships ================ -Users represent people in Mailman. Users control email addresses, and rosters -are collections of members. A member gives an email address a role, such as -`member`, `administrator`, or `moderator`. Even nonmembers are represented by -a roster. +Users represent people in Mailman, members represent subscriptions. Users +control email addresses, and rosters are collections of members. A member +ties a subscribed email address to a role, such as `member`, `administrator`, +or `moderator`. Even non-members are represented by a roster. Roster sets are collections of rosters and a mailing list has a single roster set that contains all its members, regardless of that member's role. @@ -228,18 +228,6 @@ regardless of their role. fperson@example.com MemberRole.nonmember -Double subscriptions -==================== - -It is an error to subscribe someone to a list with the same role twice. - - >>> mlist.subscribe(address_1, MemberRole.owner) - Traceback (most recent call last): - ... - AlreadySubscribedError: aperson@example.com is already a MemberRole.owner - of mailing list ant@example.com - - Moderation actions ================== @@ -276,7 +264,7 @@ Changing subscriptions When a user is subscribed to a mailing list via a specific address they control (as opposed to being subscribed with their preferred address), they can change their delivery address by setting the appropriate parameter. Note -though that the address their changing to must be verified. +though that the address they're changing to must be verified. >>> bee = create_list('bee@example.com') >>> gwen = user_manager.create_user('gwen@example.com') @@ -290,20 +278,6 @@ Gwen gets a email address. >>> new_address = gwen.register('gperson@example.com') -She wants to change her membership in the `test` mailing list to use her new -address, but the address is not yet verified. - - >>> gwen_member.address = new_address - Traceback (most recent call last): - ... - UnverifiedAddressError: gperson@example.com - -Her membership has not changed. - - >>> for m in bee.members.members: - ... print(m.member_id.int, m.mailing_list.list_id, m.address.email) - 7 bee.example.com gwen@example.com - Gwen verifies her email address, and updates her membership. >>> from mailman.utilities.datetime import now diff --git a/src/mailman/model/docs/messagestore.rst b/src/mailman/model/docs/messagestore.rst index f2f2ca9d2..933ca5619 100644 --- a/src/mailman/model/docs/messagestore.rst +++ b/src/mailman/model/docs/messagestore.rst @@ -6,28 +6,20 @@ The message store is a collection of messages keyed off of ``Message-ID`` and ``X-Message-ID-Hash`` headers. Either of these values can be combined with the message's ``List-Archive`` header to create a globally unique URI to the message object in the internet facing interface of the message store. The -``X-Message-ID-Hash`` is the Base32 SHA1 hash of the ``Message-ID``. +``X-Message-ID-Hash`` is the base-32 SHA1 hash of the ``Message-ID``. >>> from mailman.interfaces.messages import IMessageStore >>> from zope.component import getUtility >>> message_store = getUtility(IMessageStore) -If you try to add a message to the store which is missing the ``Message-ID`` -header, you will get an exception. +A message with a ``Message-ID`` header can be stored. >>> msg = message_from_string("""\ ... Subject: An important message + ... Message-ID: <87myycy5eh.fsf@uwakimon.sk.tsukuba.ac.jp> ... ... This message is very important. ... """) - >>> message_store.add(msg) - Traceback (most recent call last): - ... - ValueError: Exactly one Message-ID header required - -However, if the message has a ``Message-ID`` header, it can be stored. - - >>> msg['Message-ID'] = '<87myycy5eh.fsf@uwakimon.sk.tsukuba.ac.jp>' >>> x_message_id_hash = message_store.add(msg) >>> print(x_message_id_hash) AGDWSNXXKCWEILKKNYTBOHRDQGOX3Y35 @@ -97,15 +89,7 @@ Deleting messages from the store ================================ You delete a message from the storage service by providing the ``Message-ID`` -for the message you want to delete. If you try to delete a ``Message-ID`` -that isn't in the store, you get an exception. - - >>> message_store.delete_message('nothing') - Traceback (most recent call last): - ... - LookupError: nothing - -But if you delete an existing message, it really gets deleted. +for the message you want to delete. >>> message_id = message['message-id'] >>> message_store.delete_message(message_id) diff --git a/src/mailman/model/docs/pending.rst b/src/mailman/model/docs/pending.rst index d8206b264..a634322a1 100644 --- a/src/mailman/model/docs/pending.rst +++ b/src/mailman/model/docs/pending.rst @@ -33,12 +33,12 @@ token that can be used in urls and such. >>> len(token) 40 -There's not much you can do with tokens except to `confirm` them, which -basically means returning the ``IPendable`` structure (as a dictionary) from -the database that matches the token. If the token isn't in the database, None -is returned. +There's not much you can do with tokens except to *confirm* them, which +basically means returning the `IPendable` structure (as a dictionary) from the +database that matches the token. If the token isn't in the database, None is +returned. - >>> pendable = pendingdb.confirm(bytes('missing')) + >>> pendable = pendingdb.confirm(b'missing') >>> print(pendable) None >>> pendable = pendingdb.confirm(token) diff --git a/src/mailman/model/docs/registration.rst b/src/mailman/model/docs/registration.rst index 32ee27316..55e99f23a 100644 --- a/src/mailman/model/docs/registration.rst +++ b/src/mailman/model/docs/registration.rst @@ -8,7 +8,7 @@ additional information they may supply. All registered email addresses must be verified before Mailman will send them any list traffic. The ``IUserManager`` manages users, but it does so at a fairly low level. -Specifically, it does not handle verifications, email address syntax validity +Specifically, it does not handle verification, email address syntax validity checks, etc. The ``IRegistrar`` is the interface to the object handling all this stuff. @@ -19,7 +19,7 @@ this stuff. Here is a helper function to check the token strings. >>> def check_token(token): - ... assert isinstance(token, basestring), 'Not a string' + ... assert isinstance(token, str), 'Not a string' ... assert len(token) == 40, 'Unexpected length: %d' % len(token) ... assert token.isalnum(), 'Not alphanumeric' ... print('ok') @@ -47,31 +47,6 @@ Some amount of sanity checks are performed on the email address, although honestly, not as much as probably should be done. Still, some patently bad addresses are rejected outright. - >>> registrar.register(mlist, '') - Traceback (most recent call last): - ... - InvalidEmailAddressError - >>> registrar.register(mlist, 'some name@example.com') - Traceback (most recent call last): - ... - InvalidEmailAddressError: some name@example.com - >>> registrar.register(mlist, '<script>@example.com') - Traceback (most recent call last): - ... - InvalidEmailAddressError: <script>@example.com - >>> registrar.register(mlist, '\xa0@example.com') - Traceback (most recent call last): - ... - InvalidEmailAddressError: \xa0@example.com - >>> registrar.register(mlist, 'noatsign') - Traceback (most recent call last): - ... - InvalidEmailAddressError: noatsign - >>> registrar.register(mlist, 'nodom@ain') - Traceback (most recent call last): - ... - InvalidEmailAddressError: nodom@ain - Register an email address ========================= @@ -149,9 +124,9 @@ message is sent to the user in order to verify the registered address. <BLANKLINE> >>> dump_msgdata(items[0].msgdata) _parsemsg : False - listname : alpha@example.com + listid : alpha.example.com nodecorate : True - recipients : set([u'aperson@example.com']) + recipients : {'aperson@example.com'} reduced_list_headers: True version : 3 @@ -312,7 +287,7 @@ Corner cases If you try to confirm a token that doesn't exist in the pending database, the confirm method will just return False. - >>> registrar.confirm(bytes('no token')) + >>> registrar.confirm(bytes(b'no token')) False Likewise, if you try to confirm, through the `IUserRegistrar` interface, a diff --git a/src/mailman/model/docs/usermanager.rst b/src/mailman/model/docs/usermanager.rst index 9a8c35c00..ba328b54b 100644 --- a/src/mailman/model/docs/usermanager.rst +++ b/src/mailman/model/docs/usermanager.rst @@ -44,7 +44,7 @@ A user can be assigned a real name. A user can be assigned a password. - >>> user.password = b'secret' + >>> user.password = 'secret' >>> dump_list(user.password for user in user_manager.users) secret diff --git a/src/mailman/model/docs/users.rst b/src/mailman/model/docs/users.rst index 2e7333944..0b926d6a7 100644 --- a/src/mailman/model/docs/users.rst +++ b/src/mailman/model/docs/users.rst @@ -20,7 +20,7 @@ User data Users may have a real name and a password. >>> user_1 = user_manager.create_user() - >>> user_1.password = b'my password' + >>> user_1.password = 'my password' >>> user_1.display_name = 'Zoe Person' >>> dump_list(user.display_name for user in user_manager.users) Zoe Person @@ -30,7 +30,7 @@ Users may have a real name and a password. The password and real name can be changed at any time. >>> user_1.display_name = 'Zoe X. Person' - >>> user_1.password = b'another password' + >>> user_1.password = 'another password' >>> dump_list(user.display_name for user in user_manager.users) Zoe X. Person >>> dump_list(user.password for user in user_manager.users) @@ -44,7 +44,7 @@ When the user's password is changed, an event is triggered. ... saved_event = event >>> from mailman.testing.helpers import event_subscribers >>> with event_subscribers(save_event): - ... user_1.password = b'changed again' + ... user_1.password = 'changed again' >>> print(saved_event) <PasswordChangeEvent Zoe X. Person> @@ -59,20 +59,13 @@ The event holds a reference to the `IUser` that changed their password. Basic user identification ========================= -Although rarely visible to users, every user has a unique ID in Mailman, which -never changes. This ID is generated randomly at the time the user is created, -and is represented by a UUID. +Although rarely visible to users, every user has a unique immutable ID. This +ID is generated randomly at the time the user is created, and is represented +by a UUID. >>> print(user_1.user_id) 00000000-0000-0000-0000-000000000001 -The user id cannot change. - - >>> user_1.user_id = 'foo' - Traceback (most recent call last): - ... - AttributeError: can't set attribute - User records also have a date on which they where created. # The test suite uses a predictable timestamp. @@ -84,8 +77,8 @@ Users addresses =============== One of the pieces of information that a user links to is a set of email -addresses they control, in the form of IAddress objects. A user can control -many addresses, but addresses may be controlled by only one user. +addresses they control, in the form of ``IAddress`` objects. A user can +control many addresses, but addresses may be linked to only one user. The easiest way to link a user to an address is to just register the new address on a user object. @@ -114,14 +107,6 @@ You can also create the address separately and then link it to the user. <BLANKLINE> Zoe Person -But don't try to link an address to more than one user. - - >>> another_user = user_manager.create_user() - >>> another_user.link(address_1) - Traceback (most recent call last): - ... - AddressAlreadyLinkedError: zperson@example.net - You can also ask whether a given user controls a given address. >>> user_1.controls(address_1.email) @@ -149,17 +134,6 @@ Addresses can also be unlinked from a user. >>> print(user_manager.get_user('aperson@example.net')) None -But don't try to unlink the address from a user it's not linked to. - - >>> user_1.unlink(address_1) - Traceback (most recent call last): - ... - AddressNotLinkedError: zperson@example.net - >>> another_user.unlink(address_1) - Traceback (most recent call last): - ... - AddressNotLinkedError: zperson@example.net - Preferred address ================= @@ -183,20 +157,10 @@ preferred address. >>> print(user_2.preferred_address) None -The preferred address must be explicitly registered, however only verified -address may be registered as preferred. - - >>> anne - <Address: Anne Person <anne@example.com> [not verified] at ...> - >>> user_2.preferred_address = anne - Traceback (most recent call last): - ... - UnverifiedAddressError: Anne Person <anne@example.com> - -Once the address has been verified though, it can be set as the preferred -address, but only if the address is either controlled by the user or -uncontrolled. In the latter case, setting it as the preferred address makes -it controlled by the user. +Once the address has been verified, it can be set as the preferred address, +but only if the address is either controlled by the user or uncontrolled. In +the latter case, setting it as the preferred address makes it controlled by +the user. :: >>> from mailman.utilities.datetime import now @@ -217,17 +181,6 @@ it controlled by the user. >>> user_2.controls(aperson.email) True - >>> zperson = user_manager.get_address('zperson@example.com') - >>> zperson.verified_on = now() - >>> user_2.controls(zperson.email) - False - >>> user_1.controls(zperson.email) - True - >>> user_2.preferred_address = zperson - Traceback (most recent call last): - ... - AddressAlreadyLinkedError: Zoe Person <zperson@example.com> - A user can disavow their preferred address. >>> user_2.preferred_address @@ -328,11 +281,11 @@ membership role. >>> from zope.interface.verify import verifyObject >>> verifyObject(IRoster, memberships) True - >>> members = sorted(memberships.members) - >>> len(members) - 4 >>> def sortkey(member): ... return member.address.email, member.mailing_list, member.role.value + >>> members = sorted(memberships.members, key=sortkey) + >>> len(members) + 4 >>> for member in sorted(members, key=sortkey): ... print(member.address.email, member.mailing_list.list_id, ... member.role) diff --git a/src/mailman/model/domain.py b/src/mailman/model/domain.py index 8290cb755..b9d2c88ab 100644 --- a/src/mailman/model/domain.py +++ b/src/mailman/model/domain.py @@ -17,26 +17,22 @@ """Domains.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Domain', 'DomainManager', ] -from sqlalchemy import Column, Integer, Unicode -from urlparse import urljoin, urlparse -from zope.event import notify -from zope.interface import implementer - from mailman.database.model import Model from mailman.database.transaction import dbconnection from mailman.interfaces.domain import ( BadDomainSpecificationError, DomainCreatedEvent, DomainCreatingEvent, DomainDeletedEvent, DomainDeletingEvent, IDomain, IDomainManager) from mailman.model.mailinglist import MailingList +from six.moves.urllib_parse import urljoin, urlparse +from sqlalchemy import Column, Integer, Unicode +from zope.event import notify +from zope.interface import implementer diff --git a/src/mailman/model/language.py b/src/mailman/model/language.py index f4d48fc97..7317b6328 100644 --- a/src/mailman/model/language.py +++ b/src/mailman/model/language.py @@ -17,19 +17,15 @@ """Model for languages.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Language', ] -from sqlalchemy import Column, Integer, Unicode -from zope.interface import implementer - from mailman.database.model import Model from mailman.interfaces.languages import ILanguage +from sqlalchemy import Column, Integer, Unicode +from zope.interface import implementer diff --git a/src/mailman/model/listmanager.py b/src/mailman/model/listmanager.py index 261490a92..7c228bcb9 100644 --- a/src/mailman/model/listmanager.py +++ b/src/mailman/model/listmanager.py @@ -17,17 +17,11 @@ """A mailing list manager.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ListManager', ] -from zope.event import notify -from zope.interface import implementer - from mailman.database.transaction import dbconnection from mailman.interfaces.address import InvalidEmailAddressError from mailman.interfaces.listmanager import ( @@ -36,6 +30,8 @@ from mailman.interfaces.listmanager import ( from mailman.model.mailinglist import MailingList from mailman.model.mime import ContentFilter from mailman.utilities.datetime import now +from zope.event import notify +from zope.interface import implementer diff --git a/src/mailman/model/mailinglist.py b/src/mailman/model/mailinglist.py index 761a78b94..ea3317bb6 100644 --- a/src/mailman/model/mailinglist.py +++ b/src/mailman/model/mailinglist.py @@ -17,9 +17,6 @@ """Model for mailing lists.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'MailingList', ] @@ -27,16 +24,6 @@ __all__ = [ import os -from sqlalchemy import ( - Boolean, Column, DateTime, Float, ForeignKey, Integer, Interval, - LargeBinary, PickleType, Unicode) -from sqlalchemy.event import listen -from sqlalchemy.orm import relationship -from urlparse import urljoin -from zope.component import getUtility -from zope.event import notify -from zope.interface import implementer - from mailman.config import config from mailman.database.model import Model from mailman.database.transaction import dbconnection @@ -65,6 +52,15 @@ from mailman.model.mime import ContentFilter from mailman.model.preferences import Preferences from mailman.utilities.filesystem import makedirs from mailman.utilities.string import expand +from six.moves.urllib_parse import urljoin +from sqlalchemy import ( + Boolean, Column, DateTime, Float, ForeignKey, Integer, Interval, + LargeBinary, PickleType, Unicode) +from sqlalchemy.event import listen +from sqlalchemy.orm import relationship +from zope.component import getUtility +from zope.event import notify +from zope.interface import implementer SPACE = ' ' @@ -482,7 +478,9 @@ class MailingList(Model): Member._user == subscriber).first() if member: raise AlreadySubscribedError( - self.fqdn_listname, subscriber, role) + self.fqdn_listname, + subscriber.preferred_address.email, + role) else: raise ValueError('subscriber must be an address or user') member = Member(role=role, diff --git a/src/mailman/model/member.py b/src/mailman/model/member.py index 9da9d5d0d..19a30074e 100644 --- a/src/mailman/model/member.py +++ b/src/mailman/model/member.py @@ -17,18 +17,10 @@ """Model for members.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Member', ] -from sqlalchemy import Column, ForeignKey, Integer, Unicode -from sqlalchemy.orm import relationship -from zope.component import getUtility -from zope.event import notify -from zope.interface import implementer from mailman.core.constants import system_preferences from mailman.database.model import Model @@ -42,6 +34,11 @@ from mailman.interfaces.member import ( from mailman.interfaces.user import IUser, UnverifiedAddressError from mailman.interfaces.usermanager import IUserManager from mailman.utilities.uid import UniqueIDFactory +from sqlalchemy import Column, ForeignKey, Integer, Unicode +from sqlalchemy.orm import relationship +from zope.component import getUtility +from zope.event import notify +from zope.interface import implementer uid_factory = UniqueIDFactory(context='members') diff --git a/src/mailman/model/message.py b/src/mailman/model/message.py index 691861d46..105066daa 100644 --- a/src/mailman/model/message.py +++ b/src/mailman/model/message.py @@ -17,19 +17,16 @@ """Model for messages.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Message', ] -from sqlalchemy import Column, Integer, LargeBinary, Unicode -from zope.interface import implementer from mailman.database.model import Model from mailman.database.transaction import dbconnection from mailman.interfaces.messages import IMessage +from sqlalchemy import Column, Integer, Unicode +from zope.interface import implementer @@ -42,8 +39,8 @@ class Message(Model): id = Column(Integer, primary_key=True) # This is a Messge-ID field representation, not a database row id. message_id = Column(Unicode) - message_id_hash = Column(LargeBinary) - path = Column(LargeBinary) + message_id_hash = Column(Unicode) + path = Column(Unicode) @dbconnection def __init__(self, store, message_id, message_id_hash, path): diff --git a/src/mailman/model/messagestore.py b/src/mailman/model/messagestore.py index 12b2aef46..05069119c 100644 --- a/src/mailman/model/messagestore.py +++ b/src/mailman/model/messagestore.py @@ -17,9 +17,6 @@ """Model for message stores.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'MessageStore', ] @@ -28,16 +25,15 @@ __all__ = [ import os import errno import base64 +import pickle import hashlib -import cPickle as pickle - -from zope.interface import implementer from mailman.config import config from mailman.database.transaction import dbconnection from mailman.interfaces.messages import IMessageStore from mailman.model.message import Message from mailman.utilities.filesystem import makedirs +from zope.interface import implementer # It could be very bad if you have already stored files and you change this @@ -68,8 +64,8 @@ class MessageStore: raise ValueError( 'Message ID already exists in message store: {0}'.format( message_id)) - shaobj = hashlib.sha1(message_id) - hash32 = base64.b32encode(shaobj.digest()) + shaobj = hashlib.sha1(message_id.encode('utf-8')) + hash32 = base64.b32encode(shaobj.digest()).decode('utf-8') del message['X-Message-ID-Hash'] message['X-Message-ID-Hash'] = hash32 # Calculate the path on disk where we're going to store this message @@ -94,7 +90,7 @@ class MessageStore: # them and try again. while True: try: - with open(path, 'w') as fp: + with open(path, 'wb') as fp: # -1 says to use the highest protocol available. pickle.dump(message, fp, -1) break @@ -106,7 +102,7 @@ class MessageStore: def _get_message(self, row): path = os.path.join(config.MESSAGES_DIR, row.path) - with open(path) as fp: + with open(path, 'rb') as fp: return pickle.load(fp) @dbconnection @@ -118,11 +114,6 @@ class MessageStore: @dbconnection def get_message_by_hash(self, store, message_id_hash): - # It's possible the hash came from a message header, in which case it - # will be a Unicode. However when coming from source code, it may be - # bytes object. Coerce to the latter if necessary; it must be ASCII. - if not isinstance(message_id_hash, bytes): - message_id_hash = message_id_hash.encode('ascii') row = store.query(Message).filter_by( message_id_hash=message_id_hash).first() if row is None: diff --git a/src/mailman/model/mime.py b/src/mailman/model/mime.py index dc6a54437..240fd6e2b 100644 --- a/src/mailman/model/mime.py +++ b/src/mailman/model/mime.py @@ -17,21 +17,17 @@ """The content filter.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ContentFilter' ] -from sqlalchemy import Column, ForeignKey, Integer, Unicode -from sqlalchemy.orm import relationship -from zope.interface import implementer - from mailman.database.model import Model from mailman.database.types import Enum from mailman.interfaces.mime import IContentFilter, FilterType +from sqlalchemy import Column, ForeignKey, Integer, Unicode +from sqlalchemy.orm import relationship +from zope.interface import implementer diff --git a/src/mailman/model/pending.py b/src/mailman/model/pending.py index 49b12c16a..05cea4e29 100644 --- a/src/mailman/model/pending.py +++ b/src/mailman/model/pending.py @@ -17,33 +17,28 @@ """Implementations of the IPendable and IPending interfaces.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Pended', 'Pendings', ] +import json import time import random import hashlib from lazr.config import as_timedelta -from sqlalchemy import ( - Column, DateTime, ForeignKey, Integer, LargeBinary, Unicode) -from sqlalchemy.orm import relationship -from zope.interface import implementer -from zope.interface.verify import verifyObject - from mailman.config import config from mailman.database.model import Model from mailman.database.transaction import dbconnection from mailman.interfaces.pending import ( IPendable, IPended, IPendedKeyValue, IPendings) from mailman.utilities.datetime import now -from mailman.utilities.modules import call_name +from sqlalchemy import Column, DateTime, ForeignKey, Integer, Unicode +from sqlalchemy.orm import relationship +from zope.interface import implementer +from zope.interface.verify import verifyObject @@ -71,7 +66,7 @@ class Pended(Model): __tablename__ = 'pended' id = Column(Integer, primary_key=True) - token = Column(LargeBinary) + token = Column(Unicode) expiration_date = Column(DateTime) key_values = relationship('PendedKeyValue') @@ -108,33 +103,26 @@ class Pendings: right_now = time.time() x = random.random() + right_now % 1.0 + time.clock() % 1.0 # Use sha1 because it produces shorter strings. - token = hashlib.sha1(repr(x)).hexdigest() + token = hashlib.sha1(repr(x).encode('utf-8')).hexdigest() # In practice, we'll never get a duplicate, but we'll be anal # about checking anyway. if store.query(Pended).filter_by(token=token).count() == 0: break else: - raise AssertionError('Could not find a valid pendings token') + raise RuntimeError('Could not find a valid pendings token') # Create the record, and then the individual key/value pairs. pending = Pended( token=token, expiration_date=now() + lifetime) for key, value in pendable.items(): + # Both keys and values must be strings. if isinstance(key, bytes): key = key.decode('utf-8') if isinstance(value, bytes): - value = value.decode('utf-8') - elif type(value) is int: - value = '__builtin__.int\1%s' % value - elif type(value) is float: - value = '__builtin__.float\1%s' % value - elif type(value) is bool: - value = '__builtin__.bool\1%s' % value - elif type(value) is list: - # We expect this to be a list of strings. - value = ('mailman.model.pending.unpack_list\1' + - '\2'.join(value)) - keyval = PendedKeyValue(key=key, value=value) + # Make sure we can turn this back into a bytes. + value = dict(__encoding__='utf-8', + value=value.decode('utf-8')) + keyval = PendedKeyValue(key=key, value=json.dumps(value)) pending.key_values.append(keyval) store.add(pending) return token @@ -155,11 +143,10 @@ class Pendings: entries = store.query(PendedKeyValue).filter( PendedKeyValue.pended_id == pending.id) for keyvalue in entries: - if keyvalue.value is not None and '\1' in keyvalue.value: - type_name, value = keyvalue.value.split('\1', 1) - pendable[keyvalue.key] = call_name(type_name, value) - else: - pendable[keyvalue.key] = keyvalue.value + value = json.loads(keyvalue.value) + if isinstance(value, dict) and '__encoding__' in value: + value = value['value'].encode(value['__encoding__']) + pendable[keyvalue.key] = value if expunge: store.delete(keyvalue) if expunge: @@ -178,8 +165,3 @@ class Pendings: for keyvalue in q: store.delete(keyvalue) store.delete(pending) - - - -def unpack_list(value): - return value.split('\2') diff --git a/src/mailman/model/preferences.py b/src/mailman/model/preferences.py index 1278f80b7..8cec6036e 100644 --- a/src/mailman/model/preferences.py +++ b/src/mailman/model/preferences.py @@ -17,23 +17,19 @@ """Model for preferences.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Preferences', ] -from sqlalchemy import Boolean, Column, Integer, Unicode -from zope.component import getUtility -from zope.interface import implementer - from mailman.database.model import Model from mailman.database.types import Enum from mailman.interfaces.languages import ILanguageManager from mailman.interfaces.member import DeliveryMode, DeliveryStatus from mailman.interfaces.preferences import IPreferences +from sqlalchemy import Boolean, Column, Integer, Unicode +from zope.component import getUtility +from zope.interface import implementer diff --git a/src/mailman/model/requests.py b/src/mailman/model/requests.py index 6b130196d..9d9692b30 100644 --- a/src/mailman/model/requests.py +++ b/src/mailman/model/requests.py @@ -17,25 +17,25 @@ """Implementations of the pending requests interfaces.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ + 'DataPendable', + 'ListRequests', ] -from cPickle import dumps, loads -from datetime import timedelta -from sqlalchemy import Column, ForeignKey, Integer, LargeBinary, Unicode -from sqlalchemy.orm import relationship -from zope.component import getUtility -from zope.interface import implementer +import six +from datetime import timedelta from mailman.database.model import Model from mailman.database.transaction import dbconnection from mailman.database.types import Enum from mailman.interfaces.pending import IPendable, IPendings from mailman.interfaces.requests import IListRequests, RequestType +from six.moves.cPickle import dumps, loads +from sqlalchemy import Column, ForeignKey, Integer, Unicode +from sqlalchemy.orm import relationship +from zope.component import getUtility +from zope.interface import implementer @@ -50,8 +50,8 @@ class DataPendable(dict): # such a way that it will be properly reconstituted when unpended. clean_mapping = {} for key, value in mapping.items(): - assert isinstance(key, basestring) - if not isinstance(value, unicode): + assert isinstance(key, six.string_types) + if not isinstance(value, six.text_type): key = '_pck_' + key value = dumps(value).decode('raw-unicode-escape') clean_mapping[key] = value @@ -154,7 +154,7 @@ class _Request(Model): id = Column(Integer, primary_key=True) key = Column(Unicode) request_type = Column(Enum(RequestType)) - data_hash = Column(LargeBinary) + data_hash = Column(Unicode) mailing_list_id = Column(Integer, ForeignKey('mailinglist.id'), index=True) mailing_list = relationship('MailingList') diff --git a/src/mailman/model/roster.py b/src/mailman/model/roster.py index 54bc11617..7ea3ad2a4 100644 --- a/src/mailman/model/roster.py +++ b/src/mailman/model/roster.py @@ -22,9 +22,6 @@ the ones that fit a particular role. These are used as the member, owner, moderator, and administrator roster filters. """ -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'AdministratorRoster', 'DigestMemberRoster', @@ -37,14 +34,13 @@ __all__ = [ ] -from sqlalchemy import and_, or_ -from zope.interface import implementer - from mailman.database.transaction import dbconnection from mailman.interfaces.member import DeliveryMode, MemberRole from mailman.interfaces.roster import IRoster from mailman.model.address import Address from mailman.model.member import Member +from sqlalchemy import and_, or_ +from zope.interface import implementer diff --git a/src/mailman/model/tests/test_address.py b/src/mailman/model/tests/test_address.py index 130ec3bae..29b32f542 100644 --- a/src/mailman/model/tests/test_address.py +++ b/src/mailman/model/tests/test_address.py @@ -17,9 +17,6 @@ """Test addresses.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestAddress', ] @@ -28,8 +25,11 @@ __all__ = [ import unittest from mailman.email.validate import InvalidEmailAddressError +from mailman.interfaces.address import ExistingAddressError +from mailman.interfaces.usermanager import IUserManager from mailman.model.address import Address from mailman.testing.layers import ConfigLayer +from zope.component import getUtility @@ -38,6 +38,25 @@ class TestAddress(unittest.TestCase): layer = ConfigLayer + def setUp(self): + self._usermgr = getUtility(IUserManager) + self._address = self._usermgr.create_address('FPERSON@example.com') + def test_invalid_email_string_raises_exception(self): with self.assertRaises(InvalidEmailAddressError): Address('not_a_valid_email_string', '') + + def test_local_part_differs_only_by_case(self): + with self.assertRaises(ExistingAddressError) as cm: + self._usermgr.create_address('fperson@example.com') + self.assertEqual(cm.exception.address, 'FPERSON@example.com') + + def test_domain_part_differs_only_by_case(self): + with self.assertRaises(ExistingAddressError) as cm: + self._usermgr.create_address('fperson@EXAMPLE.COM') + self.assertEqual(cm.exception.address, 'FPERSON@example.com') + + def test_mixed_case_exact_match(self): + with self.assertRaises(ExistingAddressError) as cm: + self._usermgr.create_address('FPERSON@example.com') + self.assertEqual(cm.exception.address, 'FPERSON@example.com') diff --git a/src/mailman/model/tests/test_bounce.py b/src/mailman/model/tests/test_bounce.py index a22da4416..2929747bc 100644 --- a/src/mailman/model/tests/test_bounce.py +++ b/src/mailman/model/tests/test_bounce.py @@ -17,24 +17,21 @@ """Test bounce model objects.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ + 'TestBounceEvents', ] import unittest from datetime import datetime -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.database.transaction import transaction from mailman.interfaces.bounce import BounceContext, IBounceProcessor from mailman.testing.helpers import ( specialized_message_from_string as message_from_string) from mailman.testing.layers import ConfigLayer +from zope.component import getUtility diff --git a/src/mailman/model/tests/test_domain.py b/src/mailman/model/tests/test_domain.py index f9d1ff202..a483d9567 100644 --- a/src/mailman/model/tests/test_domain.py +++ b/src/mailman/model/tests/test_domain.py @@ -17,9 +17,6 @@ """Test domains.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestDomainLifecycleEvents', 'TestDomainManager', @@ -28,8 +25,6 @@ __all__ = [ import unittest -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.interfaces.domain import ( DomainCreatedEvent, DomainCreatingEvent, DomainDeletedEvent, @@ -37,6 +32,7 @@ from mailman.interfaces.domain import ( from mailman.interfaces.listmanager import IListManager from mailman.testing.helpers import event_subscribers from mailman.testing.layers import ConfigLayer +from zope.component import getUtility @@ -45,6 +41,7 @@ class TestDomainManager(unittest.TestCase): def setUp(self): self._events = [] + self._manager = getUtility(IDomainManager) def _record_event(self, event): self._events.append(event) @@ -53,7 +50,7 @@ class TestDomainManager(unittest.TestCase): # Test that creating a domain in the domain manager propagates the # expected events. with event_subscribers(self._record_event): - domain = getUtility(IDomainManager).add('example.org') + domain = self._manager.add('example.org') self.assertEqual(len(self._events), 2) self.assertTrue(isinstance(self._events[0], DomainCreatingEvent)) self.assertEqual(self._events[0].mail_host, 'example.org') @@ -63,15 +60,24 @@ class TestDomainManager(unittest.TestCase): def test_delete_domain_event(self): # Test that deleting a domain in the domain manager propagates the # expected event. - domain = getUtility(IDomainManager).add('example.org') + domain = self._manager.add('example.org') with event_subscribers(self._record_event): - getUtility(IDomainManager).remove('example.org') + self._manager.remove('example.org') self.assertEqual(len(self._events), 2) self.assertTrue(isinstance(self._events[0], DomainDeletingEvent)) self.assertEqual(self._events[0].domain, domain) self.assertTrue(isinstance(self._events[1], DomainDeletedEvent)) self.assertEqual(self._events[1].mail_host, 'example.org') + def test_lookup_missing_domain(self): + # Like dictionaries, getitem syntax raises KeyError on missing domain. + with self.assertRaises(KeyError): + self._manager['doesnotexist.com'] + + def test_delete_missing_domain(self): + # Trying to delete a missing domain gives you a KeyError. + self.assertRaises(KeyError, self._manager.remove, 'doesnotexist.com') + class TestDomainLifecycleEvents(unittest.TestCase): diff --git a/src/mailman/model/tests/test_listmanager.py b/src/mailman/model/tests/test_listmanager.py index b290138f3..a28698eb1 100644 --- a/src/mailman/model/tests/test_listmanager.py +++ b/src/mailman/model/tests/test_listmanager.py @@ -17,9 +17,6 @@ """Test the ListManager.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestListCreation', 'TestListLifecycleEvents', @@ -29,14 +26,13 @@ __all__ = [ import unittest -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.app.moderator import hold_message from mailman.config import config +from mailman.interfaces.address import InvalidEmailAddressError from mailman.interfaces.listmanager import ( - IListManager, ListCreatedEvent, ListCreatingEvent, ListDeletedEvent, - ListDeletingEvent) + IListManager, ListAlreadyExistsError, ListCreatedEvent, ListCreatingEvent, + ListDeletedEvent, ListDeletingEvent) from mailman.interfaces.messages import IMessageStore from mailman.interfaces.requests import IListRequests from mailman.interfaces.subscriptions import ISubscriptionService @@ -45,6 +41,7 @@ from mailman.model.mime import ContentFilter from mailman.testing.helpers import ( event_subscribers, specialized_message_from_string) from mailman.testing.layers import ConfigLayer +from zope.component import getUtility @@ -157,11 +154,23 @@ Message-ID: <argon> class TestListCreation(unittest.TestCase): layer = ConfigLayer + def setUp(self): + self._manager = getUtility(IListManager) + def test_create_list_case_folding(self): # LP: #1117176 describes a problem where list names created in upper # case are not actually usable by the LMTP server. - manager = getUtility(IListManager) - manager.create('my-LIST@example.com') - self.assertIsNone(manager.get('my-LIST@example.com')) - mlist = manager.get('my-list@example.com') + self._manager.create('my-LIST@example.com') + self.assertIsNone(self._manager.get('my-LIST@example.com')) + mlist = self._manager.get('my-list@example.com') self.assertEqual(mlist.list_id, 'my-list.example.com') + + def test_cannot_create_a_list_twice(self): + self._manager.create('ant@example.com') + self.assertRaises(ListAlreadyExistsError, + self._manager.create, 'ant@example.com') + + def test_list_name_must_be_fully_qualified(self): + with self.assertRaises(InvalidEmailAddressError) as cm: + self._manager.create('foo') + self.assertEqual(cm.exception.email, 'foo') diff --git a/src/mailman/model/tests/test_mailinglist.py b/src/mailman/model/tests/test_mailinglist.py index 9d6177b54..6e7c11fe6 100644 --- a/src/mailman/model/tests/test_mailinglist.py +++ b/src/mailman/model/tests/test_mailinglist.py @@ -17,12 +17,10 @@ """Test MailingLists and related model objects..""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ - 'TestListArchiver', 'TestDisabledListArchiver', + 'TestListArchiver', + 'TestMailingList', ] @@ -31,8 +29,47 @@ import unittest from mailman.app.lifecycle import create_list from mailman.config import config from mailman.interfaces.mailinglist import IListArchiverSet +from mailman.interfaces.member import ( + AlreadySubscribedError, MemberRole, MissingPreferredAddressError) +from mailman.interfaces.usermanager import IUserManager from mailman.testing.helpers import configuration from mailman.testing.layers import ConfigLayer +from mailman.utilities.datetime import now +from zope.component import getUtility + + + +class TestMailingList(unittest.TestCase): + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('ant@example.com') + + def test_no_duplicate_subscriptions(self): + # A user is not allowed to subscribe more than once to the mailing + # list with the same role. + anne = getUtility(IUserManager).create_user('anne@example.com') + # Give the user a preferred address. + preferred = list(anne.addresses)[0] + preferred.verified_on = now() + anne.preferred_address = preferred + # Subscribe Anne to the mailing list as a regular member. + member = self._mlist.subscribe(anne) + self.assertEqual(member.address, preferred) + self.assertEqual(member.role, MemberRole.member) + # A second subscription with the same role will fail. + with self.assertRaises(AlreadySubscribedError) as cm: + self._mlist.subscribe(anne) + self.assertEqual(cm.exception.fqdn_listname, 'ant@example.com') + self.assertEqual(cm.exception.email, 'anne@example.com') + self.assertEqual(cm.exception.role, MemberRole.member) + + def test_subscribing_user_must_have_preferred_address(self): + # A user object cannot be subscribed to a mailing list without a + # preferred address. + anne = getUtility(IUserManager).create_user('anne@example.com') + self.assertRaises(MissingPreferredAddressError, + self._mlist.subscribe, anne) diff --git a/src/mailman/model/tests/test_member.py b/src/mailman/model/tests/test_member.py index 5bd3d1594..38f36acde 100644 --- a/src/mailman/model/tests/test_member.py +++ b/src/mailman/model/tests/test_member.py @@ -17,9 +17,6 @@ """Test members.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestMember', ] diff --git a/src/mailman/model/tests/test_messagestore.py b/src/mailman/model/tests/test_messagestore.py new file mode 100644 index 000000000..39d1d97ed --- /dev/null +++ b/src/mailman/model/tests/test_messagestore.py @@ -0,0 +1,71 @@ +# Copyright (C) 2014 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test the message store.""" + +__all__ = [ + 'TestMessageStore', + ] + + +import unittest + +from mailman.interfaces.messages import IMessageStore +from mailman.testing.helpers import ( + specialized_message_from_string as mfs) +from mailman.testing.layers import ConfigLayer +from mailman.utilities.email import add_message_hash +from zope.component import getUtility + + + +class TestMessageStore(unittest.TestCase): + layer = ConfigLayer + + def setUp(self): + self._store = getUtility(IMessageStore) + + def test_message_id_required(self): + # The Message-ID header is required in order to add it to the store. + message = mfs("""\ +Subject: An important message + +This message is very important. +""") + self.assertRaises(ValueError, self._store.add, message) + + def test_get_message_by_hash(self): + # Messages have an X-Message-ID-Hash header, the value of which can be + # used to look the message up in the message store. + message = mfs("""\ +Subject: An important message +Message-ID: <ant> + +This message is very important. +""") + add_message_hash(message) + self._store.add(message) + self.assertEqual(message['x-message-id-hash'], + 'V3YEHAFKE2WVJNK63Z7RFP4JMHISI2RG') + found = self._store.get_message_by_hash( + 'V3YEHAFKE2WVJNK63Z7RFP4JMHISI2RG') + self.assertEqual(found['message-id'], '<ant>') + self.assertEqual(found['x-message-id-hash'], + 'V3YEHAFKE2WVJNK63Z7RFP4JMHISI2RG') + + def test_cannot_delete_missing_message(self): + self.assertRaises(LookupError, self._store.delete_message, 'missing') diff --git a/src/mailman/model/tests/test_registrar.py b/src/mailman/model/tests/test_registrar.py new file mode 100644 index 000000000..8d7c00e78 --- /dev/null +++ b/src/mailman/model/tests/test_registrar.py @@ -0,0 +1,64 @@ +# Copyright (C) 2014 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test `IRegistrar`.""" + +__all__ = [ + 'TestRegistrar', + ] + + +import unittest + +from functools import partial +from mailman.app.lifecycle import create_list +from mailman.interfaces.address import InvalidEmailAddressError +from mailman.interfaces.registrar import IRegistrar +from mailman.testing.layers import ConfigLayer +from zope.component import getUtility + + + +class TestRegistrar(unittest.TestCase): + layer = ConfigLayer + + def setUp(self): + mlist = create_list('test@example.com') + self._register = partial(getUtility(IRegistrar).register, mlist) + + def test_invalid_empty_string(self): + self.assertRaises(InvalidEmailAddressError, self._register, '') + + def test_invalid_space_in_name(self): + self.assertRaises(InvalidEmailAddressError, self._register, + 'some name@example.com') + + def test_invalid_funky_characters(self): + self.assertRaises(InvalidEmailAddressError, self._register, + '<script>@example.com') + + def test_invalid_nonascii(self): + self.assertRaises(InvalidEmailAddressError, self._register, + '\xa0@example.com') + + def test_invalid_no_at_sign(self): + self.assertRaises(InvalidEmailAddressError, self._register, + 'noatsign') + + def test_invalid_no_domain(self): + self.assertRaises(InvalidEmailAddressError, self._register, + 'nodom@ain') diff --git a/src/mailman/model/tests/test_requests.py b/src/mailman/model/tests/test_requests.py index 419c6077f..c47c61013 100644 --- a/src/mailman/model/tests/test_requests.py +++ b/src/mailman/model/tests/test_requests.py @@ -17,9 +17,6 @@ """Test the various pending requests interfaces.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestRequests', ] diff --git a/src/mailman/model/tests/test_roster.py b/src/mailman/model/tests/test_roster.py index 5bd06f485..8cf189e08 100644 --- a/src/mailman/model/tests/test_roster.py +++ b/src/mailman/model/tests/test_roster.py @@ -17,9 +17,6 @@ """Test rosters.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestMailingListRoster', 'TestMembershipsRoster', @@ -28,13 +25,12 @@ __all__ = [ import unittest -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.interfaces.member import DeliveryMode, MemberRole from mailman.interfaces.usermanager import IUserManager from mailman.testing.layers import ConfigLayer from mailman.utilities.datetime import now +from zope.component import getUtility diff --git a/src/mailman/model/tests/test_uid.py b/src/mailman/model/tests/test_uid.py index 4c541205a..dd61ccc51 100644 --- a/src/mailman/model/tests/test_uid.py +++ b/src/mailman/model/tests/test_uid.py @@ -17,10 +17,8 @@ """Test the UID model class.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ + 'TestUID', ] diff --git a/src/mailman/model/tests/test_user.py b/src/mailman/model/tests/test_user.py index 17d4d24ff..ba5ba116f 100644 --- a/src/mailman/model/tests/test_user.py +++ b/src/mailman/model/tests/test_user.py @@ -17,9 +17,6 @@ """Test users.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestUser', ] @@ -27,12 +24,14 @@ __all__ = [ import unittest -from zope.component import getUtility - from mailman.app.lifecycle import create_list +from mailman.interfaces.address import ( + AddressAlreadyLinkedError, AddressNotLinkedError) +from mailman.interfaces.user import UnverifiedAddressError from mailman.interfaces.usermanager import IUserManager from mailman.testing.layers import ConfigLayer from mailman.utilities.datetime import now +from zope.component import getUtility @@ -74,3 +73,38 @@ class TestUser(unittest.TestCase): self.assertEqual(len(emails), 2) self.assertEqual(emails, set(['anne@example.com', 'aperson@example.com'])) + + def test_uid_is_immutable(self): + with self.assertRaises(AttributeError): + self._anne.user_id = 'foo' + + def test_addresses_may_only_be_linked_to_one_user(self): + user = getUtility(IUserManager).create_user() + # Anne's preferred address is already linked to her. + with self.assertRaises(AddressAlreadyLinkedError) as cm: + user.link(self._anne.preferred_address) + self.assertEqual(cm.exception.address, self._anne.preferred_address) + + def test_unlink_from_address_not_linked_to(self): + # You cannot unlink an address from a user if that address is not + # already linked to the user. + user = getUtility(IUserManager).create_user() + with self.assertRaises(AddressNotLinkedError) as cm: + user.unlink(self._anne.preferred_address) + self.assertEqual(cm.exception.address, self._anne.preferred_address) + + def test_unlink_address_which_is_not_linked(self): + # You cannot unlink an address which is not linked to any user. + address = getUtility(IUserManager).create_address('bart@example.com') + user = getUtility(IUserManager).create_user() + with self.assertRaises(AddressNotLinkedError) as cm: + user.unlink(address) + self.assertEqual(cm.exception.address, address) + + def test_set_unverified_preferred_address(self): + # A user's preferred address cannot be set to an unverified address. + new_preferred = getUtility(IUserManager).create_address( + 'anne.person@example.com') + with self.assertRaises(UnverifiedAddressError) as cm: + self._anne.preferred_address = new_preferred + self.assertEqual(cm.exception.address, new_preferred) diff --git a/src/mailman/model/uid.py b/src/mailman/model/uid.py index 72ddd7b5a..94a4f1a17 100644 --- a/src/mailman/model/uid.py +++ b/src/mailman/model/uid.py @@ -17,20 +17,16 @@ """Unique IDs.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'UID', ] -from sqlalchemy import Column, Integer - from mailman.database.model import Model from mailman.database.transaction import dbconnection from mailman.database.types import UUID +from sqlalchemy import Column, Integer diff --git a/src/mailman/model/user.py b/src/mailman/model/user.py index ab581fdc8..a85ef0d00 100644 --- a/src/mailman/model/user.py +++ b/src/mailman/model/user.py @@ -17,18 +17,10 @@ """Model for users.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'User', ] -from sqlalchemy import ( - Column, DateTime, ForeignKey, Integer, LargeBinary, Unicode) -from sqlalchemy.orm import relationship, backref -from zope.event import notify -from zope.interface import implementer from mailman.database.model import Model from mailman.database.transaction import dbconnection @@ -42,6 +34,10 @@ from mailman.model.preferences import Preferences from mailman.model.roster import Memberships from mailman.utilities.datetime import factory as date_factory from mailman.utilities.uid import UniqueIDFactory +from sqlalchemy import Column, DateTime, ForeignKey, Integer, Unicode +from sqlalchemy.orm import relationship, backref +from zope.event import notify +from zope.interface import implementer uid_factory = UniqueIDFactory(context='users') @@ -56,7 +52,7 @@ class User(Model): id = Column(Integer, primary_key=True) display_name = Column(Unicode) - _password = Column('password', LargeBinary) + _password = Column('password', Unicode) _user_id = Column(UUID, index=True) _created_on = Column(DateTime) @@ -122,7 +118,7 @@ class User(Model): def unlink(self, address): """See `IUser`.""" - if address.user is None: + if address.user is None or address.user is not self: raise AddressNotLinkedError(address) address.user = None diff --git a/src/mailman/model/usermanager.py b/src/mailman/model/usermanager.py index 726aa6120..374352033 100644 --- a/src/mailman/model/usermanager.py +++ b/src/mailman/model/usermanager.py @@ -17,16 +17,11 @@ """A user manager.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'UserManager', ] -from zope.interface import implementer - from mailman.database.transaction import dbconnection from mailman.interfaces.address import ExistingAddressError from mailman.interfaces.usermanager import IUserManager @@ -34,6 +29,7 @@ from mailman.model.address import Address from mailman.model.member import Member from mailman.model.preferences import Preferences from mailman.model.user import User +from zope.interface import implementer diff --git a/src/mailman/mta/aliases.py b/src/mailman/mta/aliases.py index 1b5f37d44..c309fb27b 100644 --- a/src/mailman/mta/aliases.py +++ b/src/mailman/mta/aliases.py @@ -17,17 +17,13 @@ """Utility for generating all the aliases of a mailing list.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'MailTransportAgentAliases', ] -from zope.interface import implementer - from mailman.interfaces.mta import IMailTransportAgentAliases +from zope.interface import implementer SUBDESTINATIONS = ( diff --git a/src/mailman/mta/base.py b/src/mailman/mta/base.py index 7b9180ea3..8d7ca75af 100644 --- a/src/mailman/mta/base.py +++ b/src/mailman/mta/base.py @@ -17,9 +17,6 @@ """Base delivery class.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'BaseDelivery', 'IndividualDelivery', @@ -31,11 +28,10 @@ import socket import logging import smtplib -from zope.interface import implementer - from mailman.config import config from mailman.interfaces.mta import IMailTransportAgentDelivery from mailman.mta.connection import Connection +from zope.interface import implementer log = logging.getLogger('mailman.smtp') diff --git a/src/mailman/mta/bulk.py b/src/mailman/mta/bulk.py index 4255e0c33..0dcd2cdf6 100644 --- a/src/mailman/mta/bulk.py +++ b/src/mailman/mta/bulk.py @@ -17,9 +17,6 @@ """Bulk message delivery.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'BulkDelivery', ] @@ -108,4 +105,3 @@ class BulkDelivery(BaseDelivery): mlist, msg, msgdata, recipients) refused.update(chunk_refused) return refused - diff --git a/src/mailman/mta/connection.py b/src/mailman/mta/connection.py index 8cf419545..9c49e5fb0 100644 --- a/src/mailman/mta/connection.py +++ b/src/mailman/mta/connection.py @@ -17,9 +17,6 @@ """MTA connections.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Connection', ] diff --git a/src/mailman/mta/decorating.py b/src/mailman/mta/decorating.py index ac99b3624..b4944d960 100644 --- a/src/mailman/mta/decorating.py +++ b/src/mailman/mta/decorating.py @@ -17,9 +17,6 @@ """Individualized delivery with header/footer decorations.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'DecoratingDelivery', 'DecoratingMixin', diff --git a/src/mailman/mta/deliver.py b/src/mailman/mta/deliver.py index be04a48bd..f01390397 100644 --- a/src/mailman/mta/deliver.py +++ b/src/mailman/mta/deliver.py @@ -17,9 +17,6 @@ """Generic delivery.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'deliver', ] diff --git a/src/mailman/mta/docs/authentication.rst b/src/mailman/mta/docs/authentication.rst index 94cd2c99e..f98c00e1f 100644 --- a/src/mailman/mta/docs/authentication.rst +++ b/src/mailman/mta/docs/authentication.rst @@ -60,7 +60,7 @@ But if the user name and password does not match, the connection will fail. >>> response = bulk.deliver( ... mlist, msg, dict(recipients=['bperson@example.com'])) >>> dump_msgdata(response) - bperson@example.com: (571, 'Bad authentication') + bperson@example.com: (571, b'Bad authentication') >>> config.pop('auth') diff --git a/src/mailman/mta/docs/bulk.rst b/src/mailman/mta/docs/bulk.rst index f2a76229b..cd7873de1 100644 --- a/src/mailman/mta/docs/bulk.rst +++ b/src/mailman/mta/docs/bulk.rst @@ -332,7 +332,8 @@ recipients. >>> failures = bulk.deliver(mlist, msg, msgdata) >>> for address in sorted(failures): - ... print(address, failures[address][0], failures[address][1]) + ... print(address, failures[address][0], + ... failures[address][1].decode('ascii')) aperson@example.org 500 Error: SMTPRecipientsRefused bperson@example.org 500 Error: SMTPRecipientsRefused cperson@example.org 500 Error: SMTPRecipientsRefused @@ -350,7 +351,8 @@ Or there could be some other problem causing an SMTP response failure. >>> failures = bulk.deliver(mlist, msg, msgdata) >>> for address in sorted(failures): - ... print(address, failures[address][0], failures[address][1]) + ... print(address, failures[address][0], + ... failures[address][1].decode('ascii')) aperson@example.org 450 Error: SMTPResponseException bperson@example.org 450 Error: SMTPResponseException cperson@example.org 450 Error: SMTPResponseException @@ -361,7 +363,8 @@ Or there could be some other problem causing an SMTP response failure. >>> failures = bulk.deliver(mlist, msg, msgdata) >>> for address in sorted(failures): - ... print(address, failures[address][0], failures[address][1]) + ... print(address, failures[address][0], + ... failures[address][1].decode('ascii')) aperson@example.org 500 Error: SMTPResponseException bperson@example.org 500 Error: SMTPResponseException cperson@example.org 500 Error: SMTPResponseException diff --git a/src/mailman/mta/docs/connection.rst b/src/mailman/mta/docs/connection.rst index a57a76bb9..f4e0d8107 100644 --- a/src/mailman/mta/docs/connection.rst +++ b/src/mailman/mta/docs/connection.rst @@ -75,30 +75,6 @@ will authenticate with the mail server after each new connection. >>> reset() >>> config.pop('auth') -However, a bad user name or password generates an error. - - >>> config.push('auth', """ - ... [mta] - ... smtp_user: baduser - ... smtp_pass: badpass - ... """) - - >>> connection = Connection( - ... config.mta.smtp_host, int(config.mta.smtp_port), 0, - ... config.mta.smtp_user, config.mta.smtp_pass) - >>> connection.sendmail('anne@example.com', ['bart@example.com'], """\ - ... From: anne@example.com - ... To: bart@example.com - ... Subject: aardvarks - ... - ... """) - Traceback (most recent call last): - ... - SMTPAuthenticationError: (571, 'Bad authentication') - - >>> reset() - >>> config.pop('auth') - Sessions per connection ======================= diff --git a/src/mailman/mta/exim4.py b/src/mailman/mta/exim4.py index 1180b59eb..f25b12233 100644 --- a/src/mailman/mta/exim4.py +++ b/src/mailman/mta/exim4.py @@ -17,9 +17,6 @@ """Creation/deletion hooks for the Exim4 MTA.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'LMTP', ] diff --git a/src/mailman/mta/null.py b/src/mailman/mta/null.py index 7a3624b31..3b9f6322e 100644 --- a/src/mailman/mta/null.py +++ b/src/mailman/mta/null.py @@ -20,17 +20,13 @@ Exim one example of an MTA that Just Works. """ -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'NullMTA', ] -from zope.interface import implementer - from mailman.interfaces.mta import IMailTransportAgentLifecycle +from zope.interface import implementer diff --git a/src/mailman/mta/personalized.py b/src/mailman/mta/personalized.py index 967bca68a..4ea9075a3 100644 --- a/src/mailman/mta/personalized.py +++ b/src/mailman/mta/personalized.py @@ -17,9 +17,6 @@ """Personalized delivery.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'PersonalizedDelivery', 'PersonalizedMixin', @@ -28,11 +25,10 @@ __all__ = [ from email.header import Header from email.utils import formataddr -from zope.component import getUtility - from mailman.interfaces.mailinglist import Personalization from mailman.interfaces.usermanager import IUserManager from mailman.mta.verp import VERPDelivery +from zope.component import getUtility diff --git a/src/mailman/mta/postfix.py b/src/mailman/mta/postfix.py index bb709c6b4..f76a401fa 100644 --- a/src/mailman/mta/postfix.py +++ b/src/mailman/mta/postfix.py @@ -17,9 +17,6 @@ """Creation/deletion hooks for the Postfix MTA.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'LMTP', ] @@ -29,16 +26,15 @@ import os import logging from flufl.lock import Lock -from operator import attrgetter -from zope.component import getUtility -from zope.interface import implementer - from mailman.config import config from mailman.config.config import external_configuration from mailman.interfaces.listmanager import IListManager from mailman.interfaces.mta import ( IMailTransportAgentAliases, IMailTransportAgentLifecycle) from mailman.utilities.datetime import now +from operator import attrgetter +from zope.component import getUtility +from zope.interface import implementer log = logging.getLogger('mailman.error') diff --git a/src/mailman/mta/tests/test_aliases.py b/src/mailman/mta/tests/test_aliases.py index 30c57e292..8eeeef2c8 100644 --- a/src/mailman/mta/tests/test_aliases.py +++ b/src/mailman/mta/tests/test_aliases.py @@ -17,9 +17,6 @@ """Test the MTA file generating utility.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestAliases', 'TestPostfix', @@ -31,13 +28,12 @@ import shutil import tempfile import unittest -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.interfaces.domain import IDomainManager from mailman.interfaces.mta import IMailTransportAgentAliases from mailman.mta.postfix import LMTP from mailman.testing.layers import ConfigLayer +from zope.component import getUtility NL = '\n' diff --git a/src/mailman/mta/tests/test_connection.py b/src/mailman/mta/tests/test_connection.py new file mode 100644 index 000000000..74d0e537c --- /dev/null +++ b/src/mailman/mta/tests/test_connection.py @@ -0,0 +1,51 @@ +# Copyright (C) 2014 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test MTA connections.""" + +__all__ = [ + 'TestConnection', + ] + + +import unittest + +from mailman.config import config +from mailman.mta.connection import Connection +from mailman.testing.layers import SMTPLayer +from smtplib import SMTPAuthenticationError + + + +class TestConnection(unittest.TestCase): + layer = SMTPLayer + + def test_authentication_error(self): + # Logging in to the MTA with a bad user name and password produces a + # 571 Bad Authentication error. + with self.assertRaises(SMTPAuthenticationError) as cm: + connection = Connection( + config.mta.smtp_host, int(config.mta.smtp_port), 0, + 'baduser', 'badpass') + connection.sendmail('anne@example.com', ['bart@example.com'], """\ +From: anne@example.com +To: bart@example.com +Subject: aardvarks + +""") + self.assertEqual(cm.exception.smtp_code, 571) + self.assertEqual(cm.exception.smtp_error, b'Bad authentication') diff --git a/src/mailman/mta/tests/test_delivery.py b/src/mailman/mta/tests/test_delivery.py index 0a910c13d..a2960f7cc 100644 --- a/src/mailman/mta/tests/test_delivery.py +++ b/src/mailman/mta/tests/test_delivery.py @@ -17,9 +17,6 @@ """Test various aspects of email delivery.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestIndividualDelivery', ] diff --git a/src/mailman/mta/verp.py b/src/mailman/mta/verp.py index c3d1d0999..2d436b8cb 100644 --- a/src/mailman/mta/verp.py +++ b/src/mailman/mta/verp.py @@ -17,9 +17,6 @@ """VERP delivery.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'VERPDelivery', 'VERPMixin', diff --git a/src/mailman/options.py b/src/mailman/options.py index a4f553a09..93ada95ab 100644 --- a/src/mailman/options.py +++ b/src/mailman/options.py @@ -17,9 +17,6 @@ """Common argument parsing.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Options', 'SingleMailingListOptions', @@ -31,18 +28,17 @@ import os import sys from copy import copy -from optparse import Option, OptionParser, OptionValueError - from mailman.config import config from mailman.core.i18n import _ from mailman.core.initialize import initialize from mailman.version import MAILMAN_VERSION +from optparse import Option, OptionParser, OptionValueError def check_unicode(option, opt, value): """Check that the value is a unicode string.""" - if isinstance(value, unicode): + if not isinstance(value, bytes): return value try: return value.decode(sys.getdefaultencoding()) diff --git a/src/mailman/rest/addresses.py b/src/mailman/rest/addresses.py index f8516bc37..6cca24393 100644 --- a/src/mailman/rest/addresses.py +++ b/src/mailman/rest/addresses.py @@ -17,9 +17,6 @@ """REST for addresses.""" -from __future__ import absolute_import, print_function,unicode_literals - -__metaclass__ = type __all__ = [ 'AllAddresses', 'AnAddress', @@ -27,8 +24,7 @@ __all__ = [ ] -from operator import attrgetter -from zope.component import getUtility +import six from mailman.interfaces.address import ( ExistingAddressError, InvalidEmailAddressError) @@ -40,6 +36,8 @@ from mailman.rest.members import MemberCollection from mailman.rest.preferences import Preferences from mailman.rest.validator import Validator from mailman.utilities.datetime import now +from operator import attrgetter +from zope.component import getUtility @@ -168,6 +166,7 @@ class AnAddress(_AddressBase): from mailman.rest.users import AddressUser return AddressUser(self._address) + class UserAddresses(_AddressBase): """The addresses of a user.""" @@ -197,8 +196,8 @@ class UserAddresses(_AddressBase): not_found(response) return user_manager = getUtility(IUserManager) - validator = Validator(email=unicode, - display_name=unicode, + validator = Validator(email=six.text_type, + display_name=six.text_type, _optional=('display_name',)) try: address = user_manager.create_address(**validator(request)) diff --git a/src/mailman/rest/docs/__init__.py b/src/mailman/rest/docs/__init__.py index 2daf8a681..fcd8b41bb 100644 --- a/src/mailman/rest/docs/__init__.py +++ b/src/mailman/rest/docs/__init__.py @@ -17,9 +17,6 @@ """Doctest layer setup.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'layer', ] diff --git a/src/mailman/rest/docs/addresses.rst b/src/mailman/rest/docs/addresses.rst index bab2a7210..bcffd6830 100644 --- a/src/mailman/rest/docs/addresses.rst +++ b/src/mailman/rest/docs/addresses.rst @@ -64,13 +64,6 @@ But his address record can be accessed with the case-preserved version too. registered_on: 2005-08-01T07:49:23 self_link: http://localhost:9001/3.0/addresses/bart.person@example.com -A non-existent email address can't be retrieved. - - >>> dump_json('http://localhost:9001/3.0/addresses/nobody@example.com') - Traceback (most recent call last): - ... - HTTPError: HTTP Error 404: 404 Not Found - When an address has a real name associated with it, this is also available in the REST API. @@ -168,7 +161,7 @@ The user is now created and the address is linked to it: >>> cris.user == cris_user True >>> [a.email for a in cris_user.addresses] - [u'cris@example.com'] + ['cris@example.com'] A link to the user resource is now available as a sub-resource. @@ -188,7 +181,7 @@ parameter to the POST request and set it to a false-equivalent value like 0: ... {'display_name': 'Anne User', 'auto_create': 0}) Traceback (most recent call last): ... - HTTPError: HTTP Error 403: 403 Forbidden + urllib.error.HTTPError: HTTP Error 403: ... A request to the `/user` sub-resource will return the linked user's representation: @@ -219,7 +212,7 @@ The address and the user can be unlinked by sending a DELETE request on the >>> dump_json('http://localhost:9001/3.0/addresses/cris@example.com/user') Traceback (most recent call last): ... - HTTPError: HTTP Error 404: 404 Not Found + urllib.error.HTTPError: HTTP Error 404: ... You can link an existing user to an address by passing the user's ID in the POST request. @@ -261,7 +254,7 @@ User addresses ============== Users control addresses. The canonical URLs for these user-controlled -addresses live in the `/addresses` namespace. +addresses live in the ``/addresses`` namespace. :: >>> dave = user_manager.create_user('dave@example.com', 'Dave Person') diff --git a/src/mailman/rest/docs/basic.rst b/src/mailman/rest/docs/basic.rst index 15ce37682..7e013f598 100644 --- a/src/mailman/rest/docs/basic.rst +++ b/src/mailman/rest/docs/basic.rst @@ -24,13 +24,10 @@ Credentials When the `Authorization` header contains the proper credentials, the request succeeds. - >>> from base64 import b64encode >>> from httplib2 import Http - >>> auth = b64encode('{0}:{1}'.format(config.webservice.admin_user, - ... config.webservice.admin_pass)) >>> headers = { ... 'Content-Type': 'application/x-www-form-urlencode', - ... 'Authorization': 'Basic ' + auth, + ... 'Authorization': 'Basic cmVzdGFkbWluOnJlc3RwYXNz', ... } >>> url = 'http://localhost:9001/3.0/system/versions' >>> response, content = Http().request(url, 'GET', None, headers) diff --git a/src/mailman/rest/docs/domains.rst b/src/mailman/rest/docs/domains.rst index b28326f73..a78dacd85 100644 --- a/src/mailman/rest/docs/domains.rst +++ b/src/mailman/rest/docs/domains.rst @@ -228,13 +228,5 @@ Domains can also be deleted via the API. server: ... status: 204 -It is an error to delete a domain twice. - - >>> dump_json('http://localhost:9001/3.0/domains/lists.example.com', - ... method='DELETE') - Traceback (most recent call last): - ... - HTTPError: HTTP Error 404: 404 Not Found - .. _Domains: ../../model/docs/domains.html diff --git a/src/mailman/rest/docs/helpers.rst b/src/mailman/rest/docs/helpers.rst index 5bcf5cad4..5614e6544 100644 --- a/src/mailman/rest/docs/helpers.rst +++ b/src/mailman/rest/docs/helpers.rst @@ -45,7 +45,7 @@ gets modified to contain the etag under the ``http_etag`` key. >>> resource = dict(geddy='bass', alex='guitar', neil='drums') >>> json_data = etag(resource) >>> print(resource['http_etag']) - "96e036d66248cab746b7d97047e08896fcfb2493" + "6929ecfbda2282980a4818fb75f82e812077f77a" For convenience, the etag function also returns the JSON representation of the dictionary after tagging, since that's almost always what you want. @@ -58,7 +58,7 @@ dictionary after tagging, since that's almost always what you want. >>> dump_msgdata(data) alex : guitar geddy : bass - http_etag: "96e036d66248cab746b7d97047e08896fcfb2493" + http_etag: "6929ecfbda2282980a4818fb75f82e812077f77a" neil : drums @@ -69,8 +69,9 @@ Another helper unpacks ``POST`` and ``PUT`` request variables, validating and converting their values. :: + >>> import six >>> from mailman.rest.validator import Validator - >>> validator = Validator(one=int, two=unicode, three=bool) + >>> validator = Validator(one=int, two=six.text_type, three=bool) >>> class FakeRequest: ... params = None @@ -81,7 +82,7 @@ On valid input, the validator can be used as a ``**keyword`` argument. >>> def print_request(one, two, three): ... print(repr(one), repr(two), repr(three)) >>> print_request(**validator(FakeRequest)) - 1 u'two' True + 1 'two' True On invalid input, an exception is raised. @@ -119,7 +120,7 @@ Extra keys are also not allowed. However, if optional keys are missing, it's okay. :: - >>> validator = Validator(one=int, two=unicode, three=bool, + >>> validator = Validator(one=int, two=six.text_type, three=bool, ... four=int, five=int, ... _optional=('four', 'five')) @@ -128,15 +129,15 @@ However, if optional keys are missing, it's okay. >>> def print_request(one, two, three, four=None, five=None): ... print(repr(one), repr(two), repr(three), repr(four), repr(five)) >>> print_request(**validator(FakeRequest)) - 1 u'two' True 4 5 + 1 'two' True 4 5 >>> del FakeRequest.params['four'] >>> print_request(**validator(FakeRequest)) - 1 u'two' True None 5 + 1 'two' True None 5 >>> del FakeRequest.params['five'] >>> print_request(**validator(FakeRequest)) - 1 u'two' True None None + 1 'two' True None None But if the optional values are present, they must of course also be valid. diff --git a/src/mailman/rest/docs/membership.rst b/src/mailman/rest/docs/membership.rst index 30e69d9f5..b0b884d51 100644 --- a/src/mailman/rest/docs/membership.rst +++ b/src/mailman/rest/docs/membership.rst @@ -572,7 +572,7 @@ Elly is now a known user, and a member of the mailing list. <User "Elly Person" (...) at ...> >>> set(member.list_id for member in elly.memberships.members) - set([u'ant.example.com']) + {'ant.example.com'} >>> dump_json('http://localhost:9001/3.0/members') entry 0: @@ -674,7 +674,7 @@ so she leaves from the mailing list. Elly is no longer a member of the mailing list. >>> set(member.mailing_list for member in elly.memberships.members) - set([]) + set() Digest delivery diff --git a/src/mailman/rest/docs/moderation.rst b/src/mailman/rest/docs/moderation.rst index 6e2dbb43c..6aec921f0 100644 --- a/src/mailman/rest/docs/moderation.rst +++ b/src/mailman/rest/docs/moderation.rst @@ -141,13 +141,6 @@ The held message can be discarded. server: ... status: 204 -After which, the message is gone from the moderation queue. - - >>> dump_json(url(request_id)) - Traceback (most recent call last): - ... - HTTPError: HTTP Error 404: 404 Not Found - Messages can also be accepted via the REST API. Let's hold a new message for moderation. :: diff --git a/src/mailman/rest/docs/preferences.rst b/src/mailman/rest/docs/preferences.rst index b9332c954..172a9bedd 100644 --- a/src/mailman/rest/docs/preferences.rst +++ b/src/mailman/rest/docs/preferences.rst @@ -162,7 +162,7 @@ deleted. >>> dump_json('http://localhost:9001/3.0/addresses/anne@example.com' ... '/preferences') acknowledge_posts: True - http_etag: "1ff07b0367bede79ade27d217e12df3915aaee2b" + http_etag: "..." preferred_language: ja self_link: http://localhost:9001/3.0/addresses/anne@example.com/preferences diff --git a/src/mailman/rest/docs/queues.rst b/src/mailman/rest/docs/queues.rst new file mode 100644 index 000000000..861b6806f --- /dev/null +++ b/src/mailman/rest/docs/queues.rst @@ -0,0 +1,174 @@ +====== +Queues +====== + +You can get information about what messages are currently in the Mailman +queues by querying the top-level ``queues`` resource. Of course, this +information may be out-of-date by the time you receive a response, since queue +management is asynchronous, but the information will be as current as +possible. + +You can get the list of all queue names. + + >>> dump_json('http://localhost:9001/3.0/queues') + entry 0: + count: 0 + directory: .../queue/archive + files: [] + http_etag: ... + name: archive + self_link: http://localhost:9001/3.0/queues/archive + entry 1: + count: 0 + directory: .../queue/bad + files: [] + http_etag: ... + name: bad + self_link: http://localhost:9001/3.0/queues/bad + entry 2: + count: 0 + directory: .../queue/bounces + files: [] + http_etag: ... + name: bounces + self_link: http://localhost:9001/3.0/queues/bounces + entry 3: + count: 0 + directory: .../queue/command + files: [] + http_etag: ... + name: command + self_link: http://localhost:9001/3.0/queues/command + entry 4: + count: 0 + directory: .../queue/digest + files: [] + http_etag: ... + name: digest + self_link: http://localhost:9001/3.0/queues/digest + entry 5: + count: 0 + directory: .../queue/in + files: [] + http_etag: ... + name: in + self_link: http://localhost:9001/3.0/queues/in + entry 6: + count: 0 + directory: .../queue/nntp + files: [] + http_etag: ... + name: nntp + self_link: http://localhost:9001/3.0/queues/nntp + entry 7: + count: 0 + directory: .../queue/out + files: [] + http_etag: ... + name: out + self_link: http://localhost:9001/3.0/queues/out + entry 8: + count: 0 + directory: .../queue/pipeline + files: [] + http_etag: ... + name: pipeline + self_link: http://localhost:9001/3.0/queues/pipeline + entry 9: + count: 0 + directory: .../queue/retry + files: [] + http_etag: ... + name: retry + self_link: http://localhost:9001/3.0/queues/retry + entry 10: + count: 0 + directory: .../queue/shunt + files: [] + http_etag: ... + name: shunt + self_link: http://localhost:9001/3.0/queues/shunt + entry 11: + count: 0 + directory: .../queue/virgin + files: [] + http_etag: ... + name: virgin + self_link: http://localhost:9001/3.0/queues/virgin + http_etag: ... + self_link: http://localhost:9001/3.0/queues + start: 0 + total_size: 12 + +Query an individual queue to get a count of, and the list of file base names +in the queue. There are currently no files in the ``bad`` queue. + + >>> dump_json('http://localhost:9001/3.0/queues/bad') + count: 0 + directory: .../queue/bad + files: [] + http_etag: ... + name: bad + self_link: http://localhost:9001/3.0/queues/bad + +We can inject a message into the ``bad`` queue. It must be destined for an +existing mailing list. + + >>> dump_json('http://localhost:9001/3.0/lists', { + ... 'fqdn_listname': 'ant@example.com', + ... }) + content-length: 0 + date: ... + location: http://localhost:9001/3.0/lists/ant.example.com + server: WSGIServer/0.2 CPython/3.4.2 + status: 201 + +While list creation takes an FQDN list name, injecting a message to the queue +requires a List ID. + + >>> dump_json('http://localhost:9001/3.0/queues/bad', { + ... 'list_id': 'ant.example.com', + ... 'text': """\ + ... From: anne@example.com + ... To: ant@example.com + ... Subject: Testing + ... + ... """}) + content-length: 0 + date: ... + location: http://localhost:9001/3.0/queues/bad/... + server: ... + status: 201 + +And now the ``bad`` queue has at least one message in it. + + >>> dump_json('http://localhost:9001/3.0/queues/bad') + count: 1 + directory: .../queue/bad + files: ['...'] + http_etag: ... + name: bad + self_link: http://localhost:9001/3.0/queues/bad + +We can delete the injected message. + + >>> json = call_http('http://localhost:9001/3.0/queues/bad') + >>> len(json['files']) + 1 + >>> dump_json('http://localhost:9001/3.0/queues/bad/{}'.format( + ... json['files'][0]), + ... method='DELETE') + content-length: 0 + date: ... + server: ... + status: 204 + +And now the queue has no files. + + >>> dump_json('http://localhost:9001/3.0/queues/bad') + count: 0 + directory: .../queue/bad + files: [] + http_etag: ... + name: bad + self_link: http://localhost:9001/3.0/queues/bad diff --git a/src/mailman/rest/docs/users.rst b/src/mailman/rest/docs/users.rst index b2adcaccb..824492333 100644 --- a/src/mailman/rest/docs/users.rst +++ b/src/mailman/rest/docs/users.rst @@ -277,27 +277,6 @@ Users can also be deleted via the API. server: ... status: 204 -Cris's resource cannot be retrieved either by email address... - - >>> dump_json('http://localhost:9001/3.0/users/cris@example.com') - Traceback (most recent call last): - ... - HTTPError: HTTP Error 404: 404 Not Found - -...or user id. - - >>> dump_json('http://localhost:9001/3.0/users/3') - Traceback (most recent call last): - ... - HTTPError: HTTP Error 404: 404 Not Found - -Cris's address records no longer exist either. - - >>> dump_json('http://localhost:9001/3.0/addresses/cris@example.com') - Traceback (most recent call last): - ... - HTTPError: HTTP Error 404: 404 Not Found - User addresses ============== @@ -420,12 +399,3 @@ This time, Elly successfully logs into Mailman. date: ... server: ... status: 204 - -But this time, she is unsuccessful. - - >>> dump_json('http://localhost:9001/3.0/users/5/login', { - ... 'cleartext_password': 'not-the-password', - ... }, method='POST') - Traceback (most recent call last): - ... - HTTPError: HTTP Error 403: 403 Forbidden diff --git a/src/mailman/rest/domains.py b/src/mailman/rest/domains.py index 5d36dcab9..9bc0edf6a 100644 --- a/src/mailman/rest/domains.py +++ b/src/mailman/rest/domains.py @@ -17,15 +17,14 @@ """REST for domains.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ADomain', 'AllDomains', ] +import six + from mailman.interfaces.domain import ( BadDomainSpecificationError, IDomainManager) from mailman.rest.helpers import ( @@ -99,10 +98,10 @@ class AllDomains(_DomainBase): """Create a new domain.""" domain_manager = getUtility(IDomainManager) try: - validator = Validator(mail_host=unicode, - description=unicode, - base_url=unicode, - contact_address=unicode, + validator = Validator(mail_host=six.text_type, + description=six.text_type, + base_url=six.text_type, + contact_address=six.text_type, _optional=('description', 'base_url', 'contact_address')) domain = domain_manager.add(**validator(request)) diff --git a/src/mailman/rest/helpers.py b/src/mailman/rest/helpers.py index 0bc312b1f..a39d6ceb3 100644 --- a/src/mailman/rest/helpers.py +++ b/src/mailman/rest/helpers.py @@ -17,9 +17,6 @@ """Web service helpers.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'BadRequest', 'ChildError', @@ -59,7 +56,7 @@ def path_to(resource): :return: The full path to the resource. :rtype: bytes """ - return b'{0}://{1}:{2}/{3}/{4}'.format( + return '{0}://{1}:{2}/{3}/{4}'.format( ('https' if as_boolean(config.webservice.use_https) else 'http'), config.webservice.hostname, config.webservice.port, @@ -107,8 +104,10 @@ def etag(resource): assert 'http_etag' not in resource, 'Resource already etagged' # Calculate the tag from a predictable (i.e. sorted) representation of the # dictionary. The actual details aren't so important. pformat() is - # guaranteed to sort the keys. - etag = hashlib.sha1(pformat(resource)).hexdigest() + # guaranteed to sort the keys, however it returns a str and the hash + # library requires a bytes. Use the safest possible encoding. + hashfood = pformat(resource).encode('raw-unicode-escape') + etag = hashlib.sha1(hashfood).hexdigest() resource['http_etag'] = '"{0}"'.format(etag) return json.dumps(resource, cls=ExtendedEncoder) diff --git a/src/mailman/rest/listconf.py b/src/mailman/rest/listconf.py index b432268c7..6cf54a00e 100644 --- a/src/mailman/rest/listconf.py +++ b/src/mailman/rest/listconf.py @@ -17,14 +17,13 @@ """Mailing list configuration via REST API.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ListConfiguration', ] +import six + from lazr.config import as_boolean, as_timedelta from mailman.config import config from mailman.core.errors import ( @@ -61,7 +60,7 @@ class AcceptableAliases(GetterSetter): alias_set = IAcceptableAliasSet(mlist) alias_set.clear() for alias in value: - alias_set.add(unicode(alias)) + alias_set.add(alias) @@ -71,13 +70,16 @@ class AcceptableAliases(GetterSetter): def pipeline_validator(pipeline_name): """Convert the pipeline name to a string, but only if it's known.""" if pipeline_name in config.pipelines: - return unicode(pipeline_name) + return pipeline_name raise ValueError('Unknown pipeline: {}'.format(pipeline_name)) -def list_of_unicode(values): +def list_of_str(values): """Turn a list of things into a list of unicodes.""" - return [unicode(value) for value in values] + for value in values: + if not isinstance(value, str): + raise ValueError('Expected str, got {!r}'.format(value)) + return values @@ -96,7 +98,7 @@ def list_of_unicode(values): # (e.g. datetimes, timedeltas, enums). ATTRIBUTES = dict( - acceptable_aliases=AcceptableAliases(list_of_unicode), + acceptable_aliases=AcceptableAliases(list_of_str), admin_immed_notify=GetterSetter(as_boolean), admin_notify_mchanges=GetterSetter(as_boolean), administrivia=GetterSetter(as_boolean), @@ -106,9 +108,9 @@ ATTRIBUTES = dict( autorespond_postings=GetterSetter(enum_validator(ResponseAction)), autorespond_requests=GetterSetter(enum_validator(ResponseAction)), autoresponse_grace_period=GetterSetter(as_timedelta), - autoresponse_owner_text=GetterSetter(unicode), - autoresponse_postings_text=GetterSetter(unicode), - autoresponse_request_text=GetterSetter(unicode), + autoresponse_owner_text=GetterSetter(six.text_type), + autoresponse_postings_text=GetterSetter(six.text_type), + autoresponse_request_text=GetterSetter(six.text_type), archive_policy=GetterSetter(enum_validator(ArchivePolicy)), bounces_address=GetterSetter(None), collapse_alternatives=GetterSetter(as_boolean), @@ -116,7 +118,7 @@ ATTRIBUTES = dict( created_at=GetterSetter(None), default_member_action=GetterSetter(enum_validator(Action)), default_nonmember_action=GetterSetter(enum_validator(Action)), - description=GetterSetter(unicode), + description=GetterSetter(six.text_type), digest_last_sent_at=GetterSetter(None), digest_size_threshold=GetterSetter(float), filter_content=GetterSetter(as_boolean), @@ -135,21 +137,21 @@ ATTRIBUTES = dict( post_id=GetterSetter(None), posting_address=GetterSetter(None), posting_pipeline=GetterSetter(pipeline_validator), - display_name=GetterSetter(unicode), + display_name=GetterSetter(six.text_type), reply_goes_to_list=GetterSetter(enum_validator(ReplyToMunging)), - reply_to_address=GetterSetter(unicode), + reply_to_address=GetterSetter(six.text_type), request_address=GetterSetter(None), scheme=GetterSetter(None), send_welcome_message=GetterSetter(as_boolean), - subject_prefix=GetterSetter(unicode), + subject_prefix=GetterSetter(six.text_type), volume=GetterSetter(None), web_host=GetterSetter(None), - welcome_message_uri=GetterSetter(unicode), + welcome_message_uri=GetterSetter(six.text_type), ) VALIDATORS = ATTRIBUTES.copy() -for attribute, gettersetter in VALIDATORS.items(): +for attribute, gettersetter in list(VALIDATORS.items()): if gettersetter.decoder is None: del VALIDATORS[attribute] diff --git a/src/mailman/rest/lists.py b/src/mailman/rest/lists.py index c96d5ded9..a8546b95e 100644 --- a/src/mailman/rest/lists.py +++ b/src/mailman/rest/lists.py @@ -17,9 +17,6 @@ """REST for mailing lists.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'AList', 'AllLists', @@ -30,10 +27,9 @@ __all__ = [ ] -from lazr.config import as_boolean -from operator import attrgetter -from zope.component import getUtility +import six +from lazr.config import as_boolean from mailman.app.lifecycle import create_list, remove_list from mailman.config import config from mailman.interfaces.domain import BadDomainSpecificationError @@ -50,6 +46,8 @@ from mailman.rest.helpers import ( from mailman.rest.members import AMember, MemberCollection from mailman.rest.moderation import HeldMessages, SubscriptionRequests from mailman.rest.validator import Validator +from operator import attrgetter +from zope.component import getUtility @@ -204,16 +202,15 @@ class AllLists(_ListBase): def on_post(self, request, response): """Create a new mailing list.""" try: - validator = Validator(fqdn_listname=unicode, - style_name=unicode, + validator = Validator(fqdn_listname=six.text_type, + style_name=six.text_type, _optional=('style_name',)) mlist = create_list(**validator(request)) except ListAlreadyExistsError: bad_request(response, b'Mailing list exists') except BadDomainSpecificationError as error: - bad_request( - response, - b'Domain does not exist: {0}'.format(error.domain)) + reason = 'Domain does not exist: {}'.format(error.domain) + bad_request(response, reason.encode('utf-8')) except ValueError as error: bad_request(response, str(error)) else: @@ -273,7 +270,7 @@ class ArchiverGetterSetter(GetterSetter): # attribute will contain the (bytes) name of the archiver that is # getting a new status. value will be the representation of the new # boolean status. - archiver = self._archiver_set.get(attribute.decode('utf-8')) + archiver = self._archiver_set.get(attribute) if archiver is None: raise ValueError('No such archiver: {}'.format(attribute)) archiver.is_enabled = as_boolean(value) diff --git a/src/mailman/rest/members.py b/src/mailman/rest/members.py index 4d1c87b73..ceaf54fc2 100644 --- a/src/mailman/rest/members.py +++ b/src/mailman/rest/members.py @@ -17,9 +17,6 @@ """REST for members.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'AMember', 'AllMembers', @@ -28,9 +25,7 @@ __all__ = [ ] -from uuid import UUID -from operator import attrgetter -from zope.component import getUtility +import six from mailman.app.membership import delete_member from mailman.interfaces.address import InvalidEmailAddressError @@ -47,6 +42,9 @@ from mailman.rest.helpers import ( from mailman.rest.preferences import Preferences, ReadOnlyPreferences from mailman.rest.validator import ( Validator, enum_validator, subscriber_validator) +from operator import attrgetter +from uuid import UUID +from zope.component import getUtility @@ -176,7 +174,7 @@ class AMember(_MemberBase): return try: values = Validator( - address=unicode, + address=six.text_type, delivery_mode=enum_validator(DeliveryMode), _optional=('address', 'delivery_mode'))(request) except ValueError as error: @@ -207,9 +205,9 @@ class AllMembers(_MemberBase): service = getUtility(ISubscriptionService) try: validator = Validator( - list_id=unicode, + list_id=six.text_type, subscriber=subscriber_validator, - display_name=unicode, + display_name=six.text_type, delivery_mode=enum_validator(DeliveryMode), role=enum_validator(MemberRole), _optional=('delivery_mode', 'display_name', 'role')) @@ -256,8 +254,8 @@ class FindMembers(_MemberBase): """Find a member""" service = getUtility(ISubscriptionService) validator = Validator( - list_id=unicode, - subscriber=unicode, + list_id=six.text_type, + subscriber=six.text_type, role=enum_validator(MemberRole), _optional=('list_id', 'subscriber', 'role')) try: diff --git a/src/mailman/rest/moderation.py b/src/mailman/rest/moderation.py index 0bdc50688..da182acb7 100644 --- a/src/mailman/rest/moderation.py +++ b/src/mailman/rest/moderation.py @@ -17,9 +17,6 @@ """REST API for Message moderation.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'HeldMessage', 'HeldMessages', @@ -88,7 +85,7 @@ class _HeldMessageBase(_ModerationBase): # resource. Others we can drop. Since we're mutating the dictionary, # we need to make a copy of the keys. When you port this to Python 3, # you'll need to list()-ify the .keys() dictionary view. - for key in resource.keys(): + for key in list(resource): if key in ('_mod_subject', '_mod_hold_date', '_mod_reason', '_mod_sender', '_mod_message_id'): resource[key[5:]] = resource.pop(key) diff --git a/src/mailman/rest/preferences.py b/src/mailman/rest/preferences.py index b85388ec9..9eafa8d77 100644 --- a/src/mailman/rest/preferences.py +++ b/src/mailman/rest/preferences.py @@ -17,9 +17,6 @@ """Preferences.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ReadOnlyPreferences', 'Preferences', diff --git a/src/mailman/rest/queues.py b/src/mailman/rest/queues.py new file mode 100644 index 000000000..f1007052e --- /dev/null +++ b/src/mailman/rest/queues.py @@ -0,0 +1,129 @@ +# Copyright (C) 2015 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""<api>/queues.""" + +__all__ = [ + 'AQueue', + 'AQueueFile', + 'AllQueues', + ] + + +import six + +from mailman.config import config +from mailman.app.inject import inject_text +from mailman.interfaces.listmanager import IListManager +from mailman.rest.helpers import ( + CollectionMixin, bad_request, created, etag, no_content, not_found, okay, + paginate, path_to) +from mailman.rest.validator import Validator +from zope.component import getUtility + + + +class _QueuesBase(CollectionMixin): + """Shared base class for queues.""" + + def _resource_as_dict(self, name): + """See `CollectionMixin`.""" + switchboard = config.switchboards[name] + files = switchboard.files + return dict( + name=switchboard.name, + directory=switchboard.queue_directory, + count=len(files), + files=files, + self_link=path_to('queues/{}'.format(name)), + ) + + @paginate + def _get_collection(self, request): + """See `CollectionMixin`.""" + return sorted(config.switchboards) + + + +class AQueue(_QueuesBase): + """A single queue.""" + + def __init__(self, name): + self._name = name + + def on_get(self, request, response): + """Return a single queue resource.""" + if self._name not in config.switchboards: + not_found(response) + else: + okay(response, self._resource_as_json(self._name)) + + def on_post(self, request, response): + """Inject a message into the queue.""" + try: + validator = Validator(list_id=six.text_type, + text=six.text_type) + values = validator(request) + except ValueError as error: + bad_request(response, str(error)) + return + list_id = values['list_id'] + mlist = getUtility(IListManager).get_by_list_id(list_id) + if mlist is None: + bad_request(response, 'No such list: {}'.format(list_id)) + return + try: + filebase = inject_text( + mlist, values['text'], switchboard=self._name) + except Exception as error: + bad_request(response, str(error)) + return + else: + location = path_to('queues/{}/{}'.format(self._name, filebase)) + created(response, location) + + + +class AQueueFile: + def __init__(self, name, filebase): + self._name = name + self._filebase = filebase + + def on_delete(self, request, response): + """Delete the queue file.""" + switchboard = config.switchboards.get(self._name) + if switchboard is None: + not_found(response, 'No such queue: {}'.format(self._name)) + return + try: + switchboard.dequeue(self._filebase) + except FileNotFoundError: + not_found(response, + 'No such queue file: {}'.format(self._filebase)) + else: + no_content(response) + + + +class AllQueues(_QueuesBase): + """All queues.""" + + def on_get(self, request, response): + """<api>/queues""" + resource = self._make_collection(request) + resource['self_link'] = path_to('queues') + okay(response, etag(resource)) diff --git a/src/mailman/rest/root.py b/src/mailman/rest/root.py index a3d18c201..381bec751 100644 --- a/src/mailman/rest/root.py +++ b/src/mailman/rest/root.py @@ -17,9 +17,6 @@ """The root of the REST API.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Root', ] @@ -28,8 +25,6 @@ __all__ = [ import falcon from base64 import b64decode -from zope.component import getUtility - from mailman.config import config from mailman.core.constants import system_preferences from mailman.core.system import system @@ -41,8 +36,10 @@ from mailman.rest.helpers import ( from mailman.rest.lists import AList, AllLists, Styles from mailman.rest.members import AMember, AllMembers, FindMembers from mailman.rest.preferences import ReadOnlyPreferences +from mailman.rest.queues import AQueue, AQueueFile, AllQueues from mailman.rest.templates import TemplateFinder from mailman.rest.users import AUser, AllUsers +from zope.component import getUtility @@ -66,17 +63,18 @@ class Root: # the case where no error is raised. if request.auth is None: raise falcon.HTTPUnauthorized( - b'401 Unauthorized', - b'The REST API requires authentication') + '401 Unauthorized', + 'The REST API requires authentication') if request.auth.startswith('Basic '): - credentials = b64decode(request.auth[6:]) + # b64decode() returns bytes, but we require a str. + credentials = b64decode(request.auth[6:]).decode('utf-8') username, password = credentials.split(':', 1) if (username != config.webservice.admin_user or password != config.webservice.admin_pass): # Not authorized. raise falcon.HTTPUnauthorized( - b'401 Unauthorized', - b'User is not authorized for the REST API') + '401 Unauthorized', + 'User is not authorized for the REST API') return TopLevel() @@ -216,3 +214,15 @@ class TopLevel: content_type = None return TemplateFinder( fqdn_listname, template, language, content_type) + + @child() + def queues(self, request, segments): + """/<api>/queues[/<name>[/file]]""" + if len(segments) == 0: + return AllQueues() + elif len(segments) == 1: + return AQueue(segments[0]), [] + elif len(segments) == 2: + return AQueueFile(segments[0], segments[1]), [] + else: + return BadRequest(), [] diff --git a/src/mailman/rest/templates.py b/src/mailman/rest/templates.py index 44dcdefc5..8d448a704 100644 --- a/src/mailman/rest/templates.py +++ b/src/mailman/rest/templates.py @@ -17,9 +17,6 @@ """Template finder.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TemplateFinder', ] diff --git a/src/mailman/rest/tests/test_addresses.py b/src/mailman/rest/tests/test_addresses.py index bbdd7d763..65c0c1e5a 100644 --- a/src/mailman/rest/tests/test_addresses.py +++ b/src/mailman/rest/tests/test_addresses.py @@ -17,9 +17,6 @@ """REST address tests.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestAddresses', ] @@ -27,15 +24,14 @@ __all__ = [ import unittest -from urllib2 import HTTPError -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.database.transaction import transaction from mailman.interfaces.usermanager import IUserManager from mailman.testing.helpers import call_api from mailman.testing.layers import RESTLayer from mailman.utilities.datetime import now +from six.moves.urllib_error import HTTPError +from zope.component import getUtility @@ -53,6 +49,12 @@ class TestAddresses(unittest.TestCase): self.assertEqual(json['start'], 0) self.assertEqual(json['total_size'], 0) + def test_missing_address(self): + # An address that isn't registered yet cannot be retrieved. + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.0/addresses/nobody@example.com') + self.assertEqual(cm.exception.code, 404) + def test_membership_of_missing_address(self): # Try to get the memberships of a missing address. with self.assertRaises(HTTPError) as cm: @@ -166,7 +168,7 @@ class TestAddresses(unittest.TestCase): 'email': 'anne@example.com', }) self.assertEqual(cm.exception.code, 400) - self.assertEqual(cm.exception.reason, 'Address already exists') + self.assertEqual(cm.exception.reason, b'Address already exists') def test_invalid_address_bad_request(self): # Trying to add an invalid address string returns 400. @@ -178,7 +180,7 @@ class TestAddresses(unittest.TestCase): 'email': 'invalid_address_string' }) self.assertEqual(cm.exception.code, 400) - self.assertEqual(cm.exception.reason, 'Invalid email address') + self.assertEqual(cm.exception.reason, b'Invalid email address') def test_empty_address_bad_request(self): # The address is required. @@ -189,7 +191,7 @@ class TestAddresses(unittest.TestCase): 'http://localhost:9001/3.0/users/anne@example.com/addresses', {}) self.assertEqual(cm.exception.code, 400) - self.assertEqual(cm.exception.reason, 'Missing parameters: email') + self.assertEqual(cm.exception.reason, b'Missing parameters: email') def test_get_addresses_of_missing_user(self): # There is no user associated with the given address. diff --git a/src/mailman/rest/tests/test_domains.py b/src/mailman/rest/tests/test_domains.py index 44cf11ef3..72ba4c003 100644 --- a/src/mailman/rest/tests/test_domains.py +++ b/src/mailman/rest/tests/test_domains.py @@ -17,9 +17,6 @@ """REST domain tests.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestDomains', ] @@ -27,14 +24,13 @@ __all__ = [ import unittest -from urllib2 import HTTPError -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.database.transaction import transaction from mailman.interfaces.listmanager import IListManager from mailman.testing.helpers import call_api from mailman.testing.layers import RESTLayer +from six.moves.urllib_error import HTTPError +from zope.component import getUtility @@ -65,7 +61,7 @@ class TestDomains(unittest.TestCase): content, response = call_api( 'http://localhost:9001/3.0/domains/example.com', method='DELETE') self.assertEqual(response.status, 204) - self.assertEqual(getUtility(IListManager).get('ant@example.com'), None) + self.assertIsNone(getUtility(IListManager).get('ant@example.com')) def test_missing_domain(self): # You get a 404 if you try to access a nonexisting domain. @@ -80,3 +76,14 @@ class TestDomains(unittest.TestCase): call_api( 'http://localhost:9001/3.0/domains/does-not-exist.com/lists') self.assertEqual(cm.exception.code, 404) + + def test_double_delete(self): + # You cannot delete a domain twice. + content, response = call_api( + 'http://localhost:9001/3.0/domains/example.com', + method='DELETE') + self.assertEqual(response.status, 204) + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.0/domains/example.com', + method='DELETE') + self.assertEqual(cm.exception.code, 404) diff --git a/src/mailman/rest/tests/test_listconf.py b/src/mailman/rest/tests/test_listconf.py index 93171ec4b..d013cdce9 100644 --- a/src/mailman/rest/tests/test_listconf.py +++ b/src/mailman/rest/tests/test_listconf.py @@ -17,9 +17,6 @@ """Test list configuration via the REST API.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestConfiguration', ] diff --git a/src/mailman/rest/tests/test_lists.py b/src/mailman/rest/tests/test_lists.py index ba6f6ea59..839fd0f58 100644 --- a/src/mailman/rest/tests/test_lists.py +++ b/src/mailman/rest/tests/test_lists.py @@ -17,9 +17,6 @@ """REST list tests.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestListArchivers', 'TestListPagination', @@ -30,14 +27,13 @@ __all__ = [ import unittest -from urllib2 import HTTPError -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.database.transaction import transaction from mailman.interfaces.usermanager import IUserManager from mailman.testing.helpers import call_api from mailman.testing.layers import RESTLayer +from six.moves.urllib_error import HTTPError +from zope.component import getUtility @@ -129,7 +125,7 @@ class TestLists(unittest.TestCase): }) self.assertEqual(cm.exception.code, 400) self.assertEqual(cm.exception.reason, - 'Domain does not exist: no-domain.example.org') + b'Domain does not exist: no-domain.example.org') def test_cannot_create_duplicate_list(self): # You cannot create a list that already exists. @@ -141,7 +137,7 @@ class TestLists(unittest.TestCase): 'fqdn_listname': 'ant@example.com', }) self.assertEqual(cm.exception.code, 400) - self.assertEqual(cm.exception.reason, 'Mailing list exists') + self.assertEqual(cm.exception.reason, b'Mailing list exists') def test_cannot_delete_missing_list(self): # You cannot delete a list that does not exist. @@ -220,7 +216,7 @@ class TestListArchivers(unittest.TestCase): method='PATCH') self.assertEqual(cm.exception.code, 400) self.assertEqual(cm.exception.reason, - 'Unexpected parameters: bogus-archiver') + b'Unexpected parameters: bogus-archiver') def test_put_incomplete_statuses(self): # PUT requires the full resource representation. This one forgets to @@ -233,7 +229,7 @@ class TestListArchivers(unittest.TestCase): method='PUT') self.assertEqual(cm.exception.code, 400) self.assertEqual(cm.exception.reason, - 'Missing parameters: mhonarc, prototype') + b'Missing parameters: mhonarc, prototype') def test_patch_bogus_status(self): # Archiver statuses must be interpretable as booleans. @@ -246,7 +242,7 @@ class TestListArchivers(unittest.TestCase): }, method='PATCH') self.assertEqual(cm.exception.code, 400) - self.assertEqual(cm.exception.reason, 'Invalid boolean value: sure') + self.assertEqual(cm.exception.reason, b'Invalid boolean value: sure') diff --git a/src/mailman/rest/tests/test_membership.py b/src/mailman/rest/tests/test_membership.py index 3c7d0520b..4ca28626f 100644 --- a/src/mailman/rest/tests/test_membership.py +++ b/src/mailman/rest/tests/test_membership.py @@ -17,9 +17,6 @@ """REST membership tests.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestMembership', 'TestNonmembership', @@ -28,9 +25,6 @@ __all__ = [ import unittest -from urllib2 import HTTPError -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.config import config from mailman.database.transaction import transaction @@ -41,6 +35,8 @@ from mailman.testing.helpers import ( from mailman.runners.incoming import IncomingRunner from mailman.testing.layers import ConfigLayer, RESTLayer from mailman.utilities.datetime import now +from six.moves.urllib_error import HTTPError +from zope.component import getUtility @@ -60,7 +56,7 @@ class TestMembership(unittest.TestCase): 'subscriber': 'nobody@example.com', }) self.assertEqual(cm.exception.code, 400) - self.assertEqual(cm.exception.msg, 'No such list') + self.assertEqual(cm.exception.reason, b'No such list') def test_try_to_leave_missing_list(self): # A user tries to leave a non-existent list. @@ -100,7 +96,7 @@ class TestMembership(unittest.TestCase): 'subscriber': 'anne@example.com', }) self.assertEqual(cm.exception.code, 409) - self.assertEqual(cm.exception.msg, 'Member already subscribed') + self.assertEqual(cm.exception.reason, b'Member already subscribed') def test_join_with_invalid_delivery_mode(self): with self.assertRaises(HTTPError) as cm: @@ -111,8 +107,8 @@ class TestMembership(unittest.TestCase): 'delivery_mode': 'invalid-mode', }) self.assertEqual(cm.exception.code, 400) - self.assertEqual(cm.exception.msg, - 'Cannot convert parameters: delivery_mode') + self.assertEqual(cm.exception.reason, + b'Cannot convert parameters: delivery_mode') def test_join_email_contains_slash(self): content, response = call_api('http://localhost:9001/3.0/members', { @@ -204,7 +200,7 @@ class TestMembership(unittest.TestCase): 'powers': 'super', }, method='PATCH') self.assertEqual(cm.exception.code, 400) - self.assertEqual(cm.exception.msg, 'Unexpected parameters: powers') + self.assertEqual(cm.exception.reason, b'Unexpected parameters: powers') def test_member_all_without_preferences(self): # /members/<id>/all should return a 404 when it isn't trailed by diff --git a/src/mailman/rest/tests/test_moderation.py b/src/mailman/rest/tests/test_moderation.py index c0ec4755a..2b72b91eb 100644 --- a/src/mailman/rest/tests/test_moderation.py +++ b/src/mailman/rest/tests/test_moderation.py @@ -17,17 +17,13 @@ """REST moderation tests.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ + 'TestModeration', ] import unittest -from urllib2 import HTTPError - from mailman.app.lifecycle import create_list from mailman.app.moderator import hold_message, hold_subscription from mailman.config import config @@ -36,6 +32,7 @@ from mailman.interfaces.member import DeliveryMode from mailman.testing.helpers import ( call_api, specialized_message_from_string as mfs) from mailman.testing.layers import RESTLayer +from six.moves.urllib_error import HTTPError @@ -97,7 +94,8 @@ Something else. with self.assertRaises(HTTPError) as cm: call_api(url.format(held_id), {'action': 'bogus'}) self.assertEqual(cm.exception.code, 400) - self.assertEqual(cm.exception.msg, 'Cannot convert parameters: action') + self.assertEqual(cm.exception.msg, + b'Cannot convert parameters: action') def test_bad_subscription_request_id(self): # Bad request when request_id is not an integer. @@ -123,4 +121,18 @@ Something else. with self.assertRaises(HTTPError) as cm: call_api(url.format(held_id), {'action': 'bogus'}) self.assertEqual(cm.exception.code, 400) - self.assertEqual(cm.exception.msg, 'Cannot convert parameters: action') + self.assertEqual(cm.exception.msg, + b'Cannot convert parameters: action') + + def test_discard(self): + # Discarding a message removes it from the moderation queue. + with transaction(): + held_id = hold_message(self._mlist, self._msg) + url = 'http://localhost:9001/3.0/lists/ant@example.com/held/{}'.format( + held_id) + content, response = call_api(url, dict(action='discard')) + self.assertEqual(response.status, 204) + # Now it's gone. + with self.assertRaises(HTTPError) as cm: + call_api(url, dict(action='discard')) + self.assertEqual(cm.exception.code, 404) diff --git a/src/mailman/rest/tests/test_paginate.py b/src/mailman/rest/tests/test_paginate.py index e267100c7..a482c7007 100644 --- a/src/mailman/rest/tests/test_paginate.py +++ b/src/mailman/rest/tests/test_paginate.py @@ -17,9 +17,6 @@ """paginate helper tests.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestPaginateHelper', ] diff --git a/src/mailman/rest/tests/test_preferences.py b/src/mailman/rest/tests/test_preferences.py index 91a066cff..6d34d7763 100644 --- a/src/mailman/rest/tests/test_preferences.py +++ b/src/mailman/rest/tests/test_preferences.py @@ -17,9 +17,6 @@ """Test various preference functionality.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestPreferences', ] @@ -32,10 +29,11 @@ from mailman.database.transaction import transaction from mailman.interfaces.usermanager import IUserManager from mailman.testing.helpers import call_api from mailman.testing.layers import RESTLayer -from urllib2 import HTTPError +from six.moves.urllib_error import HTTPError from zope.component import getUtility + class TestPreferences(unittest.TestCase): """Test various preference functionality.""" diff --git a/src/mailman/rest/tests/test_queues.py b/src/mailman/rest/tests/test_queues.py new file mode 100644 index 000000000..43659a2e4 --- /dev/null +++ b/src/mailman/rest/tests/test_queues.py @@ -0,0 +1,107 @@ +# Copyright (C) 2015 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test the `queues` resource.""" + +__all__ = [ + 'TestQueues', + ] + + +import unittest + +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.database.transaction import transaction +from mailman.testing.helpers import call_api, get_queue_messages +from mailman.testing.layers import RESTLayer +from six.moves.urllib_error import HTTPError + + +TEXT = """\ +From: anne@example.com +To: test@example.com +Subject: A test +Message-ID: <ant> + +""" + + + +class TestQueues(unittest.TestCase): + layer = RESTLayer + + def setUp(self): + with transaction(): + self._mlist = create_list('test@example.com') + + def test_missing_queue(self): + # Trying to print a missing queue gives a 404. + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.0/queues/notaq') + self.assertEqual(cm.exception.code, 404) + + def test_no_such_list(self): + # POSTing to a queue with a bad list-id gives a 400. + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.0/queues/bad', { + 'list_id': 'nosuchlist.example.com', + 'text': TEXT, + }) + self.assertEqual(cm.exception.code, 400) + + def test_inject(self): + # Injecting a message leaves the message in the queue. + starting_messages = get_queue_messages('bad') + self.assertEqual(len(starting_messages), 0) + content, response = call_api('http://localhost:9001/3.0/queues/bad', { + 'list_id': 'test.example.com', + 'text': TEXT}) + self.assertEqual(response.status, 201) + location = response['location'] + filebase = location.split('/')[-1] + # The message is in the 'bad' queue. + content, response = call_api('http://localhost:9001/3.0/queues/bad') + files = content['files'] + self.assertEqual(len(files), 1) + self.assertEqual(files[0], filebase) + # Verify the files directly. + files = list(config.switchboards['bad'].files) + self.assertEqual(len(files), 1) + self.assertEqual(files[0], filebase) + # Verify the content. + items = get_queue_messages('bad') + self.assertEqual(len(items), 1) + msg = items[0].msg + # Remove some headers that get added by Mailman. + del msg['date'] + self.assertEqual(msg['x-message-id-hash'], + 'MS6QLWERIJLGCRF44J7USBFDELMNT2BW') + del msg['x-message-id-hash'] + self.assertMultiLineEqual(msg.as_string(), TEXT) + + def test_delete_file(self): + # Inject a file, then delete it. + content, response = call_api('http://localhost:9001/3.0/queues/bad', { + 'list_id': 'test.example.com', + 'text': TEXT}) + location = response['location'] + self.assertEqual(len(config.switchboards['bad'].files), 1) + # Delete the file through REST. + content, response = call_api(location, method='DELETE') + self.assertEqual(response.status, 204) + self.assertEqual(len(config.switchboards['bad'].files), 0) diff --git a/src/mailman/rest/tests/test_root.py b/src/mailman/rest/tests/test_root.py index 510120087..59cd93637 100644 --- a/src/mailman/rest/tests/test_root.py +++ b/src/mailman/rest/tests/test_root.py @@ -17,9 +17,6 @@ """REST root object tests.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestRoot', ] @@ -35,7 +32,7 @@ from mailman.config import config from mailman.core.system import system from mailman.testing.helpers import call_api from mailman.testing.layers import RESTLayer -from urllib2 import HTTPError +from six.moves.urllib_error import HTTPError @@ -106,22 +103,23 @@ class TestRoot(unittest.TestCase): } response, raw_content = Http().request(url, 'GET', None, headers) self.assertEqual(response.status, 401) - content = json.loads(raw_content) + content = json.loads(raw_content.decode('utf-8')) self.assertEqual(content['title'], '401 Unauthorized') self.assertEqual(content['description'], 'The REST API requires authentication') def test_unauthorized(self): # Bad Basic Auth credentials results in a 401 error. - auth = b64encode('baduser:badpass') + userpass = b64encode(b'baduser:badpass') + auth = 'Basic {}'.format(userpass.decode('ascii')) url = 'http://localhost:9001/3.0/system' headers = { 'Content-Type': 'application/x-www-form-urlencode', - 'Authorization': 'Basic ' + auth, + 'Authorization': auth, } response, raw_content = Http().request(url, 'GET', None, headers) self.assertEqual(response.status, 401) - content = json.loads(raw_content) + content = json.loads(raw_content.decode('utf-8')) self.assertEqual(content['title'], '401 Unauthorized') self.assertEqual(content['description'], 'User is not authorized for the REST API') diff --git a/src/mailman/rest/tests/test_systemconf.py b/src/mailman/rest/tests/test_systemconf.py index 2eb4fa251..2158a024a 100644 --- a/src/mailman/rest/tests/test_systemconf.py +++ b/src/mailman/rest/tests/test_systemconf.py @@ -128,6 +128,7 @@ class TestSystemConfiguration(unittest.TestCase): 'passwords', 'paths.dev', 'paths.fhs', + 'paths.here', 'paths.local', 'paths.testing', 'runner.archive', diff --git a/src/mailman/rest/tests/test_users.py b/src/mailman/rest/tests/test_users.py index 10cc724a3..b4bd50330 100644 --- a/src/mailman/rest/tests/test_users.py +++ b/src/mailman/rest/tests/test_users.py @@ -17,9 +17,6 @@ """REST user tests.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestLP1074374', 'TestLogin', @@ -30,15 +27,14 @@ __all__ = [ import os import unittest -from urllib2 import HTTPError -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.config import config from mailman.database.transaction import transaction from mailman.interfaces.usermanager import IUserManager from mailman.testing.helpers import call_api, configuration from mailman.testing.layers import RESTLayer +from six.moves.urllib_error import HTTPError +from zope.component import getUtility @@ -108,6 +104,48 @@ class TestUsers(unittest.TestCase): method='DELETE') self.assertEqual(cm.exception.code, 404) + def test_delete_user_twice(self): + # You cannot DELETE a user twice, either by address or user id. + with transaction(): + anne = getUtility(IUserManager).create_user( + 'anne@example.com', 'Anne Person') + user_id = anne.user_id + content, response = call_api( + 'http://localhost:9001/3.0/users/anne@example.com', + method='DELETE') + self.assertEqual(response.status, 204) + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.0/users/anne@example.com', + method='DELETE') + self.assertEqual(cm.exception.code, 404) + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.0/users/{}'.format(user_id), + method='DELETE') + self.assertEqual(cm.exception.code, 404) + + def test_get_after_delete(self): + # You cannot GET a user record after deleting them. + with transaction(): + anne = getUtility(IUserManager).create_user( + 'anne@example.com', 'Anne Person') + user_id = anne.user_id + # You can still GET the user record. + content, response = call_api( + 'http://localhost:9001/3.0/users/anne@example.com') + self.assertEqual(response.status, 200) + # Delete the user. + content, response = call_api( + 'http://localhost:9001/3.0/users/anne@example.com', + method='DELETE') + self.assertEqual(response.status, 204) + # The user record can no longer be retrieved. + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.0/users/anne@example.com') + self.assertEqual(cm.exception.code, 404) + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.0/users/{}'.format(user_id)) + self.assertEqual(cm.exception.code, 404) + def test_existing_user_error(self): # Creating a user twice results in an error. call_api('http://localhost:9001/3.0/users', { @@ -120,7 +158,7 @@ class TestUsers(unittest.TestCase): }) self.assertEqual(cm.exception.code, 400) self.assertEqual(cm.exception.reason, - 'Address already exists: anne@example.com') + b'Address already exists: anne@example.com') def test_addresses_of_missing_user_id(self): # Trying to get the /addresses of a missing user id results in error. @@ -251,6 +289,21 @@ class TestLogin(unittest.TestCase): 'anne@example.com', 'Anne Person') self.anne.password = config.password_context.encrypt('abc123') + def test_login_with_cleartext_password(self): + # A user can log in with the correct clear text password. + content, response = call_api( + 'http://localhost:9001/3.0/users/anne@example.com/login', { + 'cleartext_password': 'abc123', + }, method='POST') + self.assertEqual(response.status, 204) + # But the user cannot log in with an incorrect password. + with self.assertRaises(HTTPError) as cm: + call_api( + 'http://localhost:9001/3.0/users/anne@example.com/login', { + 'cleartext_password': 'not-the-password', + }, method='POST') + self.assertEqual(cm.exception.code, 403) + def test_wrong_parameter(self): # A bad request because it is mistyped the required attribute. with self.assertRaises(HTTPError) as cm: diff --git a/src/mailman/rest/users.py b/src/mailman/rest/users.py index 7ab1d6818..175c1f76c 100644 --- a/src/mailman/rest/users.py +++ b/src/mailman/rest/users.py @@ -17,21 +17,15 @@ """REST for users.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'AUser', + 'AddressUser', 'AllUsers', 'Login', ] from lazr.config import as_boolean -from passlib.utils import generate_password as generate -from uuid import UUID -from zope.component import getUtility - from mailman.config import config from mailman.core.errors import ( ReadOnlyPATCHRequestError, UnknownPATCHRequestError) @@ -44,8 +38,12 @@ from mailman.rest.helpers import ( path_to) from mailman.rest.preferences import Preferences from mailman.rest.validator import PatchValidator, Validator +from passlib.utils import generate_password as generate +from uuid import UUID +from zope.component import getUtility + # Attributes of a user which can be changed via the REST API. class PasswordEncrypterGetterSetter(GetterSetter): def __init__(self): @@ -60,19 +58,20 @@ class PasswordEncrypterGetterSetter(GetterSetter): ATTRIBUTES = dict( - display_name=GetterSetter(unicode), + display_name=GetterSetter(str), cleartext_password=PasswordEncrypterGetterSetter(), ) CREATION_FIELDS = dict( - email=unicode, - display_name=unicode, - password=unicode, + email=str, + display_name=str, + password=str, _optional=('display_name', 'password'), ) + def create_user(arguments, response): """Create a new user.""" # We can't pass the 'password' argument to the user creation method, so @@ -83,7 +82,7 @@ def create_user(arguments, response): user = getUtility(IUserManager).create_user(**arguments) except ExistingAddressError as error: bad_request( - response, b'Address already exists: {}'.format(error.address)) + response, 'Address already exists: {}'.format(error.address)) return None if password is None: # This will have to be reset since it cannot be retrieved. @@ -360,7 +359,7 @@ class Login: # We do not want to encrypt the plaintext password given in the POST # data. That would hash the password, but we need to have the # plaintext in order to pass into passlib. - validator = Validator(cleartext_password=GetterSetter(unicode)) + validator = Validator(cleartext_password=GetterSetter(str)) try: values = validator(request) except ValueError as error: diff --git a/src/mailman/rest/validator.py b/src/mailman/rest/validator.py index cbcc5f652..017e31847 100644 --- a/src/mailman/rest/validator.py +++ b/src/mailman/rest/validator.py @@ -17,9 +17,6 @@ """REST web form validation.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'PatchValidator', 'Validator', @@ -29,12 +26,11 @@ __all__ = [ ] -from uuid import UUID -from zope.component import getUtility - from mailman.core.errors import ( ReadOnlyPATCHRequestError, UnknownPATCHRequestError) from mailman.interfaces.languages import ILanguageManager +from uuid import UUID +from zope.component import getUtility COMMASPACE = ', ' @@ -62,7 +58,7 @@ def subscriber_validator(subscriber): try: return UUID(int=int(subscriber)) except ValueError: - return unicode(subscriber) + return subscriber def language_validator(code): diff --git a/src/mailman/rest/wsgiapp.py b/src/mailman/rest/wsgiapp.py index 698c4269d..ad62244c8 100644 --- a/src/mailman/rest/wsgiapp.py +++ b/src/mailman/rest/wsgiapp.py @@ -17,9 +17,6 @@ """Basic WSGI Application object for REST server.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'make_application', 'make_server', @@ -85,7 +82,7 @@ class RootedAPI(API): if matcher is _missing: continue result = None - if isinstance(matcher, basestring): + if isinstance(matcher, str): # Is the matcher string a regular expression or plain # string? If it starts with a caret, it's a regexp. if matcher.startswith('^'): diff --git a/src/mailman/rules/administrivia.py b/src/mailman/rules/administrivia.py index 3052dcb46..866463d6c 100644 --- a/src/mailman/rules/administrivia.py +++ b/src/mailman/rules/administrivia.py @@ -17,20 +17,16 @@ """The administrivia rule.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Administrivia', ] from email.iterators import typed_subpart_iterator -from zope.interface import implementer - from mailman.config import config from mailman.core.i18n import _ from mailman.interfaces.rules import IRule +from zope.interface import implementer # The list of email commands we search for in the Subject header and payload. @@ -74,7 +70,7 @@ class Administrivia: # Search only the first text/plain subpart of the message. There's # really no good way to find email commands in any other content type. for part in typed_subpart_iterator(msg, 'text', 'plain'): - payload = part.get_payload(decode=True) + payload = part.get_payload() lines = payload.splitlines() # Count lines without using enumerate() because blank lines in the # payload don't count against the maximum examined. diff --git a/src/mailman/rules/any.py b/src/mailman/rules/any.py index e5f80fbc4..72f6da873 100644 --- a/src/mailman/rules/any.py +++ b/src/mailman/rules/any.py @@ -17,18 +17,14 @@ """Check if any previous rules have matched.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Any', ] -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.interfaces.rules import IRule +from zope.interface import implementer diff --git a/src/mailman/rules/approved.py b/src/mailman/rules/approved.py index 3b40d5dc9..5aa66c7df 100644 --- a/src/mailman/rules/approved.py +++ b/src/mailman/rules/approved.py @@ -17,9 +17,6 @@ """Look for moderator pre-approval.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Approved', ] @@ -28,11 +25,10 @@ __all__ = [ import re from email.iterators import typed_subpart_iterator -from zope.interface import implementer - from mailman.config import config from mailman.core.i18n import _ from mailman.interfaces.rules import IRule +from zope.interface import implementer EMPTYSTRING = '' @@ -113,7 +109,7 @@ class Approved: # may not work with rtf or whatever else is possible. pattern = header + ':(\s| )*' + re.escape(password) for part in typed_subpart_iterator(msg, 'text'): - payload = part.get_payload(decode=True) + payload = part.get_payload() if payload is not None: if re.search(pattern, payload): reset_payload(part, re.sub(pattern, '', payload)) diff --git a/src/mailman/rules/emergency.py b/src/mailman/rules/emergency.py index ba7abe562..a1addcdb7 100644 --- a/src/mailman/rules/emergency.py +++ b/src/mailman/rules/emergency.py @@ -17,18 +17,14 @@ """The emergency hold rule.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Emergency', ] -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.interfaces.rules import IRule +from zope.interface import implementer diff --git a/src/mailman/rules/implicit_dest.py b/src/mailman/rules/implicit_dest.py index 0bc229b15..9d3e6d079 100644 --- a/src/mailman/rules/implicit_dest.py +++ b/src/mailman/rules/implicit_dest.py @@ -17,21 +17,18 @@ """The implicit destination rule.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ImplicitDestination', ] import re -from email.utils import getaddresses -from zope.interface import implementer +from email.utils import getaddresses from mailman.core.i18n import _ from mailman.interfaces.mailinglist import IAcceptableAliasSet from mailman.interfaces.rules import IRule +from zope.interface import implementer diff --git a/src/mailman/rules/loop.py b/src/mailman/rules/loop.py index 145af8b34..30d7dde59 100644 --- a/src/mailman/rules/loop.py +++ b/src/mailman/rules/loop.py @@ -17,18 +17,14 @@ """Look for a posting loop.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Loop', ] -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.interfaces.rules import IRule +from zope.interface import implementer diff --git a/src/mailman/rules/max_recipients.py b/src/mailman/rules/max_recipients.py index 3b1d4f0c5..485368c0b 100644 --- a/src/mailman/rules/max_recipients.py +++ b/src/mailman/rules/max_recipients.py @@ -17,19 +17,15 @@ """The maximum number of recipients rule.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'MaximumRecipients', ] from email.utils import getaddresses -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.interfaces.rules import IRule +from zope.interface import implementer diff --git a/src/mailman/rules/max_size.py b/src/mailman/rules/max_size.py index 1e2b46184..4c8b58451 100644 --- a/src/mailman/rules/max_size.py +++ b/src/mailman/rules/max_size.py @@ -17,18 +17,14 @@ """The maximum message size rule.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'MaximumSize', ] -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.interfaces.rules import IRule +from zope.interface import implementer diff --git a/src/mailman/rules/moderation.py b/src/mailman/rules/moderation.py index 46ed242fa..5b79677ed 100644 --- a/src/mailman/rules/moderation.py +++ b/src/mailman/rules/moderation.py @@ -17,23 +17,19 @@ """Membership related rules.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'MemberModeration', 'NonmemberModeration', ] -from zope.component import getUtility -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.interfaces.action import Action from mailman.interfaces.member import MemberRole from mailman.interfaces.rules import IRule from mailman.interfaces.usermanager import IUserManager +from zope.component import getUtility +from zope.interface import implementer diff --git a/src/mailman/rules/news_moderation.py b/src/mailman/rules/news_moderation.py index c4372eb80..358368624 100644 --- a/src/mailman/rules/news_moderation.py +++ b/src/mailman/rules/news_moderation.py @@ -17,19 +17,15 @@ """The news moderation rule.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ModeratedNewsgroup', ] -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.interfaces.nntp import NewsgroupModeration from mailman.interfaces.rules import IRule +from zope.interface import implementer diff --git a/src/mailman/rules/no_subject.py b/src/mailman/rules/no_subject.py index 8f01f0c15..e66046832 100644 --- a/src/mailman/rules/no_subject.py +++ b/src/mailman/rules/no_subject.py @@ -17,18 +17,14 @@ """The no-Subject header rule.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'NoSubject', ] -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.interfaces.rules import IRule +from zope.interface import implementer diff --git a/src/mailman/rules/suspicious.py b/src/mailman/rules/suspicious.py index 1841ed69e..fbd76b794 100644 --- a/src/mailman/rules/suspicious.py +++ b/src/mailman/rules/suspicious.py @@ -17,9 +17,6 @@ """The historical 'suspicious header' rule.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'SuspiciousHeader', ] @@ -28,10 +25,10 @@ __all__ = [ import re import logging -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.interfaces.rules import IRule +from zope.interface import implementer + log = logging.getLogger('mailman.error') diff --git a/src/mailman/rules/tests/test_approved.py b/src/mailman/rules/tests/test_approved.py index 9976d4eff..83088da55 100644 --- a/src/mailman/rules/tests/test_approved.py +++ b/src/mailman/rules/tests/test_approved.py @@ -17,9 +17,6 @@ """Test the `approved` handler.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestApproved', 'TestApprovedNonASCII', @@ -36,8 +33,7 @@ from mailman.app.lifecycle import create_list from mailman.config import config from mailman.rules import approved from mailman.testing.helpers import ( - configuration, - specialized_message_from_string as mfs) + configuration, specialized_message_from_string as mfs) from mailman.testing.layers import ConfigLayer @@ -433,7 +429,7 @@ class TestPasswordHashMigration(unittest.TestCase): # hash is chosen after the original password is set. As long as the # old password still validates, the migration happens automatically. self._mlist.moderator_password = config.password_context.encrypt( - b'super secret') + 'super secret') self._rule = approved.Approved() self._msg = mfs("""\ From: anne@example.com @@ -450,7 +446,7 @@ A message body. # hashing algorithm. When the old password is validated, it will be # automatically migrated to the new hash. self.assertEqual(self._mlist.moderator_password, - b'{plaintext}super secret') + '{plaintext}super secret') config_file = os.path.join(config.VAR_DIR, 'passlib.config') # XXX passlib seems to choose the default hashing scheme even if it is # deprecated. The default scheme is either specified explicitly, or @@ -466,14 +462,14 @@ deprecated = roundup_plaintext self._msg['Approved'] = 'super secret' result = self._rule.check(self._mlist, self._msg, {}) self.assertTrue(result) - self.assertEqual(self._mlist.moderator_password, b'super secret') + self.assertEqual(self._mlist.moderator_password, 'super secret') def test_invalid_password_does_not_migrate(self): # Now that the moderator password is set, change the default password # hashing algorithm. When the old password is invalid, it will not be # automatically migrated to the new hash. self.assertEqual(self._mlist.moderator_password, - b'{plaintext}super secret') + '{plaintext}super secret') config_file = os.path.join(config.VAR_DIR, 'passlib.config') # XXX passlib seems to choose the default hashing scheme even if it is # deprecated. The default scheme is either specified explicitly, or @@ -490,9 +486,10 @@ deprecated = roundup_plaintext result = self._rule.check(self._mlist, self._msg, {}) self.assertFalse(result) self.assertEqual(self._mlist.moderator_password, - b'{plaintext}super secret') + '{plaintext}super secret') + class TestApprovedNoTextPlainPart(unittest.TestCase): """Test the approved handler with HTML-only messages.""" diff --git a/src/mailman/rules/tests/test_moderation.py b/src/mailman/rules/tests/test_moderation.py index c0c3cf417..2db4e53cc 100644 --- a/src/mailman/rules/tests/test_moderation.py +++ b/src/mailman/rules/tests/test_moderation.py @@ -17,9 +17,6 @@ """Test the `member-moderation` and `nonmember-moderation` rules.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestModeration', ] diff --git a/src/mailman/rules/truth.py b/src/mailman/rules/truth.py index d50b5eae4..0bf3345b7 100644 --- a/src/mailman/rules/truth.py +++ b/src/mailman/rules/truth.py @@ -17,18 +17,14 @@ """A rule which always matches.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Truth', ] -from zope.interface import implementer - from mailman.core.i18n import _ from mailman.interfaces.rules import IRule +from zope.interface import implementer diff --git a/src/mailman/runners/archive.py b/src/mailman/runners/archive.py index b49f5c265..f81f9ee3e 100644 --- a/src/mailman/runners/archive.py +++ b/src/mailman/runners/archive.py @@ -17,9 +17,6 @@ """Archive runner.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ArchiveRunner', ] @@ -31,7 +28,6 @@ import logging from email.utils import parsedate_tz, mktime_tz from datetime import datetime from lazr.config import as_timedelta - from mailman.config import config from mailman.core.runner import Runner from mailman.interfaces.archiver import ClobberDate diff --git a/src/mailman/runners/bounce.py b/src/mailman/runners/bounce.py index 9312a9158..3a85006fe 100644 --- a/src/mailman/runners/bounce.py +++ b/src/mailman/runners/bounce.py @@ -20,11 +20,10 @@ import logging from flufl.bounce import all_failures, scan_message -from zope.component import getUtility - from mailman.app.bounces import ProbeVERP, StandardVERP, maybe_forward from mailman.core.runner import Runner from mailman.interfaces.bounce import BounceContext, IBounceProcessor +from zope.component import getUtility COMMASPACE = ', ' @@ -33,7 +32,7 @@ log = logging.getLogger('mailman.bounce') elog = logging.getLogger('mailman.error') - + class BounceRunner(Runner): """The bounce runner.""" diff --git a/src/mailman/runners/command.py b/src/mailman/runners/command.py index 3d91f663a..b0775c4f4 100644 --- a/src/mailman/runners/command.py +++ b/src/mailman/runners/command.py @@ -17,9 +17,6 @@ """-request robot command runner.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'CommandRunner', 'Results', @@ -31,21 +28,20 @@ __all__ = [ # -owner. import re +import six import logging -from StringIO import StringIO from email.errors import HeaderParseError from email.header import decode_header, make_header from email.iterators import typed_subpart_iterator -from zope.component import getUtility -from zope.interface import implementer - from mailman.config import config from mailman.core.i18n import _ from mailman.core.runner import Runner from mailman.email.message import UserNotification from mailman.interfaces.command import ContinueProcessing, IEmailResults from mailman.interfaces.languages import ILanguageManager +from zope.component import getUtility +from zope.interface import implementer NL = '\n' @@ -76,7 +72,7 @@ class CommandFinder: # Extract the subject header and do RFC 2047 decoding. raw_subject = msg.get('subject', '') try: - subject = unicode(make_header(decode_header(raw_subject))) + subject = str(make_header(decode_header(raw_subject))) # Mail commands must be ASCII. self.command_lines.append(subject.encode('us-ascii')) except (HeaderParseError, UnicodeError, LookupError): @@ -84,7 +80,7 @@ class CommandFinder: # subject is a unicode object, convert it to ASCII ignoring all # bogus characters. Otherwise, there's nothing in the subject # that we can use. - if isinstance(raw_subject, unicode): + if isinstance(raw_subject, six.text_type): safe_subject = raw_subject.encode('us-ascii', 'ignore') self.command_lines.append(safe_subject) # Find the first text/plain part of the message. @@ -98,9 +94,9 @@ class CommandFinder: if part is None: # There was no text/plain part to be found. return - body = part.get_payload(decode=True) + body = part.get_payload() # text/plain parts better have string payloads. - assert isinstance(body, basestring), 'Non-string decoded payload' + assert isinstance(body, six.string_types), 'Non-string decoded payload' lines = body.splitlines() # Use no more lines than specified max_lines = int(config.mailman.email_commands_max_lines) @@ -118,7 +114,7 @@ class CommandFinder: # Ensure that all the parts are unicodes. Since we only accept # ASCII commands and arguments, ignore anything else. parts = [(part - if isinstance(part, unicode) + if isinstance(part, six.text_type) else part.decode('ascii', 'ignore')) for part in parts] yield parts @@ -130,20 +126,20 @@ class Results: """The email command results.""" def __init__(self, charset='us-ascii'): - self._output = StringIO() + self._output = six.StringIO() self.charset = charset print(_("""\ The results of your email command are provided below. """), file=self._output) def write(self, text): - if not isinstance(text, unicode): + if isinstance(text, bytes): text = text.decode(self.charset, 'ignore') self._output.write(text) - def __unicode__(self): + def __str__(self): value = self._output.getvalue() - assert isinstance(value, unicode), 'Not a unicode: %r' % value + assert isinstance(value, six.text_type), 'Not a unicode: %r' % value return value @@ -207,12 +203,12 @@ class CommandRunner(Runner): if status == ContinueProcessing.no: break # All done. Strip blank lines and send the response. - lines = filter(None, (line.strip() for line in finder.command_lines)) + lines = [line.strip() for line in finder.command_lines if line] if len(lines) > 0: print(_('\n- Unprocessed:'), file=results) for line in lines: print(line, file=results) - lines = filter(None, (line.strip() for line in finder.ignored_lines)) + lines = [line.strip() for line in finder.ignored_lines if line] if len(lines) > 0: print(_('\n- Ignored:'), file=results) for line in lines: @@ -231,7 +227,7 @@ class CommandRunner(Runner): # Find a charset for the response body. Try the original message's # charset first, then ascii, then latin-1 and finally falling back to # utf-8. - reply_body = unicode(results) + reply_body = str(results) for charset in (results.charset, 'us-ascii', 'latin-1'): try: reply_body.encode(charset) diff --git a/src/mailman/runners/digest.py b/src/mailman/runners/digest.py index e62c14abf..52bfb8859 100644 --- a/src/mailman/runners/digest.py +++ b/src/mailman/runners/digest.py @@ -17,9 +17,6 @@ """Digest runner.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'DigestRunner', ] @@ -28,15 +25,11 @@ __all__ = [ import re import logging -# cStringIO doesn't support unicode. -from StringIO import StringIO from copy import deepcopy from email.header import Header from email.mime.message import MIMEMessage from email.mime.text import MIMEText from email.utils import formatdate, getaddresses, make_msgid -from urllib2 import URLError - from mailman.config import config from mailman.core.i18n import _ from mailman.core.runner import Runner @@ -46,6 +39,8 @@ from mailman.interfaces.member import DeliveryMode, DeliveryStatus from mailman.utilities.i18n import make from mailman.utilities.mailbox import Mailbox from mailman.utilities.string import oneline, wrap +from six.moves import cStringIO as StringIO +from six.moves.urllib_error import URLError log = logging.getLogger('mailman.error') @@ -260,17 +255,16 @@ class RFC1153Digester(Digester): # multipart message. In that case, just stringify it. payload = msg.get_payload(decode=True) if not payload: - # Split using bytes so as not to turn the payload into unicode - # strings due to unicode_literals above. - payload = msg.as_string().split(b'\n\n', 1)[1] - try: - # Do the decoding inside the try/except so that if the charset - # conversion fails, we'll just drop back to ascii. - charset = msg.get_content_charset('us-ascii') - payload = payload.decode(charset, 'replace') - except (LookupError, TypeError): - # Unknown or empty charset. - payload = payload.decode('us-ascii', 'replace') + payload = msg.as_string().split('\n\n', 1)[1] + if isinstance(payload, bytes): + try: + # Do the decoding inside the try/except so that if the charset + # conversion fails, we'll just drop back to ascii. + charset = msg.get_content_charset('us-ascii') + payload = payload.decode(charset, 'replace') + except (LookupError, TypeError): + # Unknown or empty charset. + payload = payload.decode('us-ascii', 'replace') print(payload, file=self._text) if not payload.endswith('\n'): print(file=self._text) @@ -384,9 +378,9 @@ class DigestRunner(Runner): queue = config.switchboards['virgin'] queue.enqueue(mime, recipients=mime_recipients, - listname=mlist.fqdn_listname, + listid=mlist.list_id, isdigest=True) queue.enqueue(rfc1153, recipients=rfc1153_recipients, - listname=mlist.fqdn_listname, + listid=mlist.list_id, isdigest=True) diff --git a/src/mailman/runners/docs/command.rst b/src/mailman/runners/docs/command.rst index a7a4da8ed..82ee33fbc 100644 --- a/src/mailman/runners/docs/command.rst +++ b/src/mailman/runners/docs/command.rst @@ -27,7 +27,7 @@ the sender. The command can be in the ``Subject`` header. ... """) >>> from mailman.app.inject import inject_message - >>> inject_message(mlist, msg, switchboard='command') + >>> filebase = inject_message(mlist, msg, switchboard='command') >>> from mailman.runners.command import CommandRunner >>> from mailman.testing.helpers import make_testable_runner >>> command = make_testable_runner(CommandRunner) @@ -63,9 +63,9 @@ And now the response is in the ``virgin`` queue. >>> dump_msgdata(messages[0].msgdata) _parsemsg : False - listname : test@example.com + listid : test.example.com nodecorate : True - recipients : set([u'aperson@example.com']) + recipients : {'aperson@example.com'} reduced_list_headers: True version : ... @@ -85,7 +85,7 @@ message is plain text. ... echo foo bar ... """) - >>> inject_message(mlist, msg, switchboard='command') + >>> filebase = inject_message(mlist, msg, switchboard='command') >>> command.run() >>> messages = get_queue_messages('virgin') >>> len(messages) @@ -133,7 +133,8 @@ address, and the other is the results of his email command. ... ... """) - >>> inject_message(mlist, msg, switchboard='command', subaddress='join') + >>> filebase = inject_message( + ... mlist, msg, switchboard='command', subaddress='join') >>> command.run() >>> messages = get_queue_messages('virgin', sort_on='subject') >>> len(messages) @@ -165,7 +166,8 @@ Similarly, to leave a mailing list, the user need only email the ``-leave`` or ... ... """) - >>> inject_message(mlist, msg, switchboard='command', subaddress='leave') + >>> filebase = inject_message( + ... mlist, msg, switchboard='command', subaddress='leave') >>> command.run() >>> messages = get_queue_messages('virgin') >>> len(messages) @@ -200,7 +202,8 @@ The ``-confirm`` address is also available as an implicit command. ... ... """) - >>> inject_message(mlist, msg, switchboard='command', subaddress='confirm') + >>> filebase = inject_message( + ... mlist, msg, switchboard='command', subaddress='confirm') >>> command.run() >>> messages = get_queue_messages('virgin') >>> len(messages) @@ -244,7 +247,7 @@ looked at by the command queue. ... echo baz qux ... """) - >>> inject_message(mlist, msg, switchboard='command') + >>> filebase = inject_message(mlist, msg, switchboard='command') >>> command.run() >>> messages = get_queue_messages('virgin') >>> len(messages) @@ -276,7 +279,7 @@ The ``stop`` command is an alias for ``end``. ... echo baz qux ... """) - >>> inject_message(mlist, msg, switchboard='command') + >>> filebase = inject_message(mlist, msg, switchboard='command') >>> command.run() >>> messages = get_queue_messages('virgin') >>> len(messages) diff --git a/src/mailman/runners/docs/digester.rst b/src/mailman/runners/docs/digester.rst index cd0fba67c..fc59954bc 100644 --- a/src/mailman/runners/docs/digester.rst +++ b/src/mailman/runners/docs/digester.rst @@ -57,10 +57,11 @@ But the message metadata has a reference to the digest file. _parsemsg : False digest_number: 1 digest_path : .../lists/test@example.com/digest.1.1.mmdf - listname : test@example.com + listid : test.example.com version : 3 volume : 1 +.. # Put the messages back in the queue for the runner to handle. >>> filebase = digestq.enqueue(entry.msg, entry.msgdata) @@ -281,205 +282,6 @@ The RFC 1153 contains the digest in a single plain text message. <BLANKLINE> -Internationalized digests -========================= - -When messages come in with a content-type character set different than that of -the list's preferred language, recipients will get an internationalized -digest. French is not enabled by default site-wide, so enable that now. -:: - - # Simulate the site administrator setting the default server language to - # French in the configuration file. Without this, the English template - # will be found and the masthead won't be translated. - >>> config.push('french', """ - ... [mailman] - ... default_language: fr - ... """) - - >>> mlist.preferred_language = 'fr' - >>> msg = message_from_string("""\ - ... From: aperson@example.org - ... To: test@example.com - ... Subject: =?iso-2022-jp?b?GyRCMGxIVhsoQg==?= - ... MIME-Version: 1.0 - ... Content-Type: text/plain; charset=iso-2022-jp - ... Content-Transfer-Encoding: 7bit - ... - ... \x1b$B0lHV\x1b(B - ... """) - -Set the digest threshold to zero so that the digests will be sent immediately. - - >>> mlist.digest_size_threshold = 0 - >>> process(mlist, msg, {}) - -The marker message is sitting in the digest queue. - - >>> len(digestq.files) - 1 - >>> entry = get_queue_messages('digest')[0] - >>> dump_msgdata(entry.msgdata) - _parsemsg : False - digest_number: 2 - digest_path : .../lists/test@example.com/digest.1.2.mmdf - listname : test@example.com - version : 3 - volume : 1 - -The digest runner runs a loop, placing the two digests into the virgin queue. - - # Put the messages back in the queue for the runner to handle. - >>> filebase = digestq.enqueue(entry.msg, entry.msgdata) - >>> runner.run() - >>> messages = get_queue_messages('virgin') - >>> len(messages) - 2 - -One of which is the MIME digest and the other of which is the RFC 1153 digest. - - >>> mime, rfc1153 = mime_rfc1153(messages) - -You can see that the digests contain a mix of French and Japanese. - - >>> print(mime.msg.as_string()) - Content-Type: multipart/mixed; boundary="===============...==" - MIME-Version: 1.0 - From: test-request@example.com - Subject: Groupe Test, Vol 1, Parution 2 - To: test@example.com - Reply-To: test@example.com - Date: ... - Message-ID: ... - <BLANKLINE> - --===============...== - Content-Type: text/plain; charset="iso-8859-1" - MIME-Version: 1.0 - Content-Transfer-Encoding: quoted-printable - Content-Description: Groupe Test, Vol 1, Parution 2 - <BLANKLINE> - Envoyez vos messages pour la liste Test =E0 - test@example.com - <BLANKLINE> - Pour vous (d=E9s)abonner par le web, consultez - http://lists.example.com/listinfo/test@example.com - <BLANKLINE> - ou, par courriel, envoyez un message avec =AB=A0help=A0=BB dans le corps ou - dans le sujet =E0 - test-request@example.com - <BLANKLINE> - Vous pouvez contacter l'administrateur de la liste =E0 l'adresse - test-owner@example.com - <BLANKLINE> - Si vous r=E9pondez, n'oubliez pas de changer l'objet du message afin - qu'il soit plus sp=E9cifique que =AB=A0Re: Contenu du groupe de Test...=A0= - =BB - --===============...== - Content-Type: text/plain; charset="utf-8" - MIME-Version: 1.0 - Content-Transfer-Encoding: base64 - Content-Description: Today's Topics (1 messages) - <BLANKLINE> - VGjDqG1lcyBkdSBqb3VyIDoKCiAgIDEuIOS4gOeVqiAoYXBlcnNvbkBleGFtcGxlLm9yZykK - <BLANKLINE> - --===============...== - Content-Type: message/rfc822 - MIME-Version: 1.0 - <BLANKLINE> - From: aperson@example.org - To: test@example.com - Subject: =?iso-2022-jp?b?GyRCMGxIVhsoQg==?= - MIME-Version: 1.0 - Content-Type: text/plain; charset=iso-2022-jp - Content-Transfer-Encoding: 7bit - <BLANKLINE> - $B0lHV(B - <BLANKLINE> - --===============...== - Content-Type: text/plain; charset="iso-8859-1" - MIME-Version: 1.0 - Content-Transfer-Encoding: quoted-printable - Content-Description: =?utf-8?q?Pied_de_page_des_remises_group=C3=A9es?= - <BLANKLINE> - _______________________________________________ - Test mailing list - test@example.com - http://lists.example.com/listinfo/test@example.com - <BLANKLINE> - --===============...==-- - -The RFC 1153 digest will be encoded in UTF-8 since it contains a mixture of -French and Japanese characters. - - >>> print(rfc1153.msg.as_string()) - From: test-request@example.com - Subject: Groupe Test, Vol 1, Parution 2 - To: test@example.com - Reply-To: test@example.com - Date: ... - Message-ID: ... - MIME-Version: 1.0 - Content-Type: text/plain; charset="utf-8" - Content-Transfer-Encoding: base64 - <BLANKLINE> - RW52b... - <BLANKLINE> - -The content can be decoded to see the actual digest text. -:: - - # We must display the repr of the decoded value because doctests cannot - # handle the non-ascii characters. - >>> [repr(line) - ... for line in rfc1153.msg.get_payload(decode=True).splitlines()] - ["'Envoyez vos messages pour la liste Test \\xc3\\xa0'", - "'\\ttest@example.com'", - "''", - "'Pour vous (d\\xc3\\xa9s)abonner par le web, consultez'", - "'\\thttp://lists.example.com/listinfo/test@example.com'", - "''", - "'ou, par courriel, envoyez un message avec \\xc2\\xab\\xc2\\xa0... - "'dans le sujet \\xc3\\xa0'", - "'\\ttest-request@example.com'", - "''", - '"Vous pouvez contacter l\'administrateur de la liste \\xc3\\xa0 ... - "'\\ttest-owner@example.com'", - "''", - '"Si vous r\\xc3\\xa9pondez, n\'oubliez pas de changer l\'objet du ... - '"qu\'il soit plus sp\\xc3\\xa9cifique que \\xc2\\xab\\xc2\\xa0Re: ... - "''", - "'Th\\xc3\\xa8mes du jour :'", - "''", - "' 1. \\xe4\\xb8\\x80\\xe7\\x95\\xaa (aperson@example.org)'", - "''", - "''", - "'---------------------------------------------------------------------... - "''", - "'From: aperson@example.org'", - "'Subject: \\xe4\\xb8\\x80\\xe7\\x95\\xaa'", - "'To: test@example.com'", - "'Content-Type: text/plain; charset=iso-2022-jp'", - "''", - "'\\xe4\\xb8\\x80\\xe7\\x95\\xaa'", - "''", - "'------------------------------'", - "''", - "'Subject: Pied de page des remises group\\xc3\\xa9es'", - "''", - "'_______________________________________________'", - "'Test mailing list'", - "'test@example.com'", - "'http://lists.example.com/listinfo/test@example.com'", - "''", - "''", - "'------------------------------'", - "''", - "'Fin de Groupe Test, Vol 1, Parution 2'", - "'*************************************'"] - - >>> config.pop('french') - - Digest delivery =============== @@ -538,12 +340,12 @@ and the other is the RFC 1153 digest. Only wperson and xperson get the MIME digests. >>> sorted(mime.msgdata['recipients']) - [u'wperson@example.com', u'xperson@example.com'] + ['wperson@example.com', 'xperson@example.com'] Only yperson and zperson get the RFC 1153 digests. >>> sorted(rfc1153.msgdata['recipients']) - [u'yperson@example.com', u'zperson@example.com'] + ['yperson@example.com', 'zperson@example.com'] Now uperson decides that they would like to start receiving digests too. :: @@ -558,10 +360,10 @@ Now uperson decides that they would like to start receiving digests too. >>> mime, rfc1153 = mime_rfc1153(messages) >>> sorted(mime.msgdata['recipients']) - [u'uperson@example.com', u'wperson@example.com', u'xperson@example.com'] + ['uperson@example.com', 'wperson@example.com', 'xperson@example.com'] >>> sorted(rfc1153.msgdata['recipients']) - [u'yperson@example.com', u'zperson@example.com'] + ['yperson@example.com', 'zperson@example.com'] At this point, both uperson and wperson decide that they'd rather receive regular deliveries instead of digests. uperson would like to get any last @@ -581,10 +383,10 @@ as much and does not want to receive one last digest. >>> messages = get_queue_messages('virgin') >>> mime, rfc1153 = mime_rfc1153(messages) >>> sorted(mime.msgdata['recipients']) - [u'uperson@example.com', u'xperson@example.com'] + ['uperson@example.com', 'xperson@example.com'] >>> sorted(rfc1153.msgdata['recipients']) - [u'yperson@example.com', u'zperson@example.com'] + ['yperson@example.com', 'zperson@example.com'] Since uperson has received their last digest, they will not get any more of them. @@ -599,7 +401,7 @@ them. >>> mime, rfc1153 = mime_rfc1153(messages) >>> sorted(mime.msgdata['recipients']) - [u'xperson@example.com'] + ['xperson@example.com'] >>> sorted(rfc1153.msgdata['recipients']) - [u'yperson@example.com', u'zperson@example.com'] + ['yperson@example.com', 'zperson@example.com'] diff --git a/src/mailman/runners/docs/incoming.rst b/src/mailman/runners/docs/incoming.rst index 0ae3336ca..d4fb65c85 100644 --- a/src/mailman/runners/docs/incoming.rst +++ b/src/mailman/runners/docs/incoming.rst @@ -54,7 +54,7 @@ Inject the message into the incoming queue, similar to the way the upstream mail server normally would. >>> from mailman.app.inject import inject_message - >>> inject_message(mlist, msg) + >>> filebase = inject_message(mlist, msg) The incoming runner runs until it is empty. @@ -103,7 +103,7 @@ that it will be accepted and forward to the pipeline queue. Inject the message into the incoming queue and run until the queue is empty. - >>> inject_message(mlist, msg) + >>> filebase = inject_message(mlist, msg) >>> incoming.run() There are no messages left in the incoming queue. @@ -156,7 +156,7 @@ pipeline queue. >>> from mailman.testing.helpers import event_subscribers >>> with event_subscribers(on_chain): - ... inject_message(mlist, msg) + ... filebase = inject_message(mlist, msg) ... incoming.run() <mailman.interfaces.chain.HoldEvent ...> <mailman.chains.hold.HoldChain ...> @@ -191,7 +191,7 @@ new chain and set it as the mailing list's start chain. >>> msg.replace_header('message-id', '<second>') >>> with event_subscribers(on_chain): - ... inject_message(mlist, msg) + ... filebase = inject_message(mlist, msg) ... incoming.run() <mailman.interfaces.chain.DiscardEvent ...> <mailman.chains.discard.DiscardChain ...> @@ -220,7 +220,7 @@ just create a new chain that does. >>> msg.replace_header('message-id', '<third>') >>> with event_subscribers(on_chain): - ... inject_message(mlist, msg) + ... filebase = inject_message(mlist, msg) ... incoming.run() <mailman.interfaces.chain.RejectEvent ...> <mailman.chains.reject.RejectChain ...> diff --git a/src/mailman/runners/docs/lmtp.rst b/src/mailman/runners/docs/lmtp.rst index c2227581f..45e8a3453 100644 --- a/src/mailman/runners/docs/lmtp.rst +++ b/src/mailman/runners/docs/lmtp.rst @@ -20,7 +20,7 @@ Let's start a testable LMTP runner. It also helps to have a nice LMTP client. >>> lmtp = helpers.get_lmtp_client() - (220, '... Python LMTP runner 1.0') + (220, b'... Python LMTP runner 1.0') >>> lmtp.lhlo('remote.example.org') (250, ...) @@ -28,24 +28,8 @@ It also helps to have a nice LMTP client. Posting address =============== -If the mail server tries to send a message to a nonexistent mailing list, it -will get a 550 error. - - >>> lmtp.sendmail( - ... 'anne.person@example.com', - ... ['mylist@example.com'], """\ - ... From: anne.person@example.com - ... To: mylist@example.com - ... Subject: An interesting message - ... Message-ID: <aardvark> - ... - ... This is an interesting message. - ... """) - Traceback (most recent call last): - ... - SMTPDataError: (550, 'Requested action not taken: mailbox unavailable') - -Once the mailing list is created, the posting address is valid. +Once the mailing list is created, the posting address is valid, and messages +can be sent to the list. :: >>> create_list('mylist@example.com') @@ -82,7 +66,7 @@ queue. This is an interesting message. >>> dump_msgdata(messages[0].msgdata) _parsemsg : False - listname : mylist@example.com + listid : mylist.example.com original_size: ... to_list : True version : ... @@ -92,24 +76,8 @@ Sub-addresses ============= The LMTP server understands each of the list's sub-addreses, such as `-join`, -`-leave`, `-request` and so on. If the message is posted to an invalid -sub-address though, it is rejected. - - >>> lmtp.sendmail( - ... 'anne.person@example.com', - ... ['mylist-bogus@example.com'], """\ - ... From: anne.person@example.com - ... To: mylist-bogus@example.com - ... Subject: Help - ... Message-ID: <cow> - ... - ... Please help me. - ... """) - Traceback (most recent call last): - ... - SMTPDataError: (550, 'Requested action not taken: mailbox unavailable') - -But the message is accepted if posted to a valid sub-address. +`-leave`, `-request` and so on. The message is accepted if posted to a valid +sub-address. >>> lmtp.sendmail( ... 'anne.person@example.com', @@ -145,7 +113,7 @@ command queue for processing. Please help me. >>> dump_msgdata(messages[0].msgdata) _parsemsg : False - listname : mylist@example.com + listid : mylist.example.com original_size: ... subaddress : request version : ... @@ -172,7 +140,7 @@ A message to the `-bounces` address goes to the bounce processor. 1 >>> dump_msgdata(messages[0].msgdata) _parsemsg : False - listname : mylist@example.com + listid : mylist.example.com original_size: ... subaddress : bounces version : ... @@ -199,7 +167,7 @@ Confirmation messages go to the command processor... 1 >>> dump_msgdata(messages[0].msgdata) _parsemsg : False - listname : mylist@example.com + listid : mylist.example.com original_size: ... subaddress : confirm version : ... @@ -221,7 +189,7 @@ Confirmation messages go to the command processor... 1 >>> dump_msgdata(messages[0].msgdata) _parsemsg : False - listname : mylist@example.com + listid : mylist.example.com original_size: ... subaddress : join version : ... @@ -240,7 +208,7 @@ Confirmation messages go to the command processor... 1 >>> dump_msgdata(messages[0].msgdata) _parsemsg : False - listname : mylist@example.com + listid : mylist.example.com original_size: ... subaddress : join version : ... @@ -262,7 +230,7 @@ Confirmation messages go to the command processor... 1 >>> dump_msgdata(messages[0].msgdata) _parsemsg : False - listname : mylist@example.com + listid : mylist.example.com original_size: ... subaddress : leave version : ... @@ -281,7 +249,7 @@ Confirmation messages go to the command processor... 1 >>> dump_msgdata(messages[0].msgdata) _parsemsg : False - listname : mylist@example.com + listid : mylist.example.com original_size: ... subaddress : leave version : ... @@ -307,7 +275,7 @@ Messages to the `-owner` address also go to the incoming processor. >>> dump_msgdata(messages[0].msgdata) _parsemsg : False envsender : noreply@example.com - listname : mylist@example.com + listid : mylist.example.com original_size: ... subaddress : owner to_owner : True diff --git a/src/mailman/runners/docs/nntp.rst b/src/mailman/runners/docs/nntp.rst index 372fa5744..4bd73cbab 100644 --- a/src/mailman/runners/docs/nntp.rst +++ b/src/mailman/runners/docs/nntp.rst @@ -37,7 +37,7 @@ are prohibited by NNTP servers such as INN. The message gets copied to the NNTP queue for preparation and posting. >>> filebase = config.switchboards['nntp'].enqueue( - ... msg, listname='test@example.com') + ... msg, listid='test.example.com') >>> from mailman.testing.helpers import make_testable_runner >>> from mailman.runners.nntp import NNTPRunner >>> runner = make_testable_runner(NNTPRunner, 'nntp') diff --git a/src/mailman/runners/docs/outgoing.rst b/src/mailman/runners/docs/outgoing.rst index d4a20d497..7c3d1a989 100644 --- a/src/mailman/runners/docs/outgoing.rst +++ b/src/mailman/runners/docs/outgoing.rst @@ -57,7 +57,7 @@ destination mailing list name. Simulate that here too. >>> ignore = outgoing_queue.enqueue( ... msg, msgdata, ... tolist=True, - ... listname=mlist.fqdn_listname) + ... listid=mlist.list_id) Running the outgoing runner processes the message, delivering it to the upstream SMTP. @@ -105,7 +105,7 @@ just one. >>> ignore = outgoing_queue.enqueue( ... msg, msgdata, - ... listname=mlist.fqdn_listname) + ... listid=mlist.list_id) >>> outgoing.run() >>> messages = list(smtpd.messages) >>> len(messages) @@ -147,7 +147,7 @@ A handler can force VERP by setting the ``verp`` key in the message metadata. >>> ignore = outgoing_queue.enqueue( ... msg, msgdata, ... verp=True, - ... listname=mlist.fqdn_listname) + ... listid=mlist.list_id) >>> outgoing.run() >>> messages = list(smtpd.messages) >>> len(messages) @@ -174,7 +174,7 @@ Again, we get three individual messages, with VERP'd ``Sender`` headers. >>> ignore = outgoing_queue.enqueue( ... msg, msgdata, - ... listname=mlist.fqdn_listname) + ... listid=mlist.list_id) >>> outgoing.run() >>> messages = list(smtpd.messages) >>> len(messages) @@ -215,7 +215,7 @@ VERP'd. >>> ignore = outgoing_queue.enqueue( ... msg, msgdata, - ... listname=mlist.fqdn_listname) + ... listid=mlist.list_id) >>> outgoing.run() >>> messages = list(smtpd.messages) >>> len(messages) @@ -235,7 +235,7 @@ The second message sent to the list is also not VERP'd. >>> ignore = outgoing_queue.enqueue( ... msg, msgdata, - ... listname=mlist.fqdn_listname) + ... listid=mlist.list_id) >>> outgoing.run() >>> messages = list(smtpd.messages) >>> len(messages) @@ -254,7 +254,7 @@ The third message though is VERP'd. >>> ignore = outgoing_queue.enqueue( ... msg, msgdata, - ... listname=mlist.fqdn_listname) + ... listid=mlist.list_id) >>> outgoing.run() >>> messages = list(smtpd.messages) >>> len(messages) @@ -274,7 +274,7 @@ The next one is back to bulk delivery. >>> ignore = outgoing_queue.enqueue( ... msg, msgdata, - ... listname=mlist.fqdn_listname) + ... listid=mlist.list_id) >>> outgoing.run() >>> messages = list(smtpd.messages) >>> len(messages) @@ -308,7 +308,7 @@ The first message is VERP'd. >>> ignore = outgoing_queue.enqueue( ... msg, msgdata, - ... listname=mlist.fqdn_listname) + ... listid=mlist.list_id) >>> outgoing.run() >>> messages = list(smtpd.messages) >>> len(messages) @@ -328,7 +328,7 @@ As is the second message. >>> ignore = outgoing_queue.enqueue( ... msg, msgdata, - ... listname=mlist.fqdn_listname) + ... listid=mlist.list_id) >>> outgoing.run() >>> messages = list(smtpd.messages) >>> len(messages) @@ -348,7 +348,7 @@ And the third message. >>> ignore = outgoing_queue.enqueue( ... msg, msgdata, - ... listname=mlist.fqdn_listname) + ... listid=mlist.list_id) >>> outgoing.run() >>> messages = list(smtpd.messages) >>> len(messages) @@ -387,7 +387,7 @@ Neither the first message... >>> ignore = outgoing_queue.enqueue( ... msg, msgdata, - ... listname=mlist.fqdn_listname) + ... listid=mlist.list_id) >>> outgoing.run() >>> messages = list(smtpd.messages) >>> len(messages) @@ -402,7 +402,7 @@ Neither the first message... >>> ignore = outgoing_queue.enqueue( ... msg, msgdata, - ... listname=mlist.fqdn_listname) + ... listid=mlist.list_id) >>> outgoing.run() >>> messages = list(smtpd.messages) >>> len(messages) diff --git a/src/mailman/runners/incoming.py b/src/mailman/runners/incoming.py index d75469a5e..a5d8fbea3 100644 --- a/src/mailman/runners/incoming.py +++ b/src/mailman/runners/incoming.py @@ -26,21 +26,17 @@ prepared for delivery. Rejections, discards, and holds are processed immediately. """ -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'IncomingRunner', ] -from zope.component import getUtility - from mailman.core.chains import process from mailman.core.runner import Runner from mailman.database.transaction import transaction from mailman.interfaces.address import ExistingAddressError from mailman.interfaces.usermanager import IUserManager +from zope.component import getUtility diff --git a/src/mailman/runners/lmtp.py b/src/mailman/runners/lmtp.py index 7560fd962..85730bb7d 100644 --- a/src/mailman/runners/lmtp.py +++ b/src/mailman/runners/lmtp.py @@ -34,9 +34,6 @@ so that the peer mail server can provide better diagnostics. http://www.faqs.org/rfcs/rfc2033.html """ -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'LMTPRunner', ] @@ -48,8 +45,6 @@ import logging import asyncore from email.utils import parseaddr -from zope.component import getUtility - from mailman.config import config from mailman.core.runner import Runner from mailman.database.transaction import transactional @@ -57,6 +52,7 @@ from mailman.email.message import Message from mailman.interfaces.listmanager import IListManager from mailman.utilities.datetime import now from mailman.utilities.email import add_message_hash +from zope.component import getUtility elog = logging.getLogger('mailman.error') @@ -91,15 +87,15 @@ SUBADDRESS_QUEUES = dict( ) DASH = '-' -CRLF = b'\r\n' -ERR_451 = b'451 Requested action aborted: error in processing' -ERR_501 = b'501 Message has defects' -ERR_502 = b'502 Error: command HELO not implemented' -ERR_550 = b'550 Requested action not taken: mailbox unavailable' -ERR_550_MID = b'550 No Message-ID header provided' +CRLF = '\r\n' +ERR_451 = '451 Requested action aborted: error in processing' +ERR_501 = '501 Message has defects' +ERR_502 = '502 Error: command HELO not implemented' +ERR_550 = '550 Requested action not taken: mailbox unavailable' +ERR_550_MID = '550 No Message-ID header provided' # XXX Blech -smtpd.__version__ = b'Python LMTP runner 1.0' +smtpd.__version__ = 'Python LMTP runner 1.0' @@ -147,6 +143,10 @@ class Channel(smtpd.SMTPChannel): """HELO is not a valid LMTP command.""" self.push(ERR_502) + ## def push(self, arg): + ## import pdb; pdb.set_trace() + ## return super().push(arg) + class LMTPRunner(Runner, smtpd.SMTPServer): @@ -202,18 +202,19 @@ class LMTPRunner(Runner, smtpd.SMTPServer): for to in rcpttos: try: to = parseaddr(to)[1].lower() - listname, subaddress, domain = split_recipient(to) + local, subaddress, domain = split_recipient(to) slog.debug('%s to: %s, list: %s, sub: %s, dom: %s', - message_id, to, listname, subaddress, domain) - listname += '@' + domain + message_id, to, local, subaddress, domain) + listname = '{}@{}'.format(local, domain) if listname not in listnames: status.append(ERR_550) continue + listid = '{}.{}'.format(local, domain) # The recipient is a valid mailing list. Find the subaddress # if there is one, and set things up to enqueue to the proper # queue. queue = None - msgdata = dict(listname=listname, + msgdata = dict(listid=listid, original_size=msg.original_size, received_time=received_time) canonical_subaddress = SUBADDRESS_NAMES.get(subaddress) @@ -243,7 +244,7 @@ class LMTPRunner(Runner, smtpd.SMTPServer): config.switchboards[queue].enqueue(msg, msgdata) slog.debug('%s subaddress: %s, queue: %s', message_id, canonical_subaddress, queue) - status.append(b'250 Ok') + status.append('250 Ok') except Exception: slog.exception('Queue detection: %s', msg['message-id']) config.db.abort() diff --git a/src/mailman/runners/nntp.py b/src/mailman/runners/nntp.py index 493f8d09a..7fb16f1b2 100644 --- a/src/mailman/runners/nntp.py +++ b/src/mailman/runners/nntp.py @@ -17,9 +17,6 @@ """NNTP runner.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'NNTPRunner', ] @@ -31,11 +28,11 @@ import socket import logging import nntplib -from cStringIO import StringIO - from mailman.config import config from mailman.core.runner import Runner from mailman.interfaces.nntp import NewsgroupModeration +from six.moves import cStringIO as StringIO + COMMA = ',' COMMASPACE = ', ' @@ -82,7 +79,7 @@ class NNTPRunner(Runner): user=config.nntp.user, password=config.nntp.password) conn.post(fp) - except nntplib.error_temp: + except nntplib.NNTPTemporaryError: log.exception('{0} NNTP error for {1}'.format( msg.get('message-id', 'n/a'), mlist.fqdn_listname)) except socket.error: @@ -111,9 +108,9 @@ def prepare_message(mlist, msg, msgdata): del msg['approved'] msg['Approved'] = mlist.posting_address # Should we restore the original, non-prefixed subject for gatewayed - # messages? TK: We use stripped_subject (prefix stripped) which was - # crafted in CookHeaders.py to ensure prefix was stripped from the subject - # came from mailing list user. + # messages? TK: We use stripped_subject (prefix stripped) which was crafted + # in the subject-prefix handler to ensure prefix was stripped from the + # subject came from mailing list user. stripped_subject = msgdata.get('stripped_subject', msgdata.get('original_subject')) if not mlist.nntp_prefix_subject_too and stripped_subject is not None: diff --git a/src/mailman/runners/outgoing.py b/src/mailman/runners/outgoing.py index db0d847c4..9af4e7c11 100644 --- a/src/mailman/runners/outgoing.py +++ b/src/mailman/runners/outgoing.py @@ -17,14 +17,16 @@ """Outgoing runner.""" +__all__ = [ + 'OutgoingRunner', + ] + + import socket import logging from datetime import datetime from lazr.config import as_boolean, as_timedelta -from uuid import UUID -from zope.component import getUtility - from mailman.config import config from mailman.core.runner import Runner from mailman.interfaces.bounce import BounceContext, IBounceProcessor @@ -34,6 +36,8 @@ from mailman.interfaces.pending import IPendings from mailman.interfaces.subscriptions import ISubscriptionService from mailman.utilities.datetime import now from mailman.utilities.modules import find_name +from uuid import UUID +from zope.component import getUtility # This controls how often _do_periodic() will try to deal with deferred diff --git a/src/mailman/runners/pipeline.py b/src/mailman/runners/pipeline.py index 13226c6fc..357863d2e 100644 --- a/src/mailman/runners/pipeline.py +++ b/src/mailman/runners/pipeline.py @@ -22,6 +22,11 @@ through the 'preparation pipeline'. This pipeline adds, deletes and modifies headers, calculates message recipients, and more. """ +__all__ = [ + 'PipelineRunner', + ] + + from mailman.core.pipelines import process from mailman.core.runner import Runner diff --git a/src/mailman/runners/rest.py b/src/mailman/runners/rest.py index 5980e6263..d39a8a6ff 100644 --- a/src/mailman/runners/rest.py +++ b/src/mailman/runners/rest.py @@ -17,9 +17,6 @@ """Start the administrative HTTP server.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'RESTRunner', ] diff --git a/src/mailman/runners/retry.py b/src/mailman/runners/retry.py index b4148ee3a..f4705ba75 100644 --- a/src/mailman/runners/retry.py +++ b/src/mailman/runners/retry.py @@ -17,9 +17,6 @@ """Retry delivery.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'RetryRunner', ] diff --git a/src/mailman/runners/tests/test_archiver.py b/src/mailman/runners/tests/test_archiver.py index e11b6c805..9e3d9626c 100644 --- a/src/mailman/runners/tests/test_archiver.py +++ b/src/mailman/runners/tests/test_archiver.py @@ -17,9 +17,6 @@ """Test the archive runner.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestArchiveRunner', ] @@ -29,19 +26,17 @@ import os import unittest from email import message_from_file -from zope.interface import implementer - from mailman.app.lifecycle import create_list from mailman.config import config from mailman.interfaces.archiver import IArchiver from mailman.interfaces.mailinglist import IListArchiverSet from mailman.runners.archive import ArchiveRunner from mailman.testing.helpers import ( - configuration, - make_testable_runner, + configuration, make_testable_runner, specialized_message_from_string as mfs) from mailman.testing.layers import ConfigLayer from mailman.utilities.datetime import RFC822_DATE_FMT, factory, now +from zope.interface import implementer @@ -110,7 +105,7 @@ First post! # Ensure that the archive runner ends up archiving the message. self._archiveq.enqueue( self._msg, {}, - listname=self._mlist.fqdn_listname, + listid=self._mlist.list_id, received_time=now()) self._runner.run() # There should now be a copy of the message in the file system. @@ -126,7 +121,7 @@ First post! self._msg['Date'] = now(strip_tzinfo=False).strftime(RFC822_DATE_FMT) self._archiveq.enqueue( self._msg, {}, - listname=self._mlist.fqdn_listname, + listid=self._mlist.list_id, received_time=now()) self._runner.run() # There should now be a copy of the message in the file system. @@ -144,7 +139,7 @@ First post! self._msg['Date'] = now(strip_tzinfo=False).strftime(RFC822_DATE_FMT) self._archiveq.enqueue( self._msg, {}, - listname=self._mlist.fqdn_listname, + listid=self._mlist.list_id, received_time=now()) self._runner.run() # There should now be a copy of the message in the file system. @@ -163,7 +158,7 @@ First post! # again), fast forward a few days. self._archiveq.enqueue( self._msg, {}, - listname=self._mlist.fqdn_listname, + listid=self._mlist.list_id, received_time=now(strip_tzinfo=False)) self._runner.run() # There should now be a copy of the message in the file system. @@ -182,7 +177,7 @@ First post! # again as will happen in the runner), fast forward a few days. self._archiveq.enqueue( self._msg, {}, - listname=self._mlist.fqdn_listname) + listid=self._mlist.list_id) factory.fast_forward(days=4) self._runner.run() # There should now be a copy of the message in the file system. @@ -205,7 +200,7 @@ First post! # again as will happen in the runner), fast forward a few days. self._archiveq.enqueue( self._msg, {}, - listname=self._mlist.fqdn_listname) + listid=self._mlist.list_id) factory.fast_forward(days=4) self._runner.run() # There should now be a copy of the message in the file system. @@ -228,7 +223,7 @@ First post! # again as will happen in the runner), fast forward a few days. self._archiveq.enqueue( self._msg, {}, - listname=self._mlist.fqdn_listname) + listid=self._mlist.list_id) factory.fast_forward(days=4) self._runner.run() # There should now be a copy of the message in the file system. @@ -249,6 +244,6 @@ First post! config.db.store.commit() self._archiveq.enqueue( self._msg, {}, - listname=self._mlist.fqdn_listname) + listid=self._mlist.list_id) self._runner.run() self.assertEqual(os.listdir(config.MESSAGES_DIR), []) diff --git a/src/mailman/runners/tests/test_bounce.py b/src/mailman/runners/tests/test_bounce.py index 315a81c22..875437dc2 100644 --- a/src/mailman/runners/tests/test_bounce.py +++ b/src/mailman/runners/tests/test_bounce.py @@ -17,9 +17,6 @@ """Test the bounce runner.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestBounceRunner', 'TestBounceRunnerBug876774', @@ -29,9 +26,6 @@ __all__ = [ import unittest -from zope.component import getUtility -from zope.interface import implementer - from mailman.app.bounces import send_probe from mailman.app.lifecycle import create_list from mailman.config import config @@ -42,11 +36,11 @@ from mailman.interfaces.styles import IStyle, IStyleManager from mailman.interfaces.usermanager import IUserManager from mailman.runners.bounce import BounceRunner from mailman.testing.helpers import ( - LogFileMark, - get_queue_messages, - make_testable_runner, + LogFileMark, get_queue_messages, make_testable_runner, specialized_message_from_string as message_from_string) from mailman.testing.layers import ConfigLayer +from zope.component import getUtility +from zope.interface import implementer @@ -69,7 +63,7 @@ To: test-bounces+anne=example.com@example.com Message-Id: <first> """) - self._msgdata = dict(listname='test@example.com') + self._msgdata = dict(listid='test.example.com') self._processor = getUtility(IBounceProcessor) config.push('site owner', """ [mailman] @@ -284,7 +278,7 @@ To: test-bounces+anne=example.com@example.com Message-Id: <first> """) - self._bounceq.enqueue(bounce, dict(listname='test@example.com')) + self._bounceq.enqueue(bounce, dict(listid='test.example.com')) self.assertEqual(len(self._bounceq.files), 1) self._runner.run() self.assertEqual(len(get_queue_messages('bounces')), 0) diff --git a/src/mailman/runners/tests/test_confirm.py b/src/mailman/runners/tests/test_confirm.py index 40fae368f..11514044a 100644 --- a/src/mailman/runners/tests/test_confirm.py +++ b/src/mailman/runners/tests/test_confirm.py @@ -17,9 +17,6 @@ """Test the `confirm` command.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestConfirm', ] @@ -29,8 +26,6 @@ import unittest from datetime import datetime from email.iterators import body_line_iterator -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.config import config from mailman.database.transaction import transaction @@ -38,10 +33,10 @@ from mailman.interfaces.registrar import IRegistrar from mailman.interfaces.usermanager import IUserManager from mailman.runners.command import CommandRunner from mailman.testing.helpers import ( - get_queue_messages, - make_testable_runner, + get_queue_messages, make_testable_runner, specialized_message_from_string as mfs) from mailman.testing.layers import ConfigLayer +from zope.component import getUtility @@ -68,7 +63,7 @@ To: test-confirm@example.com """) msg['Subject'] = subject - self._commandq.enqueue(msg, dict(listname='test@example.com')) + self._commandq.enqueue(msg, dict(listid='test.example.com')) self._runner.run() # Anne is now a confirmed member so her user record and email address # should exist in the database. @@ -88,7 +83,7 @@ To: test-confirm@example.com """) msg['Subject'] = subject - self._commandq.enqueue(msg, dict(listname='test@example.com')) + self._commandq.enqueue(msg, dict(listid='test.example.com')) self._runner.run() # Anne is now a confirmed member so her user record and email address # should exist in the database. @@ -144,7 +139,7 @@ Franziskanerstra=C3=9Fe """) msg['Subject'] = subject msg['To'] = to - self._commandq.enqueue(msg, dict(listname='test@example.com')) + self._commandq.enqueue(msg, dict(listid='test.example.com')) self._runner.run() # Anne is now a confirmed member so her user record and email address # should exist in the database. @@ -177,7 +172,7 @@ Franziskanerstra=C3=9Fe """) msg['Subject'] = subject msg['To'] = to - self._commandq.enqueue(msg, dict(listname='test@example.com')) + self._commandq.enqueue(msg, dict(listid='test.example.com')) self._runner.run() # Anne is now a confirmed member so her user record and email address # should exist in the database. @@ -208,7 +203,7 @@ From: Anne Person <anne@example.org> """) msg['Subject'] = subject msg['To'] = to - self._commandq.enqueue(msg, dict(listname='test@example.com', + self._commandq.enqueue(msg, dict(listid='test.example.com', subaddress='confirm')) self._runner.run() # Anne is now a confirmed member so her user record and email address @@ -223,7 +218,7 @@ From: Anne Person <anne@example.org> # one 'Confirmation email' line. confirmation_lines = [] in_results = False - for line in body_line_iterator(messages[0].msg, decode=True): + for line in body_line_iterator(messages[0].msg): line = line.strip() if in_results: if line.startswith('- Done'): @@ -253,7 +248,7 @@ From: Anne Person <anne@example.org> """) msg['Subject'] = subject msg['To'] = to - self._commandq.enqueue(msg, dict(listname='test@example.com', + self._commandq.enqueue(msg, dict(listid='test.example.com', subaddress='confirm')) self._runner.run() # Now there's a email command notification and a welcome message. All diff --git a/src/mailman/runners/tests/test_digest.py b/src/mailman/runners/tests/test_digest.py index fb1bb7071..83156f04e 100644 --- a/src/mailman/runners/tests/test_digest.py +++ b/src/mailman/runners/tests/test_digest.py @@ -17,26 +17,25 @@ """Test the digest runner.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestDigest', + 'TestI18nDigest', ] import unittest -from StringIO import StringIO from email.iterators import _structure as structure from email.mime.text import MIMEText +from io import StringIO from mailman.app.lifecycle import create_list from mailman.config import config from mailman.email.message import Message from mailman.runners.digest import DigestRunner from mailman.testing.helpers import ( LogFileMark, digest_mbox, get_queue_messages, make_digest_messages, - make_testable_runner, message_from_string) + make_testable_runner, message_from_string, + specialized_message_from_string as mfs) from mailman.testing.layers import ConfigLayer from string import Template @@ -140,3 +139,77 @@ multipart/mixed text/plain text/plain """) + + + +class TestI18nDigest(unittest.TestCase): + layer = ConfigLayer + maxDiff = None + + def setUp(self): + config.push('french', """ + [mailman] + default_language: fr + """) + self.addCleanup(config.pop, 'french') + self._mlist = create_list('test@example.com') + self._mlist.preferred_language = 'fr' + self._mlist.digest_size_threshold = 0 + self._process = config.handlers['to-digest'].process + self._runner = make_testable_runner(DigestRunner) + + def test_multilingual_digest(self): + # When messages come in with a content-type character set different + # than that of the list's preferred language, recipients will get an + # internationalized digest. + msg = mfs("""\ +From: aperson@example.org +To: test@example.com +Subject: =?iso-2022-jp?b?GyRCMGxIVhsoQg==?= +MIME-Version: 1.0 +Content-Type: text/plain; charset=iso-2022-jp +Content-Transfer-Encoding: 7bit + +\x1b$B0lHV\x1b(B +""") + self._process(self._mlist, msg, {}) + self._runner.run() + # There are two digests in the virgin queue; one is the MIME digest + # and the other is the RFC 1153 digest. + messages = get_queue_messages('virgin') + self.assertEqual(len(messages), 2) + if messages[0].msg.is_multipart(): + mime, rfc1153 = messages[0].msg, messages[1].msg + else: + rfc1153, mime = messages[0].msg, messages[1].msg + # The MIME version contains a mix of French and Japanese. The digest + # chrome added by Mailman is in French. + self.assertEqual(mime['subject'].encode(), + '=?iso-8859-1?q?Groupe_Test=2C_Vol_1=2C_Parution_1?=') + self.assertEqual(str(mime['subject']), + 'Groupe Test, Vol 1, Parution 1') + # The first subpart contains the iso-8859-1 masthead. + masthead = mime.get_payload(0).get_payload(decode=True).decode( + 'iso-8859-1') + self.assertMultiLineEqual(masthead.splitlines()[0], + 'Envoyez vos messages pour la liste Test à ') + # The second subpart contains the utf-8 table of contents. + self.assertEqual(mime.get_payload(1)['content-description'], + "Today's Topics (1 messages)") + toc = mime.get_payload(1).get_payload(decode=True).decode('utf-8') + self.assertMultiLineEqual(toc.splitlines()[0], 'Thèmes du jour :') + # The third subpart contains the posted message in Japanese. + self.assertEqual(mime.get_payload(2).get_content_type(), + 'message/rfc822') + post = mime.get_payload(2).get_payload(0) + self.assertEqual(post['subject'], '=?iso-2022-jp?b?GyRCMGxIVhsoQg==?=') + # Compare the bytes so that this module doesn't contain string + # literals in multiple incompatible character sets. + self.assertEqual(post.get_payload(decode=True), b'\x1b$B0lHV\x1b(B\n') + # The RFC 1153 digest will have the same subject, but its payload will + # be recast into utf-8. + self.assertEqual(str(rfc1153['subject']), + 'Groupe Test, Vol 1, Parution 1') + self.assertEqual(rfc1153.get_charset(), 'utf-8') + lines = rfc1153.get_payload(decode=True).decode('utf-8').splitlines() + self.assertEqual(lines[0], 'Envoyez vos messages pour la liste Test à ') diff --git a/src/mailman/runners/tests/test_incoming.py b/src/mailman/runners/tests/test_incoming.py index 9830fedb9..77fe2da02 100644 --- a/src/mailman/runners/tests/test_incoming.py +++ b/src/mailman/runners/tests/test_incoming.py @@ -17,9 +17,6 @@ """Test the incoming queue runner.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestIncoming', ] @@ -32,8 +29,7 @@ from mailman.chains.base import TerminalChainBase from mailman.config import config from mailman.runners.incoming import IncomingRunner from mailman.testing.helpers import ( - get_queue_messages, - make_testable_runner, + get_queue_messages, make_testable_runner, specialized_message_from_string as mfs) from mailman.testing.layers import ConfigLayer @@ -76,7 +72,7 @@ To: test@example.com def test_posting(self): # A message posted to the list goes through the posting chain. - msgdata = dict(listname='test@example.com') + msgdata = dict(listid='test.example.com') config.switchboards['in'].enqueue(self._msg, msgdata) self._in.run() messages = get_queue_messages('out') @@ -85,7 +81,7 @@ To: test@example.com def test_owner(self): # A message posted to the list goes through the posting chain. - msgdata = dict(listname='test@example.com', + msgdata = dict(listid='test.example.com', to_owner=True) config.switchboards['in'].enqueue(self._msg, msgdata) self._in.run() diff --git a/src/mailman/runners/tests/test_join.py b/src/mailman/runners/tests/test_join.py index fbea9e661..df24bc06b 100644 --- a/src/mailman/runners/tests/test_join.py +++ b/src/mailman/runners/tests/test_join.py @@ -17,9 +17,6 @@ """Test mailing list joins.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestJoin', 'TestJoinWithDigests', @@ -29,8 +26,6 @@ __all__ = [ import unittest from email.iterators import body_line_iterator -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.config import config from mailman.interfaces.member import DeliveryMode @@ -42,6 +37,7 @@ from mailman.testing.helpers import ( get_queue_messages, make_testable_runner, reset_the_world, specialized_message_from_string as mfs) from mailman.testing.layers import ConfigLayer +from zope.component import getUtility @@ -72,7 +68,7 @@ subscribe # Adding the subaddress to the metadata dictionary mimics what happens # when the above email message is first processed by the lmtp runner. # For convenience, we skip that step in this test. - self._commandq.enqueue(msg, dict(listname='test@example.com', + self._commandq.enqueue(msg, dict(listid='test.example.com', subaddress='join')) self._runner.run() # There will be two messages in the queue. The first one is a reply @@ -87,7 +83,7 @@ subscribe # one 'Confirmation email' line. confirmation_lines = [] in_results = False - for line in body_line_iterator(messages[0].msg, decode=True): + for line in body_line_iterator(messages[0].msg): line = line.strip() if in_results: if line.startswith('- Done'): @@ -112,7 +108,7 @@ To: test-join@example.com Subject: join """) - self._commandq.enqueue(msg, dict(listname='test@example.com')) + self._commandq.enqueue(msg, dict(listid='test.example.com')) self._runner.run() # There will be one message in the queue - a reply to Anne notifying # her of the status of her command email. Because Anne is already @@ -125,7 +121,7 @@ Subject: join # one 'Confirmation email' line. confirmation_lines = [] in_results = False - for line in body_line_iterator(messages[0].msg, decode=True): + for line in body_line_iterator(messages[0].msg): line = line.strip() if in_results: if line.startswith('- Done'): @@ -181,7 +177,7 @@ To: test-request@example.com join """) - self._commandq.enqueue(msg, dict(listname='test@example.com')) + self._commandq.enqueue(msg, dict(listid='test.example.com')) self._runner.run() anne = self._confirm() self.assertEqual(anne.address.email, 'anne@example.org') @@ -195,7 +191,7 @@ To: test-request@example.com join digest=no """) - self._commandq.enqueue(msg, dict(listname='test@example.com')) + self._commandq.enqueue(msg, dict(listid='test.example.com')) self._runner.run() anne = self._confirm() self.assertEqual(anne.address.email, 'anne@example.org') @@ -209,7 +205,7 @@ To: test-request@example.com join digest=mime """) - self._commandq.enqueue(msg, dict(listname='test@example.com')) + self._commandq.enqueue(msg, dict(listid='test.example.com')) self._runner.run() anne = self._confirm() self.assertEqual(anne.address.email, 'anne@example.org') @@ -223,7 +219,7 @@ To: test-request@example.com join digest=plain """) - self._commandq.enqueue(msg, dict(listname='test@example.com')) + self._commandq.enqueue(msg, dict(listid='test.example.com')) self._runner.run() anne = self._confirm() self.assertEqual(anne.address.email, 'anne@example.org') diff --git a/src/mailman/runners/tests/test_lmtp.py b/src/mailman/runners/tests/test_lmtp.py index 26308548c..44b6a0889 100644 --- a/src/mailman/runners/tests/test_lmtp.py +++ b/src/mailman/runners/tests/test_lmtp.py @@ -17,9 +17,6 @@ """Tests for the LMTP server.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestLMTP', ] @@ -30,7 +27,6 @@ import smtplib import unittest from datetime import datetime - from mailman.config import config from mailman.app.lifecycle import create_list from mailman.database.transaction import transaction @@ -67,7 +63,7 @@ Subject: This has no Message-ID header # reasons) self.assertEqual(cm.exception.smtp_code, 550) self.assertEqual(cm.exception.smtp_error, - 'No Message-ID header provided') + b'No Message-ID header provided') def test_message_id_hash_is_added(self): self._lmtp.sendmail('anne@example.com', ['test@example.com'], """\ @@ -118,6 +114,36 @@ Message-ID: <ant> queue_directory = os.path.join(config.QUEUE_DIR, 'lmtp') self.assertFalse(os.path.isdir(queue_directory)) + def test_nonexistent_mailing_list(self): + # Trying to post to a nonexistent mailing list is an error. + with self.assertRaises(smtplib.SMTPDataError) as cm: + self._lmtp.sendmail('anne@example.com', + ['notalist@example.com'], """\ +From: anne.person@example.com +To: notalist@example.com +Subject: An interesting message +Message-ID: <aardvark> + +""") + self.assertEqual(cm.exception.smtp_code, 550) + self.assertEqual(cm.exception.smtp_error, + b'Requested action not taken: mailbox unavailable') + + def test_missing_subaddress(self): + # Trying to send a message to a bogus subaddress is an error. + with self.assertRaises(smtplib.SMTPDataError) as cm: + self._lmtp.sendmail('anne@example.com', + ['test-bogus@example.com'], """\ +From: anne.person@example.com +To: test-bogus@example.com +Subject: An interesting message +Message-ID: <aardvark> + +""") + self.assertEqual(cm.exception.smtp_code, 550) + self.assertEqual(cm.exception.smtp_error, + b'Requested action not taken: mailbox unavailable') + class TestBugs(unittest.TestCase): @@ -142,5 +168,5 @@ Message-ID: <alpha> """) messages = get_queue_messages('in') self.assertEqual(len(messages), 1) - self.assertEqual(messages[0].msgdata['listname'], - 'my-list@example.com') + self.assertEqual(messages[0].msgdata['listid'], + 'my-list.example.com') diff --git a/src/mailman/runners/tests/test_nntp.py b/src/mailman/runners/tests/test_nntp.py index 3570d1a6f..e3218af33 100644 --- a/src/mailman/runners/tests/test_nntp.py +++ b/src/mailman/runners/tests/test_nntp.py @@ -17,9 +17,6 @@ """Test the NNTP runner and related utilities.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestPrepareMessage', 'TestNNTPRunner', @@ -36,10 +33,7 @@ from mailman.config import config from mailman.interfaces.nntp import NewsgroupModeration from mailman.runners import nntp from mailman.testing.helpers import ( - LogFileMark, - configuration, - get_queue_messages, - make_testable_runner, + LogFileMark, configuration, get_queue_messages, make_testable_runner, specialized_message_from_string as mfs) from mailman.testing.layers import ConfigLayer @@ -257,7 +251,7 @@ Testing @mock.patch('nntplib.NNTP') def test_connect(self, class_mock): # Test connection to the NNTP server with default values. - self._nntpq.enqueue(self._msg, {}, listname='test@example.com') + self._nntpq.enqueue(self._msg, {}, listid='test.example.com') self._runner.run() class_mock.assert_called_once_with( '', 119, user='', password='', readermode=True) @@ -267,7 +261,7 @@ Testing @mock.patch('nntplib.NNTP') def test_connect_with_configuration(self, class_mock): # Test connection to the NNTP server with specific values. - self._nntpq.enqueue(self._msg, {}, listname='test@example.com') + self._nntpq.enqueue(self._msg, {}, listid='test.example.com') self._runner.run() class_mock.assert_called_once_with( 'nntp.example.com', 2112, @@ -276,7 +270,7 @@ Testing @mock.patch('nntplib.NNTP') def test_post(self, class_mock): # Test that the message is posted to the NNTP server. - self._nntpq.enqueue(self._msg, {}, listname='test@example.com') + self._nntpq.enqueue(self._msg, {}, listid='test.example.com') self._runner.run() # Get the mocked instance, which was used in the runner. conn_mock = class_mock() @@ -295,7 +289,7 @@ Testing def test_connection_got_quit(self, class_mock): # The NNTP connection gets closed after a successful post. # Test that the message is posted to the NNTP server. - self._nntpq.enqueue(self._msg, {}, listname='test@example.com') + self._nntpq.enqueue(self._msg, {}, listid='test.example.com') self._runner.run() # Get the mocked instance, which was used in the runner. conn_mock = class_mock() @@ -304,18 +298,19 @@ Testing # and make some simple checks that the message is what we expected. conn_mock.quit.assert_called_once_with() - @mock.patch('nntplib.NNTP', side_effect=nntplib.error_temp) + @mock.patch('nntplib.NNTP', side_effect=nntplib.NNTPTemporaryError) def test_connect_with_nntplib_failure(self, class_mock): - self._nntpq.enqueue(self._msg, {}, listname='test@example.com') + self._nntpq.enqueue(self._msg, {}, listid='test.example.com') mark = LogFileMark('mailman.error') self._runner.run() log_message = mark.readline()[:-1] - self.assertTrue(log_message.endswith( - 'NNTP error for test@example.com')) + self.assertTrue( + log_message.endswith('NNTP error for test@example.com'), + log_message) @mock.patch('nntplib.NNTP', side_effect=socket.error) def test_connect_with_socket_failure(self, class_mock): - self._nntpq.enqueue(self._msg, {}, listname='test@example.com') + self._nntpq.enqueue(self._msg, {}, listid='test.example.com') mark = LogFileMark('mailman.error') self._runner.run() log_message = mark.readline()[:-1] @@ -330,7 +325,7 @@ Testing # I.e. stop immediately, since the queue will not be empty. return True runner = make_testable_runner(nntp.NNTPRunner, 'nntp', predicate=once) - self._nntpq.enqueue(self._msg, {}, listname='test@example.com') + self._nntpq.enqueue(self._msg, {}, listid='test.example.com') mark = LogFileMark('mailman.error') runner.run() log_message = mark.readline()[:-1] @@ -338,14 +333,14 @@ Testing 'NNTP unexpected exception for test@example.com')) messages = get_queue_messages('nntp') self.assertEqual(len(messages), 1) - self.assertEqual(messages[0].msgdata['listname'], 'test@example.com') + self.assertEqual(messages[0].msgdata['listid'], 'test.example.com') self.assertEqual(messages[0].msg['subject'], 'A newsgroup posting') - @mock.patch('nntplib.NNTP', side_effect=nntplib.error_temp) + @mock.patch('nntplib.NNTP', side_effect=nntplib.NNTPTemporaryError) def test_connection_never_gets_quit_after_failures(self, class_mock): # The NNTP connection doesn't get closed after a unsuccessful # connection, since there's nothing to close. - self._nntpq.enqueue(self._msg, {}, listname='test@example.com') + self._nntpq.enqueue(self._msg, {}, listid='test.example.com') self._runner.run() # Get the mocked instance, which was used in the runner. Turn off the # exception raising side effect first though! @@ -361,8 +356,8 @@ Testing # The NNTP connection does get closed after a unsuccessful post. # Add a side-effect to the instance mock's .post() method. conn_mock = class_mock() - conn_mock.post.side_effect = nntplib.error_temp - self._nntpq.enqueue(self._msg, {}, listname='test@example.com') + conn_mock.post.side_effect = nntplib.NNTPTemporaryError + self._nntpq.enqueue(self._msg, {}, listid='test.example.com') self._runner.run() # The connection object's post() method was called once with a # file-like object containing the message's bytes. Read those bytes diff --git a/src/mailman/runners/tests/test_outgoing.py b/src/mailman/runners/tests/test_outgoing.py index 62f6776b1..8f51c4ce2 100644 --- a/src/mailman/runners/tests/test_outgoing.py +++ b/src/mailman/runners/tests/test_outgoing.py @@ -17,10 +17,11 @@ """Test the outgoing runner.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ + 'TestOnce', + 'TestSocketError', + 'TestSomeRecipientsFailed', + 'TestVERPSettings', ] @@ -32,8 +33,6 @@ import unittest from contextlib import contextmanager from datetime import datetime, timedelta from lazr.config import as_timedelta -from zope.component import getUtility - from mailman.app.bounces import send_probe from mailman.app.lifecycle import create_list from mailman.config import config @@ -45,12 +44,11 @@ from mailman.interfaces.pending import IPendings from mailman.interfaces.usermanager import IUserManager from mailman.runners.outgoing import OutgoingRunner from mailman.testing.helpers import ( - LogFileMark, - get_queue_messages, - make_testable_runner, + LogFileMark, get_queue_messages, make_testable_runner, specialized_message_from_string as message_from_string) from mailman.testing.layers import ConfigLayer, SMTPLayer from mailman.utilities.datetime import factory, now +from zope.component import getUtility @@ -96,7 +94,7 @@ Message-Id: <first> deliver_after = now() + timedelta(days=10) self._msgdata['deliver_after'] = deliver_after self._outq.enqueue(self._msg, self._msgdata, - tolist=True, listname='test@example.com') + tolist=True, listid='test.example.com') self._runner.run() items = get_queue_messages('out') self.assertEqual(len(items), 1) @@ -149,20 +147,20 @@ Message-Id: <first> def test_delivery_callback(self): # Test that the configuration variable calls the appropriate callback. - self._outq.enqueue(self._msg, {}, listname='test@example.com') + self._outq.enqueue(self._msg, {}, listid='test.example.com') self._runner.run() self.assertEqual(captured_mlist, self._mlist) self.assertEqual(captured_msg.as_string(), self._msg.as_string()) # Of course, the message metadata will contain a bunch of keys added # by the processing. We don't really care about the details, so this # test is a good enough stand-in. - self.assertEqual(captured_msgdata['listname'], 'test@example.com') + self.assertEqual(captured_msgdata['listid'], 'test.example.com') def test_verp_in_metadata(self): # Test that if the metadata has a 'verp' key, it is unchanged. marker = 'yepper' msgdata = dict(verp=marker) - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') + self._outq.enqueue(self._msg, msgdata, listid='test.example.com') self._runner.run() self.assertEqual(captured_msgdata['verp'], marker) @@ -171,7 +169,7 @@ Message-Id: <first> # indicates, messages will be VERP'd. msgdata = {} self._mlist.personalize = Personalization.individual - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') + self._outq.enqueue(self._msg, msgdata, listid='test.example.com') with temporary_config('personalize', """ [mta] verp_personalized_deliveries: yes @@ -184,7 +182,7 @@ Message-Id: <first> # indicates, messages will be VERP'd. msgdata = {} self._mlist.personalize = Personalization.full - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') + self._outq.enqueue(self._msg, msgdata, listid='test.example.com') with temporary_config('personalize', """ [mta] verp_personalized_deliveries: yes @@ -197,14 +195,14 @@ Message-Id: <first> # does not indicate, messages will not be VERP'd. msgdata = {} self._mlist.personalize = Personalization.full - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') + self._outq.enqueue(self._msg, msgdata, listid='test.example.com') self._runner.run() self.assertFalse('verp' in captured_msgdata) def test_verp_never(self): # Never VERP when the interval is zero. msgdata = {} - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') + self._outq.enqueue(self._msg, msgdata, listid='test.example.com') with temporary_config('personalize', """ [mta] verp_delivery_interval: 0 @@ -215,7 +213,7 @@ Message-Id: <first> def test_verp_always(self): # Always VERP when the interval is one. msgdata = {} - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') + self._outq.enqueue(self._msg, msgdata, listid='test.example.com') with temporary_config('personalize', """ [mta] verp_delivery_interval: 1 @@ -227,7 +225,7 @@ Message-Id: <first> # VERP every so often, when the post_id matches. self._mlist.post_id = 5 msgdata = {} - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') + self._outq.enqueue(self._msg, msgdata, listid='test.example.com') with temporary_config('personalize', """ [mta] verp_delivery_interval: 5 @@ -239,7 +237,7 @@ Message-Id: <first> # VERP every so often, when the post_id matches. self._mlist.post_id = 4 msgdata = {} - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') + self._outq.enqueue(self._msg, msgdata, listid='test.example.com') with temporary_config('personalize', """ [mta] verp_delivery_interval: 5 @@ -287,7 +285,7 @@ Message-Id: <first> error_log = logging.getLogger('mailman.error') filename = error_log.handlers[0].filename filepos = os.stat(filename).st_size - self._outq.enqueue(self._msg, {}, listname='test@example.com') + self._outq.enqueue(self._msg, {}, listid='test.example.com') with temporary_config('port 0', """ [mta] smtp_port: 0 @@ -308,7 +306,7 @@ Message-Id: <first> # that is a log message. Start by opening the error log and reading # the current file position. mark = LogFileMark('mailman.error') - self._outq.enqueue(self._msg, {}, listname='test@example.com') + self._outq.enqueue(self._msg, {}, listid='test.example.com') with temporary_config('port 0', """ [mta] smtp_port: 2112 @@ -369,7 +367,7 @@ Message-Id: <first> token = send_probe(member, self._msg) msgdata = dict(probe_token=token) permanent_failures.append('anne@example.com') - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') + self._outq.enqueue(self._msg, msgdata, listid='test.example.com') self._runner.run() events = list(self._processor.unprocessed) self.assertEqual(len(events), 1) @@ -390,7 +388,7 @@ Message-Id: <first> getUtility(IPendings).confirm(token) msgdata = dict(probe_token=token) permanent_failures.append('anne@example.com') - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') + self._outq.enqueue(self._msg, msgdata, listid='test.example.com') self._runner.run() events = list(self._processor.unprocessed) self.assertEqual(len(events), 0) @@ -404,7 +402,7 @@ Message-Id: <first> getUtility(IPendings).confirm(token) msgdata = dict(probe_token=token) temporary_failures.append('anne@example.com') - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') + self._outq.enqueue(self._msg, msgdata, listid='test.example.com') self._runner.run() events = list(self._processor.unprocessed) self.assertEqual(len(events), 0) @@ -412,7 +410,7 @@ Message-Id: <first> def test_one_permanent_failure(self): # Normal (i.e. non-probe) permanent failures just get registered. permanent_failures.append('anne@example.com') - self._outq.enqueue(self._msg, {}, listname='test@example.com') + self._outq.enqueue(self._msg, {}, listid='test.example.com') self._runner.run() events = list(self._processor.unprocessed) self.assertEqual(len(events), 1) @@ -423,7 +421,7 @@ Message-Id: <first> # Two normal (i.e. non-probe) permanent failures just get registered. permanent_failures.append('anne@example.com') permanent_failures.append('bart@example.com') - self._outq.enqueue(self._msg, {}, listname='test@example.com') + self._outq.enqueue(self._msg, {}, listid='test.example.com') self._runner.run() events = list(self._processor.unprocessed) self.assertEqual(len(events), 2) @@ -437,7 +435,7 @@ Message-Id: <first> # put in the retry queue, but with some metadata to prevent infinite # retries. temporary_failures.append('cris@example.com') - self._outq.enqueue(self._msg, {}, listname='test@example.com') + self._outq.enqueue(self._msg, {}, listid='test.example.com') self._runner.run() events = list(self._processor.unprocessed) self.assertEqual(len(events), 0) @@ -458,7 +456,7 @@ Message-Id: <first> # retries. temporary_failures.append('cris@example.com') temporary_failures.append('dave@example.com') - self._outq.enqueue(self._msg, {}, listname='test@example.com') + self._outq.enqueue(self._msg, {}, listid='test.example.com') self._runner.run() events = list(self._processor.unprocessed) self.assertEqual(len(events), 0) @@ -476,7 +474,7 @@ Message-Id: <first> permanent_failures.append('fred@example.com') temporary_failures.append('gwen@example.com') temporary_failures.append('herb@example.com') - self._outq.enqueue(self._msg, {}, listname='test@example.com') + self._outq.enqueue(self._msg, {}, listid='test.example.com') self._runner.run() # Let's look at the permanent failures. events = list(self._processor.unprocessed) @@ -503,7 +501,7 @@ Message-Id: <first> as_timedelta(config.mta.delivery_retry_period)) msgdata = dict(last_recip_count=2, deliver_until=deliver_until) - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') + self._outq.enqueue(self._msg, msgdata, listid='test.example.com') self._runner.run() # The retry queue should have our message waiting to be retried. items = get_queue_messages('retry') @@ -522,7 +520,7 @@ Message-Id: <first> deliver_until = datetime(2005, 8, 1, 7, 49, 23) + retry_period msgdata = dict(last_recip_count=2, deliver_until=deliver_until) - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') + self._outq.enqueue(self._msg, msgdata, listid='test.example.com') # Before the runner runs, several days pass. factory.fast_forward(retry_period.days + 1) mark = LogFileMark('mailman.smtp') diff --git a/src/mailman/runners/tests/test_owner.py b/src/mailman/runners/tests/test_owner.py index 6c68e91cc..15ca07c2e 100644 --- a/src/mailman/runners/tests/test_owner.py +++ b/src/mailman/runners/tests/test_owner.py @@ -22,9 +22,6 @@ # tests. They're not exactly integration tests, but they do touch lots of # parts of the system. -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestEmailToOwner', ] @@ -32,22 +29,19 @@ __all__ = [ import unittest -from operator import itemgetter -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.config import config from mailman.database.transaction import transaction from mailman.interfaces.member import MemberRole from mailman.interfaces.usermanager import IUserManager from mailman.testing.helpers import ( - TestableMaster, - get_lmtp_client, - make_testable_runner) + TestableMaster, get_lmtp_client, make_testable_runner) from mailman.runners.incoming import IncomingRunner from mailman.runners.outgoing import OutgoingRunner from mailman.runners.pipeline import PipelineRunner from mailman.testing.layers import SMTPLayer +from operator import itemgetter +from zope.component import getUtility @@ -89,7 +83,7 @@ class TestEmailToOwner(unittest.TestCase): # get a copy of the message. lmtp = get_lmtp_client(quiet=True) lmtp.lhlo('remote.example.org') - lmtp.sendmail('zuzu@example.org', ['test-owner@example.com'], """\ + lmtp.sendmail('zuzu@example.org', ['test-owner@example.com'], b"""\ From: Zuzu Person <zuzu@example.org> To: test-owner@example.com Message-ID: <ant> diff --git a/src/mailman/runners/tests/test_pipeline.py b/src/mailman/runners/tests/test_pipeline.py index 50ec6cb9a..347bde16b 100644 --- a/src/mailman/runners/tests/test_pipeline.py +++ b/src/mailman/runners/tests/test_pipeline.py @@ -17,9 +17,6 @@ """Test the pipeline runner.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestPipelineRunner', ] @@ -27,17 +24,15 @@ __all__ = [ import unittest -from zope.interface import implementer - from mailman.app.lifecycle import create_list from mailman.config import config from mailman.interfaces.handler import IHandler from mailman.interfaces.pipeline import IPipeline from mailman.runners.pipeline import PipelineRunner from mailman.testing.helpers import ( - make_testable_runner, - specialized_message_from_string as mfs) + make_testable_runner, specialized_message_from_string as mfs) from mailman.testing.layers import ConfigLayer +from zope.interface import implementer @@ -101,7 +96,7 @@ To: test@example.com def test_posting(self): # A message accepted for posting gets processed through the posting # pipeline. - msgdata = dict(listname='test@example.com') + msgdata = dict(listid='test.example.com') config.switchboards['pipeline'].enqueue(self._msg, msgdata) self._pipeline.run() self.assertEqual(len(self._markers), 1) @@ -110,7 +105,7 @@ To: test@example.com def test_owner(self): # A message accepted for posting to a list's owners gets processed # through the owner pipeline. - msgdata = dict(listname='test@example.com', + msgdata = dict(listid='test.example.com', to_owner=True) config.switchboards['pipeline'].enqueue(self._msg, msgdata) self._pipeline.run() diff --git a/src/mailman/runners/tests/test_rest.py b/src/mailman/runners/tests/test_rest.py index bbe026ad6..96ca19089 100644 --- a/src/mailman/runners/tests/test_rest.py +++ b/src/mailman/runners/tests/test_rest.py @@ -17,9 +17,6 @@ """Test the REST runner.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestRESTRunner', ] diff --git a/src/mailman/runners/tests/test_retry.py b/src/mailman/runners/tests/test_retry.py index 28289bc05..0a0929991 100644 --- a/src/mailman/runners/tests/test_retry.py +++ b/src/mailman/runners/tests/test_retry.py @@ -17,9 +17,6 @@ """Test the retry runner.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestRetryRunner', ] @@ -31,8 +28,7 @@ from mailman.app.lifecycle import create_list from mailman.config import config from mailman.runners.retry import RetryRunner from mailman.testing.helpers import ( - get_queue_messages, - make_testable_runner, + get_queue_messages, make_testable_runner, specialized_message_from_string as message_from_string) from mailman.testing.layers import ConfigLayer @@ -54,7 +50,7 @@ To: test@example.com Message-Id: <first> """) - self._msgdata = dict(listname='test@example.com') + self._msgdata = dict(listid='test.example.com') def test_message_put_in_outgoing_queue(self): self._retryq.enqueue(self._msg, self._msgdata) diff --git a/src/mailman/runners/virgin.py b/src/mailman/runners/virgin.py index 0f91d61af..8ff45e86e 100644 --- a/src/mailman/runners/virgin.py +++ b/src/mailman/runners/virgin.py @@ -23,6 +23,11 @@ to go through some minimal processing before they can be sent out to the recipient. """ +__all__ = [ + 'VirginRunner', + ] + + from mailman.core.pipelines import process from mailman.core.runner import Runner diff --git a/src/mailman/styles/base.py b/src/mailman/styles/base.py index 0d65bbebb..db4072b5c 100644 --- a/src/mailman/styles/base.py +++ b/src/mailman/styles/base.py @@ -23,9 +23,6 @@ methods in your compositional derived class. """ -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Announcement', 'BasicOperation', @@ -38,7 +35,6 @@ __all__ = [ from datetime import timedelta - from mailman.core.i18n import _ from mailman.interfaces.action import Action, FilterAction from mailman.interfaces.archiver import ArchivePolicy diff --git a/src/mailman/styles/default.py b/src/mailman/styles/default.py index b12999f0e..f7ea3447f 100644 --- a/src/mailman/styles/default.py +++ b/src/mailman/styles/default.py @@ -17,21 +17,17 @@ """Application of list styles to new and existing lists.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'LegacyDefaultStyle', 'LegacyAnnounceOnly', ] -from zope.interface import implementer - from mailman.interfaces.styles import IStyle from mailman.styles.base import ( Announcement, BasicOperation, Bounces, Discussion, Identity, Moderation, Public) +from zope.interface import implementer diff --git a/src/mailman/styles/manager.py b/src/mailman/styles/manager.py index 397902c17..59cbb1471 100644 --- a/src/mailman/styles/manager.py +++ b/src/mailman/styles/manager.py @@ -17,23 +17,19 @@ """Style manager.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'StyleManager', 'handle_ConfigurationUpdatedEvent', ] -from zope.component import getUtility -from zope.interface import implementer -from zope.interface.verify import verifyObject - from mailman.interfaces.configuration import ConfigurationUpdatedEvent from mailman.interfaces.styles import ( DuplicateStyleError, IStyle, IStyleManager) from mailman.utilities.modules import find_components +from zope.component import getUtility +from zope.interface import implementer +from zope.interface.verify import verifyObject diff --git a/src/mailman/styles/tests/test_styles.py b/src/mailman/styles/tests/test_styles.py index 1fb7a8410..8e8d2eb19 100644 --- a/src/mailman/styles/tests/test_styles.py +++ b/src/mailman/styles/tests/test_styles.py @@ -17,9 +17,6 @@ """Test styles.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestStyle', ] @@ -27,13 +24,12 @@ __all__ = [ import unittest -from zope.component import getUtility -from zope.interface import implementer -from zope.interface.exceptions import DoesNotImplement - from mailman.interfaces.styles import ( DuplicateStyleError, IStyle, IStyleManager) from mailman.testing.layers import ConfigLayer +from zope.component import getUtility +from zope.interface import implementer +from zope.interface.exceptions import DoesNotImplement diff --git a/src/mailman/testing/documentation.py b/src/mailman/testing/documentation.py index b8d852fed..e7511fb9b 100644 --- a/src/mailman/testing/documentation.py +++ b/src/mailman/testing/documentation.py @@ -21,9 +21,6 @@ Note that doctest extraction does not currently work for zip file distributions. doctest discovery currently requires file system traversal. """ -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'setup', 'teardown' @@ -31,7 +28,6 @@ __all__ = [ from inspect import isfunction, ismethod - from mailman.app.lifecycle import create_list from mailman.config import config from mailman.testing.helpers import call_api, specialized_message_from_string @@ -145,11 +141,6 @@ def dump_json(url, data=None, method=None, username=None, password=None): def setup(testobj): """Test setup.""" - # Make sure future statements in our doctests are the same as everywhere - # else. - testobj.globs['absolute_import'] = absolute_import - testobj.globs['print_function'] = print_function - testobj.globs['unicode_literals'] = unicode_literals # In general, I don't like adding convenience functions, since I think # doctests should do the imports themselves. It makes for better # documentation that way. However, a few are really useful, or help to diff --git a/src/mailman/testing/helpers.py b/src/mailman/testing/helpers.py index 38e210b06..b00534490 100644 --- a/src/mailman/testing/helpers.py +++ b/src/mailman/testing/helpers.py @@ -17,9 +17,6 @@ """Various test helpers.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'LogFileMark', 'TestableMaster', @@ -60,11 +57,6 @@ from contextlib import contextmanager from email import message_from_string from httplib2 import Http from lazr.config import as_timedelta -from urllib import urlencode -from urllib2 import HTTPError -from zope import event -from zope.component import getUtility - from mailman.bin.master import Loop as Master from mailman.config import config from mailman.database.transaction import transaction @@ -75,6 +67,10 @@ from mailman.interfaces.styles import IStyleManager from mailman.interfaces.usermanager import IUserManager from mailman.runners.digest import DigestRunner from mailman.utilities.mailbox import Mailbox +from six.moves.urllib_error import HTTPError +from six.moves.urllib_parse import urlencode +from zope import event +from zope.component import getUtility NL = '\n' @@ -335,7 +331,10 @@ def call_api(url, data=None, method=None, username=None, password=None): basic_auth = '{0}:{1}'.format( (config.webservice.admin_user if username is None else username), (config.webservice.admin_pass if password is None else password)) - headers['Authorization'] = 'Basic ' + b64encode(basic_auth) + # b64encode() requires a bytes, but the header value must be str. Do the + # necessary conversion dances. + token = b64encode(basic_auth.encode('utf-8')).decode('ascii') + headers['Authorization'] = 'Basic ' + token response, content = Http().request(url, method, data, headers) # If we did not get a 2xx status code, make this look like a urllib2 # exception, for backward compatibility with existing doctests. @@ -470,10 +469,11 @@ def reset_the_world(): """ # Reset the database between tests. config.db._reset() - # Remove any digest files. + # Remove any digest files and members.txt file (for the file-recips + # handler) in the lists' data directories. for dirpath, dirnames, filenames in os.walk(config.LIST_DATA_DIR): for filename in filenames: - if filename.endswith('.mmdf'): + if filename.endswith('.mmdf') or filename == 'members.txt': os.remove(os.path.join(dirpath, filename)) # Remove all residual queue files. for dirpath, dirnames, filenames in os.walk(config.QUEUE_DIR): @@ -508,9 +508,8 @@ def specialized_message_from_string(unicode_text): """ # This mimic what Switchboard.dequeue() does when parsing a message from # text into a Message instance. - text = unicode_text.encode('ascii') - original_size = len(text) - message = message_from_string(text, Message) + original_size = len(unicode_text) + message = message_from_string(unicode_text, Message) message.original_size = original_size return message diff --git a/src/mailman/testing/i18n.py b/src/mailman/testing/i18n.py index 933a5ec0f..6718f5dda 100644 --- a/src/mailman/testing/i18n.py +++ b/src/mailman/testing/i18n.py @@ -17,9 +17,6 @@ """Internationalization for the tests.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestingStrategy', 'initialize', @@ -29,9 +26,8 @@ __all__ = [ from contextlib import closing from flufl.i18n import registry from gettext import GNUTranslations, NullTranslations -from pkg_resources import resource_stream - from mailman.core.i18n import initialize as core_initialize +from pkg_resources import resource_stream diff --git a/src/mailman/testing/layers.py b/src/mailman/testing/layers.py index 74ad99dc8..d38878160 100644 --- a/src/mailman/testing/layers.py +++ b/src/mailman/testing/layers.py @@ -20,14 +20,10 @@ # XXX 2012-03-23 BAW: Layers really really suck. For example, the # test_owners_get_email() test requires that both the SMTPLayer and LMTPLayer # be set up, but there's apparently no way to do that and make zope.testing -# happy. This causes no tests failures, but it does cause errors at the end -# of the full test run. For now, I'll ignore that, but I do want to -# eventually get rid of the zope.test* dependencies and use something like -# testresources or some such. +# happy. This causes no test failures, but it does cause errors at the end of +# the full test run. For now, I'll ignore that, but I do want to eventually +# get rid of the layers and use something like testresources or some such. -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'ConfigLayer', 'LMTPLayer', @@ -46,10 +42,6 @@ import datetime import tempfile from lazr.config import as_boolean -from pkg_resources import resource_string -from textwrap import dedent -from zope.component import getUtility - from mailman.config import config from mailman.core import initialize from mailman.core.initialize import INHIBIT_CONFIG_FILE @@ -60,6 +52,9 @@ from mailman.testing.helpers import ( TestableMaster, get_lmtp_client, reset_the_world, wait_for_webservice) from mailman.testing.mta import ConnectionCountingController from mailman.utilities.string import expand +from pkg_resources import resource_string as resource_bytes +from textwrap import dedent +from zope.component import getUtility TEST_TIMEOUT = datetime.timedelta(seconds=5) @@ -132,7 +127,8 @@ class ConfigLayer(MockAndMonkeyLayer): configuration: {1} """.format(cls.var_dir, postfix_cfg)) # Read the testing config and push it. - test_config += resource_string('mailman.testing', 'testing.cfg') + more = resource_bytes('mailman.testing', 'testing.cfg') + test_config += more.decode('utf-8') config.create_paths = True config.push('test config', test_config) # Initialize everything else. diff --git a/src/mailman/testing/mta.py b/src/mailman/testing/mta.py index 875647485..81a6bf1ac 100644 --- a/src/mailman/testing/mta.py +++ b/src/mailman/testing/mta.py @@ -17,9 +17,6 @@ """Fake MTA for testing purposes.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'FakeMTA', ] @@ -27,13 +24,11 @@ __all__ = [ import logging -from Queue import Empty, Queue - from lazr.smtptest.controller import QueueController from lazr.smtptest.server import Channel, QueueServer -from zope.interface import implementer - from mailman.interfaces.mta import IMailTransportAgentLifecycle +from six.moves.queue import Empty, Queue +from zope.interface import implementer log = logging.getLogger('lazr.smtptest') @@ -60,28 +55,28 @@ class StatisticsChannel(Channel): def smtp_EHLO(self, arg): if not arg: - self.push(b'501 Syntax: HELO hostname') + self.push('501 Syntax: HELO hostname') return if self._SMTPChannel__greeting: - self.push(b'503 Duplicate HELO/EHLO') + self.push('503 Duplicate HELO/EHLO') else: self._SMTPChannel__greeting = arg - self.push(b'250-%s' % self._SMTPChannel__fqdn) - self.push(b'250 AUTH PLAIN') + self.push('250-%s' % self._SMTPChannel__fqdn) + self.push('250 AUTH PLAIN') def smtp_STAT(self, arg): """Cause the server to send statistics to its controller.""" self._server.send_statistics() - self.push(b'250 Ok') + self.push('250 Ok') def smtp_AUTH(self, arg): """Record that the AUTH occurred.""" if arg == 'PLAIN AHRlc3R1c2VyAHRlc3RwYXNz': # testuser:testpass - self.push(b'235 Ok') + self.push('235 Ok') self._server.send_auth(arg) else: - self.push(b'571 Bad authentication') + self.push('571 Bad authentication') def smtp_RCPT(self, arg): """For testing, sometimes cause a non-25x response.""" @@ -92,7 +87,7 @@ class StatisticsChannel(Channel): else: # The test suite wants this to fail. The message corresponds to # the exception we expect smtplib.SMTP to raise. - self.push(b'%d Error: SMTPRecipientsRefused' % code) + self.push('%d Error: SMTPRecipientsRefused' % code) def smtp_MAIL(self, arg): """For testing, sometimes cause a non-25x response.""" @@ -103,7 +98,7 @@ class StatisticsChannel(Channel): else: # The test suite wants this to fail. The message corresponds to # the exception we expect smtplib.SMTP to raise. - self.push(b'%d Error: SMTPResponseException' % code) + self.push('%d Error: SMTPResponseException' % code) @@ -211,7 +206,7 @@ class ConnectionCountingController(QueueController): :rtype: integer """ smtpd = self._connect() - smtpd.docmd(b'STAT') + smtpd.docmd('STAT') # An Empty exception will occur if the data isn't available in 10 # seconds. Let that propagate. return self.oob_queue.get(block=True, timeout=10) @@ -232,4 +227,4 @@ class ConnectionCountingController(QueueController): def reset(self): smtpd = self._connect() - smtpd.docmd(b'RSET') + smtpd.docmd('RSET') diff --git a/src/mailman/testing/nose.py b/src/mailman/testing/nose.py index 8fe7017c0..181048b64 100644 --- a/src/mailman/testing/nose.py +++ b/src/mailman/testing/nose.py @@ -17,9 +17,6 @@ """nose2 test infrastructure.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'NosePlugin', ] @@ -35,6 +32,7 @@ from mailman.testing.documentation import setup, teardown from mailman.testing.layers import ConfigLayer, MockAndMonkeyLayer, SMTPLayer from nose2.events import Plugin + DOT = '.' FLAGS = doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF TOPDIR = os.path.dirname(mailman.__file__) @@ -116,3 +114,9 @@ class NosePlugin(Plugin): # Suppress the extra "Doctest: ..." line. test.shortDescription = lambda: None event.extraTests.append(test) + + ## def startTest(self, event): + ## import sys; print('vvvvv', event.test, file=sys.stderr) + + ## def stopTest(self, event): + ## import sys; print('^^^^^', event.test, file=sys.stderr) diff --git a/src/mailman/tests/test_configfile.py b/src/mailman/tests/test_configfile.py index 22442c767..0807c0648 100644 --- a/src/mailman/tests/test_configfile.py +++ b/src/mailman/tests/test_configfile.py @@ -17,10 +17,10 @@ """Test configuration file searching.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ + 'TestConfigFileBase', + 'TestConfigFileSearch', + 'TestConfigFileSearchWithChroot', ] @@ -31,7 +31,6 @@ import tempfile import unittest from contextlib import contextmanager - from mailman.core.initialize import search_for_configuration_file @@ -107,6 +106,7 @@ class TestConfigFileBase(unittest.TestCase): return os.path.join(self._root, path) + class TestConfigFileSearch(TestConfigFileBase): """Test various aspects of searching for configuration files. @@ -128,6 +128,7 @@ class TestConfigFileSearch(TestConfigFileBase): self.assertEqual(found, config_file) + class TestConfigFileSearchWithChroot(TestConfigFileBase): """Like `TestConfigFileSearch` but with a special os.path.exists().""" diff --git a/src/mailman/utilities/datetime.py b/src/mailman/utilities/datetime.py index b494e2513..3cea0d0cd 100644 --- a/src/mailman/utilities/datetime.py +++ b/src/mailman/utilities/datetime.py @@ -22,10 +22,6 @@ datetime.datetime.now() and datetime.date.today(). These are better instrumented for testing purposes. """ - -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'DateFactory', 'RFC822_DATE_FMT', diff --git a/src/mailman/utilities/email.py b/src/mailman/utilities/email.py index ea44ad0a4..bedbd2ae9 100644 --- a/src/mailman/utilities/email.py +++ b/src/mailman/utilities/email.py @@ -17,9 +17,6 @@ """Email helpers.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'add_message_hash', 'split_email', @@ -70,7 +67,10 @@ def add_message_hash(msg): message_id = message_id[1:-1] else: message_id = message_id.strip() - digest = sha1(message_id).digest() + # Because .digest() returns bytes, b32encode() will return bytes, however + # we need a string for the header value. We know the b32encoded byte + # string must be ascii-only. + digest = sha1(message_id.encode('utf-8')).digest() message_id_hash = b32encode(digest) del msg['x-message-id-hash'] - msg['X-Message-ID-Hash'] = message_id_hash + msg['X-Message-ID-Hash'] = message_id_hash.decode('ascii') diff --git a/src/mailman/utilities/filesystem.py b/src/mailman/utilities/filesystem.py index f2a5b705b..4ef52cbfa 100644 --- a/src/mailman/utilities/filesystem.py +++ b/src/mailman/utilities/filesystem.py @@ -17,9 +17,6 @@ """Filesystem utilities.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'makedirs', 'umask', diff --git a/src/mailman/utilities/i18n.py b/src/mailman/utilities/i18n.py index e22bd6c18..16f2fee6b 100644 --- a/src/mailman/utilities/i18n.py +++ b/src/mailman/utilities/i18n.py @@ -17,9 +17,6 @@ """i18n template search and interpolation.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TemplateNotFoundError', 'find', @@ -29,17 +26,17 @@ __all__ = [ import os +import six import sys import errno from itertools import product -from pkg_resources import resource_filename - from mailman.config import config from mailman.core.constants import system_preferences from mailman.core.errors import MailmanException from mailman.core.i18n import _ from mailman.utilities.string import expand, wrap as wrap_text +from pkg_resources import resource_filename @@ -203,7 +200,8 @@ def make(template_file, mlist=None, language=None, wrap=True, template = _(fp.read()[:-1]) finally: fp.close() - assert isinstance(template, unicode), 'Translated template is not unicode' + assert isinstance(template, six.text_type), ( + 'Translated template is not unicode') text = expand(template, kw) if wrap: return wrap_text(text) diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py index cc8a0cf44..2db5f3ace 100644 --- a/src/mailman/utilities/importer.py +++ b/src/mailman/utilities/importer.py @@ -17,9 +17,6 @@ """Importer routines.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Import21Error', 'import_config_pck', @@ -48,7 +45,7 @@ from mailman.interfaces.nntp import NewsgroupModeration from mailman.interfaces.usermanager import IUserManager from mailman.utilities.filesystem import makedirs from mailman.utilities.i18n import search -from urllib2 import URLError +from six.moves.urllib_error import URLError from zope.component import getUtility @@ -58,7 +55,7 @@ class Import21Error(MailmanError): -def str_to_unicode(value): +def bytes_to_str(value): # Convert a string to unicode when the encoding is not declared. if not isinstance(value, bytes): return value @@ -71,8 +68,10 @@ def str_to_unicode(value): return value.decode('ascii', 'replace') -def unicode_to_string(value): - return None if value is None else str(value) +def str_to_bytes(value): + if value is None or isinstance(value, bytes): + return value + return value.encode('utf-8') def seconds_to_delta(value): @@ -84,7 +83,7 @@ def days_to_delta(value): def list_members_to_unicode(value): - return [str_to_unicode(item) for item in value] + return [bytes_to_str(item) for item in value] @@ -132,7 +131,7 @@ def nonmember_action_mapping(value): def check_language_code(code): if code is None: return None - code = str_to_unicode(code) + code = bytes_to_str(code) if code not in getUtility(ILanguageManager): msg = """Missing language: {0} You must add a section describing this language to your mailman.cfg file. @@ -170,7 +169,7 @@ TYPES = dict( forward_unrecognized_bounces_to=UnrecognizedBounceDisposition, gateway_to_mail=bool, include_rfc2369_headers=bool, - moderator_password=unicode_to_string, + moderator_password=str_to_bytes, newsgroup_moderation=NewsgroupModeration, nntp_prefix_subject_too=bool, pass_extensions=list_members_to_unicode, @@ -213,8 +212,10 @@ DATETIME_COLUMNS = [ ] EXCLUDES = set(( + 'delivery_status', 'digest_members', 'members', + 'user_options', )) @@ -243,9 +244,9 @@ def import_config_pck(mlist, config_dict): # If the mailing list has a preferred language that isn't registered # in the configuration file, hasattr() will swallow the KeyError this # raises and return False. Treat that attribute specially. - if hasattr(mlist, key) or key == 'preferred_language': - if isinstance(value, str): - value = str_to_unicode(value) + if key == 'preferred_language' or hasattr(mlist, key): + if isinstance(value, bytes): + value = bytes_to_str(value) # Some types require conversion. converter = TYPES.get(key) try: @@ -279,17 +280,19 @@ def import_config_pck(mlist, config_dict): # Handle ban list. ban_manager = IBanManager(mlist) for address in config_dict.get('ban_list', []): - ban_manager.ban(str_to_unicode(address)) + ban_manager.ban(bytes_to_str(address)) # Handle acceptable aliases. acceptable_aliases = config_dict.get('acceptable_aliases', '') - if isinstance(acceptable_aliases, basestring): + if isinstance(acceptable_aliases, bytes): + acceptable_aliases = acceptable_aliases.decode('utf-8') + if isinstance(acceptable_aliases, str): acceptable_aliases = acceptable_aliases.splitlines() alias_set = IAcceptableAliasSet(mlist) for address in acceptable_aliases: address = address.strip() if len(address) == 0: continue - address = str_to_unicode(address) + address = bytes_to_str(address) try: alias_set.add(address) except ValueError: @@ -343,7 +346,8 @@ def import_config_pck(mlist, config_dict): if oldvar not in config_dict: continue text = config_dict[oldvar] - text = text.decode('utf-8', 'replace') + if isinstance(text, bytes): + text = text.decode('utf-8', 'replace') for oldph, newph in convert_placeholders: text = text.replace(oldph, newph) default_value, default_text = defaults.get(newvar, (None, None)) @@ -380,8 +384,9 @@ def import_config_pck(mlist, config_dict): with codecs.open(filepath, 'w', encoding='utf-8') as fp: fp.write(text) # Import rosters. - members = set(config_dict.get('members', {}).keys() - + config_dict.get('digest_members', {}).keys()) + regulars_set = set(config_dict.get('members', {})) + digesters_set = set(config_dict.get('digest_members', {})) + members = regulars_set.union(digesters_set) import_roster(mlist, config_dict, members, MemberRole.member) import_roster(mlist, config_dict, config_dict.get('owner', []), MemberRole.owner) @@ -407,7 +412,7 @@ def import_roster(mlist, config_dict, members, role): for email in members: # For owners and members, the emails can have a mixed case, so # lowercase them all. - email = str_to_unicode(email).lower() + email = bytes_to_str(email).lower() if roster.get_member(email) is not None: print('{} is already imported with role {}'.format(email, role), file=sys.stderr) @@ -421,7 +426,7 @@ def import_roster(mlist, config_dict, members, role): merged_members.update(config_dict.get('members', {})) merged_members.update(config_dict.get('digest_members', {})) if merged_members.get(email, 0) != 0: - original_email = str_to_unicode(merged_members[email]) + original_email = bytes_to_str(merged_members[email]) else: original_email = email address = usermanager.create_address(original_email) @@ -449,9 +454,9 @@ def import_roster(mlist, config_dict, members, role): # overwritten. if email in config_dict.get('usernames', {}): address.display_name = \ - str_to_unicode(config_dict['usernames'][email]) + bytes_to_str(config_dict['usernames'][email]) user.display_name = \ - str_to_unicode(config_dict['usernames'][email]) + bytes_to_str(config_dict['usernames'][email]) if email in config_dict.get('passwords', {}): user.password = config.password_context.encrypt( config_dict['passwords'][email]) diff --git a/src/mailman/utilities/interact.py b/src/mailman/utilities/interact.py index 8bca9ee40..c7531f302 100644 --- a/src/mailman/utilities/interact.py +++ b/src/mailman/utilities/interact.py @@ -17,9 +17,6 @@ """Provide an interactive prompt, mimicking the Python interpreter.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'interact', ] @@ -78,9 +75,3 @@ Type "help", "copyright", "credits" or "license" for more information.''' % ( elif not banner: banner = None interp.interact(banner) - # When an exception occurs in the InteractiveConsole, the various - # sys.exc_* attributes get set so that error handling works the same way - # there as it does in the built-in interpreter. Be anal about clearing - # any exception information before we're done. - sys.exc_clear() - sys.last_type = sys.last_value = sys.last_traceback = None diff --git a/src/mailman/utilities/mailbox.py b/src/mailman/utilities/mailbox.py index 4f085e127..71e083792 100644 --- a/src/mailman/utilities/mailbox.py +++ b/src/mailman/utilities/mailbox.py @@ -15,11 +15,8 @@ # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see <http://www.gnu.org/licenses/>. -"""Module stuff.""" +"""MMDF helper for digests.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'Mailbox', ] diff --git a/src/mailman/utilities/modules.py b/src/mailman/utilities/modules.py index 9ff0e50cd..2a63ac501 100644 --- a/src/mailman/utilities/modules.py +++ b/src/mailman/utilities/modules.py @@ -17,9 +17,6 @@ """Package and module utilities.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'call_name', 'expand_path', diff --git a/src/mailman/utilities/passwords.py b/src/mailman/utilities/passwords.py index 6fb7f08c0..f29482572 100644 --- a/src/mailman/utilities/passwords.py +++ b/src/mailman/utilities/passwords.py @@ -17,19 +17,14 @@ """A wrapper around passlib.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'handle_ConfigurationUpdatedEvent', ] - -from passlib.context import CryptContext - from mailman.config.config import load_external from mailman.interfaces.configuration import ConfigurationUpdatedEvent +from passlib.context import CryptContext diff --git a/src/mailman/utilities/string.py b/src/mailman/utilities/string.py index d6f0da286..6bbf3c6ea 100644 --- a/src/mailman/utilities/string.py +++ b/src/mailman/utilities/string.py @@ -17,9 +17,6 @@ """String utilities.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'expand', 'oneline', @@ -73,9 +70,8 @@ def oneline(s, cset='us-ascii', in_unicode=False): :rtype: string """ try: - h = make_header(decode_header(s)) - ustr = h.__unicode__() - line = EMPTYSTRING.join(ustr.splitlines()) + h = str(make_header(decode_header(s))) + line = EMPTYSTRING.join(h.splitlines()) if in_unicode: return line else: diff --git a/src/mailman/utilities/tests/test_email.py b/src/mailman/utilities/tests/test_email.py index 1448fb32b..838d50862 100644 --- a/src/mailman/utilities/tests/test_email.py +++ b/src/mailman/utilities/tests/test_email.py @@ -17,9 +17,6 @@ """Testing functions in the email utilities.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestEmail', ] diff --git a/src/mailman/utilities/tests/test_import.py b/src/mailman/utilities/tests/test_import.py index 42608ae45..192e08df5 100644 --- a/src/mailman/utilities/tests/test_import.py +++ b/src/mailman/utilities/tests/test_import.py @@ -17,26 +17,24 @@ """Tests for config.pck imports.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestArchiveImport', 'TestBasicImport', + 'TestConvertToURI', + 'TestFilterActionImport', + 'TestMemberActionImport', + 'TestPreferencesImport', + 'TestRosterImport', ] import os +import six import mock -import cPickle import unittest from datetime import timedelta, datetime from enum import Enum -from pkg_resources import resource_filename -from sqlalchemy.exc import IntegrityError -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.config import config from mailman.handlers.decorate import decorate @@ -55,6 +53,9 @@ from mailman.testing.layers import ConfigLayer from mailman.utilities.filesystem import makedirs from mailman.utilities.importer import import_config_pck, Import21Error from mailman.utilities.string import expand +from pkg_resources import resource_filename +from six.moves.cPickle import load +from zope.component import getUtility @@ -77,8 +78,8 @@ class TestBasicImport(unittest.TestCase): def setUp(self): self._mlist = create_list('blank@example.com') pickle_file = resource_filename('mailman.testing', 'config.pck') - with open(pickle_file) as fp: - self._pckdict = cPickle.load(fp) + with open(pickle_file, 'rb') as fp: + self._pckdict = load(fp) def _import(self): import_config_pck(self._mlist, self._pckdict) @@ -180,15 +181,15 @@ class TestBasicImport(unittest.TestCase): def test_moderator_password(self): # mod_password -> moderator_password - self._mlist.moderator_password = str('TESTDATA') + self._mlist.moderator_password = b'TESTDATA' self._import() self.assertEqual(self._mlist.moderator_password, None) def test_moderator_password_str(self): # moderator_password must not be unicode - self._pckdict[b'mod_password'] = b'TESTVALUE' + self._pckdict['mod_password'] = b'TESTVALUE' self._import() - self.assertFalse(isinstance(self._mlist.moderator_password, unicode)) + self.assertNotIsInstance(self._mlist.moderator_password, six.text_type) self.assertEqual(self._mlist.moderator_password, b'TESTVALUE') def test_newsgroup_moderation(self): @@ -227,7 +228,7 @@ class TestBasicImport(unittest.TestCase): 'alias2@exemple.com', 'non-ascii-\xe8@example.com', ] - self._pckdict[b'acceptable_aliases'] = list_to_string(aliases) + self._pckdict['acceptable_aliases'] = list_to_string(aliases) self._import() alias_set = IAcceptableAliasSet(self._mlist) self.assertEqual(sorted(alias_set.aliases), aliases) @@ -236,7 +237,7 @@ class TestBasicImport(unittest.TestCase): # Values without an '@' sign used to be matched against the local # part, now we need to add the '^' sign to indicate it's a regexp. aliases = ['invalid-value'] - self._pckdict[b'acceptable_aliases'] = list_to_string(aliases) + self._pckdict['acceptable_aliases'] = list_to_string(aliases) self._import() alias_set = IAcceptableAliasSet(self._mlist) self.assertEqual(sorted(alias_set.aliases), @@ -246,29 +247,31 @@ class TestBasicImport(unittest.TestCase): # In some versions of the pickle, this can be a list, not a string # (seen in the wild). aliases = [b'alias1@example.com', b'alias2@exemple.com' ] - self._pckdict[b'acceptable_aliases'] = aliases + self._pckdict['acceptable_aliases'] = aliases self._import() alias_set = IAcceptableAliasSet(self._mlist) - self.assertEqual(sorted(alias_set.aliases), aliases) + self.assertEqual(sorted(alias_set.aliases), + sorted(a.decode('utf-8') for a in aliases)) def test_info_non_ascii(self): # info can contain non-ascii characters. info = 'O idioma aceito \xe9 somente Portugu\xeas do Brasil' - self._pckdict[b'info'] = info.encode('utf-8') + self._pckdict['info'] = info.encode('utf-8') self._import() self.assertEqual(self._mlist.info, info, 'Encoding to UTF-8 is not handled') # Test fallback to ascii with replace. - self._pckdict[b'info'] = info.encode('iso-8859-1') + self._pckdict['info'] = info.encode('iso-8859-1') # Suppress warning messages in test output. with mock.patch('sys.stderr'): self._import() - self.assertEqual(self._mlist.info, - unicode(self._pckdict[b'info'], 'ascii', 'replace'), - "We don't fall back to replacing non-ascii chars") + self.assertEqual( + self._mlist.info, + self._pckdict['info'].decode('ascii', 'replace'), + "We don't fall back to replacing non-ascii chars") def test_preferred_language(self): - self._pckdict[b'preferred_language'] = b'ja' + self._pckdict['preferred_language'] = b'ja' english = getUtility(ILanguageManager).get('en') japanese = getUtility(ILanguageManager).get('ja') self.assertEqual(self._mlist.preferred_language, english) @@ -283,7 +286,7 @@ class TestBasicImport(unittest.TestCase): self.assertEqual(self._mlist.preferred_language, english) def test_new_language(self): - self._pckdict[b'preferred_language'] = b'xx_XX' + self._pckdict['preferred_language'] = b'xx_XX' try: self._import() except Import21Error as error: @@ -409,35 +412,35 @@ class TestMemberActionImport(unittest.TestCase): # Suppress warning messages in the test output. with mock.patch('sys.stderr'): import_config_pck(self._mlist, self._pckdict) - for key, value in expected.iteritems(): + for key, value in expected.items(): self.assertEqual(getattr(self._mlist, key), value) def test_member_hold(self): - self._pckdict[b'member_moderation_action'] = 0 + self._pckdict['member_moderation_action'] = 0 self._do_test(dict(default_member_action=Action.hold)) def test_member_reject(self): - self._pckdict[b'member_moderation_action'] = 1 + self._pckdict['member_moderation_action'] = 1 self._do_test(dict(default_member_action=Action.reject)) def test_member_discard(self): - self._pckdict[b'member_moderation_action'] = 2 + self._pckdict['member_moderation_action'] = 2 self._do_test(dict(default_member_action=Action.discard)) def test_nonmember_accept(self): - self._pckdict[b'generic_nonmember_action'] = 0 + self._pckdict['generic_nonmember_action'] = 0 self._do_test(dict(default_nonmember_action=Action.accept)) def test_nonmember_hold(self): - self._pckdict[b'generic_nonmember_action'] = 1 + self._pckdict['generic_nonmember_action'] = 1 self._do_test(dict(default_nonmember_action=Action.hold)) def test_nonmember_reject(self): - self._pckdict[b'generic_nonmember_action'] = 2 + self._pckdict['generic_nonmember_action'] = 2 self._do_test(dict(default_nonmember_action=Action.reject)) def test_nonmember_discard(self): - self._pckdict[b'generic_nonmember_action'] = 3 + self._pckdict['generic_nonmember_action'] = 3 self._do_test(dict(default_nonmember_action=Action.discard)) @@ -524,9 +527,9 @@ class TestConvertToURI(unittest.TestCase): # if it changed from the default so don't import. We may do more harm # than good and it's easy to change if needed. test_value = b'TEST-VALUE' - for oldvar, newvar in self._conf_mapping.iteritems(): + for oldvar, newvar in self._conf_mapping.items(): self._mlist.mail_host = 'example.com' - self._pckdict[b'mail_host'] = b'test.example.com' + self._pckdict['mail_host'] = b'test.example.com' self._pckdict[str(oldvar)] = test_value old_value = getattr(self._mlist, newvar) # Suppress warning messages in the test output. @@ -541,7 +544,7 @@ class TestConvertToURI(unittest.TestCase): for oldvar in self._conf_mapping: self._pckdict[str(oldvar)] = b'Ol\xe1!' import_config_pck(self._mlist, self._pckdict) - for oldvar, newvar in self._conf_mapping.iteritems(): + for oldvar, newvar in self._conf_mapping.items(): newattr = getattr(self._mlist, newvar) text = decorate(self._mlist, newattr) expected = u'Ol\ufffd!' @@ -557,7 +560,7 @@ class TestConvertToURI(unittest.TestCase): makedirs(os.path.dirname(footer_path)) with open(footer_path, 'wb') as fp: fp.write(footer) - self._pckdict[b'msg_footer'] = b'NEW-VALUE' + self._pckdict['msg_footer'] = b'NEW-VALUE' import_config_pck(self._mlist, self._pckdict) text = decorate(self._mlist, self._mlist.footer_uri) self.assertEqual(text, 'NEW-VALUE') @@ -609,6 +612,8 @@ class TestRosterImport(unittest.TestCase): self._usermanager = getUtility(IUserManager) language_manager = getUtility(ILanguageManager) for code in self._pckdict['language'].values(): + if isinstance(code, bytes): + code = code.decode('utf-8') if code not in language_manager.codes: language_manager.add(code, 'utf-8', code) @@ -641,11 +646,13 @@ class TestRosterImport(unittest.TestCase): addr = '%s@example.com' % name member = self._mlist.members.get_member(addr) self.assertIsNotNone(member, 'Address %s was not imported' % addr) - self.assertEqual(member.preferred_language.code, - self._pckdict['language'][addr]) + code = self._pckdict['language'][addr] + if isinstance(code, bytes): + code = code.decode('utf-8') + self.assertEqual(member.preferred_language.code, code) def test_new_language(self): - self._pckdict[b'language']['anne@example.com'] = b'xx_XX' + self._pckdict['language']['anne@example.com'] = b'xx_XX' try: import_config_pck(self._mlist, self._pckdict) except Import21Error as error: @@ -698,7 +705,7 @@ class TestRosterImport(unittest.TestCase): user = self._usermanager.get_user(addr) self.assertIsNotNone(user, 'Address %s was not imported' % addr) self.assertEqual( - user.password, b'{plaintext}%spass' % name, + user.password, '{plaintext}%spass' % name, 'Password for %s was not imported' % addr) def test_same_user(self): @@ -765,7 +772,7 @@ class TestPreferencesImport(unittest.TestCase): self.assertIsNotNone(user, 'User was not imported') member = self._mlist.members.get_member('anne@example.com') self.assertIsNotNone(member, 'Address was not subscribed') - for exp_name, exp_val in expected.iteritems(): + for exp_name, exp_val in expected.items(): try: currentval = getattr(member, exp_name) except AttributeError: @@ -831,8 +838,10 @@ class TestPreferencesImport(unittest.TestCase): def test_multiple_options(self): # DontReceiveDuplicates & DisableMime & SuppressPasswordReminder - self._pckdict[b'digest_members'] = self._pckdict[b'members'].copy() - self._pckdict[b'members'] = dict() + # Keys might be Python 2 str/bytes or unicode. + members = self._pckdict['members'] + self._pckdict['digest_members'] = members.copy() + self._pckdict['members'] = dict() self._do_test(296, dict( receive_list_copy=False, delivery_mode=DeliveryMode.plaintext_digests, diff --git a/src/mailman/utilities/tests/test_passwords.py b/src/mailman/utilities/tests/test_passwords.py index 0dd49cb85..b11a7654b 100644 --- a/src/mailman/utilities/tests/test_passwords.py +++ b/src/mailman/utilities/tests/test_passwords.py @@ -17,9 +17,6 @@ """Testing the password utility.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'TestPasswords', ] diff --git a/src/mailman/utilities/tests/test_templates.py b/src/mailman/utilities/tests/test_templates.py index 6507bf8e5..b59d2aa1c 100644 --- a/src/mailman/utilities/tests/test_templates.py +++ b/src/mailman/utilities/tests/test_templates.py @@ -17,10 +17,10 @@ """Testing i18n template search and interpolation.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ + 'TestFind', + 'TestMake', + 'TestSearchOrder', ] @@ -29,14 +29,13 @@ import shutil import tempfile import unittest -from pkg_resources import resource_filename -from zope.component import getUtility - from mailman.app.lifecycle import create_list from mailman.config import config from mailman.interfaces.languages import ILanguageManager from mailman.testing.layers import ConfigLayer from mailman.utilities.i18n import TemplateNotFoundError, find, make, search +from pkg_resources import resource_filename +from zope.component import getUtility @@ -191,14 +190,14 @@ class TestFind(unittest.TestCase): with open(path, 'w') as fp: fp.write(text) self.xxsite = os.path.join( - self.var_dir, 'templates', 'site', 'xx', 'site.txt') + self.var_dir, 'templates', 'site', 'xx', 'site.txt') write('Site template', self.xxsite) - self.xxdomain = os.path.join( - self.var_dir, 'templates', + self.xxdomain = os.path.join( + self.var_dir, 'templates', 'domains', 'example.com', 'xx', 'domain.txt') write('Domain template', self.xxdomain) self.xxlist = os.path.join( - self.var_dir, 'templates', + self.var_dir, 'templates', 'lists', 'test@example.com', 'xx', 'list.txt') write('List template', self.xxlist) diff --git a/src/mailman/utilities/tests/test_wrap.py b/src/mailman/utilities/tests/test_wrap.py index eca6f93be..b9feeed92 100644 --- a/src/mailman/utilities/tests/test_wrap.py +++ b/src/mailman/utilities/tests/test_wrap.py @@ -17,10 +17,8 @@ """Test text wrapping.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ + 'TestWrap', ] diff --git a/src/mailman/utilities/uid.py b/src/mailman/utilities/uid.py index 4fe862868..0b41b63c2 100644 --- a/src/mailman/utilities/uid.py +++ b/src/mailman/utilities/uid.py @@ -21,9 +21,6 @@ Use these functions to create unique ids rather than inlining calls to hashlib and whatnot. These are better instrumented for testing purposes. """ -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ 'UniqueIDFactory', 'factory', @@ -35,7 +32,6 @@ import uuid import errno from flufl.lock import Lock - from mailman.config import config from mailman.model.uid import UID from mailman.testing import layers diff --git a/template.py b/template.py index 5504d10fd..609e0af95 100644 --- a/template.py +++ b/template.py @@ -1,4 +1,4 @@ -# Copyright (C) 2014 by the Free Software Foundation, Inc. +# Copyright (C) 2015 by the Free Software Foundation, Inc. # # This file is part of GNU Mailman. # @@ -17,8 +17,5 @@ """Module stuff.""" -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type __all__ = [ ] @@ -1,16 +1,16 @@ [tox] -envlist = py27 +envlist = py34 recreate = True [testenv] -commands = python -3 -m nose2 -v +commands = python -m nose2 -v #sitepackages = True usedevelop = True # This environment requires you to set up PostgreSQL and create a .cfg file # somewhere outside of the source tree. [testenv:pg] -basepython = python2.7 +basepython = python3.4 commands = python -m nose2 -v usedevelop = True deps = psycopg2 @@ -20,7 +20,7 @@ rcfile = {toxinidir}/coverage.ini rc = --rcfile={[coverage]rcfile} [testenv:coverage] -basepython = python2.7 +basepython = python3.4 commands = coverage run {[coverage]rc} -m nose2 -v coverage combine {[coverage]rc} |