diff options
219 files changed, 3632 insertions, 1684 deletions
@@ -98,16 +98,17 @@ case second `m'. Any other spelling is incorrect.""", 'flufl.enum', 'flufl.i18n', 'flufl.lock', - 'flufl.password', 'httplib2', 'lazr.config', 'lazr.smtptest', 'mock', + 'passlib', 'restish', 'storm', 'zc.buildout', 'zope.component', 'zope.configuration', + 'zope.event', 'zope.interface', 'zope.testing<4', ], diff --git a/src/mailman/app/bounces.py b/src/mailman/app/bounces.py index 4107a8c6e..41a8f3560 100644 --- a/src/mailman/app/bounces.py +++ b/src/mailman/app/bounces.py @@ -17,7 +17,7 @@ """Application level bounce handling.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -38,7 +38,7 @@ from email.mime.text import MIMEText from email.utils import parseaddr from string import Template from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.core.i18n import _ @@ -176,9 +176,9 @@ class ProbeVERP(_BaseVERPParser): +@implementer(IPendable) class _ProbePendable(dict): """The pendable dictionary for probe messages.""" - implements(IPendable) def send_probe(member, msg): @@ -192,7 +192,8 @@ def send_probe(member, msg): :return: The token representing this probe in the pendings database. :rtype: string """ - mlist = getUtility(IListManager).get(member.mailing_list) + mlist = getUtility(IListManager).get_by_list_id( + member.mailing_list.list_id) text = make('probe.txt', mlist, member.preferred_language.code, listname=mlist.fqdn_listname, address= member.address.email, diff --git a/src/mailman/app/docs/lifecycle.rst b/src/mailman/app/docs/lifecycle.rst index 08a25ccff..f6bb7ddae 100644 --- a/src/mailman/app/docs/lifecycle.rst +++ b/src/mailman/app/docs/lifecycle.rst @@ -44,10 +44,10 @@ Creating a list applies its styles Start by registering a test style. :: - >>> from zope.interface import implements + >>> from zope.interface import implementer >>> from mailman.interfaces.styles import IStyle - >>> class TestStyle(object): - ... implements(IStyle) + >>> @implementer(IStyle) + ... class TestStyle(object): ... name = 'test' ... priority = 10 ... def apply(self, mailing_list): @@ -140,7 +140,7 @@ artifacts. :: >>> from mailman.app.lifecycle import remove_list - >>> remove_list(mlist_2.fqdn_listname, mlist_2) + >>> remove_list(mlist_2) >>> from mailman.interfaces.listmanager import IListManager >>> from zope.component import getUtility diff --git a/src/mailman/app/docs/subscriptions.rst b/src/mailman/app/docs/subscriptions.rst index f897d219e..dd8298cb3 100644 --- a/src/mailman/app/docs/subscriptions.rst +++ b/src/mailman/app/docs/subscriptions.rst @@ -30,13 +30,13 @@ role. At a minimum, a mailing list and an address for the new user is required. >>> mlist = create_list('test@example.com') - >>> anne = service.join('test@example.com', 'anne@example.com') + >>> anne = service.join('test.example.com', 'anne@example.com') >>> anne <Member: anne <anne@example.com> on test@example.com as MemberRole.member> The real name of the new member can be given. - >>> bart = service.join('test@example.com', 'bart@example.com', + >>> bart = service.join('test.example.com', 'bart@example.com', ... 'Bart Person') >>> bart <Member: Bart Person <bart@example.com> @@ -45,7 +45,7 @@ The real name of the new member can be given. Other roles can also be subscribed. >>> from mailman.interfaces.member import MemberRole - >>> anne_owner = service.join('test@example.com', 'anne@example.com', + >>> anne_owner = service.join('test.example.com', 'anne@example.com', ... role=MemberRole.owner) >>> anne_owner <Member: anne <anne@example.com> on test@example.com as MemberRole.owner> @@ -67,7 +67,7 @@ New members can also be added by providing an existing user id instead of an email address. However, the user must have a preferred email address. :: - >>> service.join('test@example.com', bart.user.user_id, + >>> service.join('test.example.com', bart.user.user_id, ... role=MemberRole.owner) Traceback (most recent call last): ... @@ -78,7 +78,7 @@ email address. However, the user must have a preferred email address. >>> address = list(bart.user.addresses)[0] >>> address.verified_on = now() >>> bart.user.preferred_address = address - >>> service.join('test@example.com', bart.user.user_id, + >>> service.join('test.example.com', bart.user.user_id, ... role=MemberRole.owner) <Member: Bart Person <bart@example.com> on test@example.com as MemberRole.owner> @@ -89,7 +89,7 @@ Removing members Regular members can also be removed. - >>> cris = service.join('test@example.com', 'cris@example.com') + >>> cris = service.join('test.example.com', 'cris@example.com') >>> service.get_members() [<Member: anne <anne@example.com> on test@example.com as MemberRole.owner>, @@ -103,7 +103,7 @@ Regular members can also be removed. as MemberRole.member>] >>> sum(1 for member in service) 5 - >>> service.leave('test@example.com', 'cris@example.com') + >>> service.leave('test.example.com', 'cris@example.com') >>> service.get_members() [<Member: anne <anne@example.com> on test@example.com as MemberRole.owner>, @@ -173,7 +173,7 @@ Memberships can also be searched for by user id. You can find all the memberships for a specific mailing list. - >>> service.find_members(fqdn_listname='test@example.com') + >>> service.find_members(list_id='test.example.com') [<Member: anne <anne@example.com> on test@example.com as MemberRole.member>, <Member: anne <anne@example.com> on test@example.com as MemberRole.owner>, @@ -184,9 +184,11 @@ You can find all the memberships for a specific mailing list. <Member: Bart Person <bart@example.com> on test@example.com as MemberRole.owner>] -You can find all the memberships for an address on a specific mailing list. +You can find all the memberships for an address on a specific mailing list, +but you have to give it the list id, not the fqdn listname since the former is +stable but the latter could change if the list is moved. - >>> service.find_members('anne@example.com', 'test@example.com') + >>> service.find_members('anne@example.com', 'test.example.com') [<Member: anne <anne@example.com> on test@example.com as MemberRole.member>, <Member: anne <anne@example.com> on test@example.com @@ -203,7 +205,7 @@ You can find all the memberships for an address with a specific role. You can also find a specific membership by all three criteria. - >>> service.find_members('anne@example.com', 'test@example.com', + >>> service.find_members('anne@example.com', 'test.example.com', ... MemberRole.owner) [<Member: anne <anne@example.com> on test@example.com as MemberRole.owner>] diff --git a/src/mailman/app/events.py b/src/mailman/app/events.py index 79376cca1..28d78e001 100644 --- a/src/mailman/app/events.py +++ b/src/mailman/app/events.py @@ -28,13 +28,22 @@ __all__ = [ from zope import event from mailman.app import domain, moderator, subscriptions +from mailman.core import i18n, switchboard +from mailman.languages import manager as language_manager +from mailman.styles import manager as style_manager +from mailman.utilities import passwords def initialize(): """Initialize global event subscribers.""" event.subscribers.extend([ - moderator.handle_ListDeletingEvent, - subscriptions.handle_ListDeletedEvent, domain.handle_DomainDeletingEvent, + moderator.handle_ListDeletingEvent, + passwords.handle_ConfigurationUpdatedEvent, + subscriptions.handle_ListDeletingEvent, + switchboard.handle_ConfigurationUpdatedEvent, + i18n.handle_ConfigurationUpdatedEvent, + style_manager.handle_ConfigurationUpdatedEvent, + language_manager.handle_ConfigurationUpdatedEvent, ]) diff --git a/src/mailman/app/lifecycle.py b/src/mailman/app/lifecycle.py index 5082034bc..326498478 100644 --- a/src/mailman/app/lifecycle.py +++ b/src/mailman/app/lifecycle.py @@ -89,23 +89,19 @@ def create_list(fqdn_listname, owners=None): -def remove_list(fqdn_listname, mailing_list=None): +def remove_list(mlist): """Remove the list and all associated artifacts and subscriptions.""" + fqdn_listname = mlist.fqdn_listname removeables = [] - # mailing_list will be None when only residual archives are being removed. - if mailing_list is not None: - # Remove all subscriptions, regardless of role. - for member in mailing_list.subscribers.members: - member.unsubscribe() - # Delete the mailing list from the database. - getUtility(IListManager).delete(mailing_list) - # Do the MTA-specific list deletion tasks - call_name(config.mta.incoming).create(mailing_list) - # Remove the list directory. - removeables.append(os.path.join(config.LIST_DATA_DIR, fqdn_listname)) + # Delete the mailing list from the database. + getUtility(IListManager).delete(mlist) + # Do the MTA-specific list deletion tasks + call_name(config.mta.incoming).delete(mlist) + # Remove the list directory. + removeables.append(os.path.join(config.LIST_DATA_DIR, fqdn_listname)) # Remove any stale locks associated with the list. for filename in os.listdir(config.LOCK_DIR): - fn_listname = filename.split('.')[0] + fn_listname, dot, rest = filename.partition('.') if fn_listname == fqdn_listname: removeables.append(os.path.join(config.LOCK_DIR, filename)) # Now that we know what files and directories to delete, delete them. diff --git a/src/mailman/app/membership.py b/src/mailman/app/membership.py index e31a1695c..c73735b35 100644 --- a/src/mailman/app/membership.py +++ b/src/mailman/app/membership.py @@ -27,7 +27,6 @@ __all__ = [ from email.utils import formataddr -from flufl.password import lookup, make_secret from zope.component import getUtility from mailman.app.notifications import send_goodbye_message @@ -96,10 +95,8 @@ def add_member(mlist, email, display_name, password, delivery_mode, language, user.display_name = ( display_name if display_name else address.display_name) user.link(address) - # Encrypt the password using the currently selected scheme. The - # scheme is recorded in the hashed password string. - scheme = lookup(config.passwords.password_scheme.upper()) - user.password = make_secret(password, scheme) + # Encrypt the password using the currently selected hash scheme. + user.password = config.password_context.encrypt(password) user.preferences.preferred_language = language member = mlist.subscribe(address, role) member.preferences.delivery_mode = delivery_mode diff --git a/src/mailman/app/registrar.py b/src/mailman/app/registrar.py index 030a504f7..63e3c5144 100644 --- a/src/mailman/app/registrar.py +++ b/src/mailman/app/registrar.py @@ -28,7 +28,7 @@ __all__ = [ import logging from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer from mailman.app.notifications import send_welcome_message from mailman.core.i18n import _ @@ -47,17 +47,16 @@ log = logging.getLogger('mailman.error') +@implementer(IPendable) class PendableRegistration(dict): - implements(IPendable) PEND_KEY = 'registration' +@implementer(IRegistrar) class Registrar: """Handle registrations and confirmations for subscriptions.""" - implements(IRegistrar) - def register(self, mlist, email, display_name=None, delivery_mode=None): """See `IUserRegistrar`.""" if delivery_mode is None: diff --git a/src/mailman/app/subscriptions.py b/src/mailman/app/subscriptions.py index 60f8cdebe..3937d5b5c 100644 --- a/src/mailman/app/subscriptions.py +++ b/src/mailman/app/subscriptions.py @@ -17,28 +17,29 @@ """Module stuff.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ 'SubscriptionService', - 'handle_ListDeletedEvent', + 'handle_ListDeletingEvent', ] -from flufl.password import generate from operator import attrgetter +from passlib.utils import generate_password as generate from storm.expr import And, Or from uuid import UUID from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer from mailman.app.membership import add_member, delete_member from mailman.config import config from mailman.core.constants import system_preferences +from mailman.database.transaction import dbconnection from mailman.interfaces.address import IEmailValidator from mailman.interfaces.listmanager import ( - IListManager, ListDeletedEvent, NoSuchListError) + IListManager, ListDeletingEvent, NoSuchListError) from mailman.interfaces.member import DeliveryMode, MemberRole from mailman.interfaces.subscriptions import ( ISubscriptionService, MissingUserError) @@ -48,40 +49,35 @@ from mailman.model.member import Member def _membership_sort_key(member): - """Sort function for get_members(). + """Sort function for find_members(). - The members are sorted first by fully-qualified mailing list name, - then by subscribed email address, then by role. + The members are sorted first by unique list id, then by subscribed email + address, then by role. """ - # member.mailing_list is already the fqdn_listname, not the IMailingList - # object. - return (member.mailing_list, - member.address.email, - int(member.role)) + return (member.list_id, member.address.email, int(member.role)) +@implementer(ISubscriptionService) class SubscriptionService: """Subscription services for the REST API.""" - implements(ISubscriptionService) - __name__ = 'members' def get_members(self): """See `ISubscriptionService`.""" - # {fqdn_listname -> {role -> [members]}} + # {list_id -> {role -> [members]}} by_list = {} user_manager = getUtility(IUserManager) for member in user_manager.members: - by_role = by_list.setdefault(member.mailing_list, {}) + by_role = by_list.setdefault(member.list_id, {}) members = by_role.setdefault(member.role.name, []) members.append(member) # Flatten into single list sorted as per the interface. all_members = [] address_of_member = attrgetter('address.email') - for fqdn_listname in sorted(by_list): - by_role = by_list[fqdn_listname] + for list_id in sorted(by_list): + by_role = by_list[list_id] all_members.extend( sorted(by_role.get('owner', []), key=address_of_member)) all_members.extend( @@ -90,9 +86,10 @@ class SubscriptionService: sorted(by_role.get('member', []), key=address_of_member)) return all_members - def get_member(self, member_id): + @dbconnection + def get_member(self, store, member_id): """See `ISubscriptionService`.""" - members = config.db.store.find( + members = store.find( Member, Member._member_id == member_id) if members.count() == 0: @@ -101,13 +98,14 @@ class SubscriptionService: assert members.count() == 1, 'Too many matching members' return members[0] - def find_members(self, subscriber=None, fqdn_listname=None, role=None): + @dbconnection + def find_members(self, store, subscriber=None, list_id=None, role=None): """See `ISubscriptionService`.""" # If `subscriber` is a user id, then we'll search for all addresses # which are controlled by the user, otherwise we'll just search for # the given address. user_manager = getUtility(IUserManager) - if subscriber is None and fqdn_listname is None and role is None: + if subscriber is None and list_id is None and role is None: return [] # Querying for the subscriber is the most complicated part, because # the parameter can either be an email address or a user id. @@ -133,25 +131,25 @@ class SubscriptionService: Member.address_id.is_in(address_ids))) # Calculate the rest of the query expression, which will get And'd # with the Or clause above (if there is one). - if fqdn_listname is not None: - query.append(Member.mailing_list == fqdn_listname) + if list_id is not None: + query.append(Member.list_id == list_id) if role is not None: query.append(Member.role == role) - results = config.db.store.find(Member, And(*query)) + results = store.find(Member, And(*query)) return sorted(results, key=_membership_sort_key) def __iter__(self): for member in self.get_members(): yield member - def join(self, fqdn_listname, subscriber, + def join(self, list_id, subscriber, display_name=None, delivery_mode=DeliveryMode.regular, role=MemberRole.member): """See `ISubscriptionService`.""" - mlist = getUtility(IListManager).get(fqdn_listname) + mlist = getUtility(IListManager).get_by_list_id(list_id) if mlist is None: - raise NoSuchListError(fqdn_listname) + raise NoSuchListError(list_id) # Is the subscriber an email address or user id? if isinstance(subscriber, basestring): # It's an email address, so we'll want a real name. Make sure @@ -178,23 +176,23 @@ class SubscriptionService: raise MissingUserError(subscriber) return mlist.subscribe(user, role) - def leave(self, fqdn_listname, email): + def leave(self, list_id, email): """See `ISubscriptionService`.""" - mlist = getUtility(IListManager).get(fqdn_listname) + mlist = getUtility(IListManager).get_by_list_id(list_id) if mlist is None: - raise NoSuchListError(fqdn_listname) + raise NoSuchListError(list_id) # XXX for now, no notification or user acknowledgment. delete_member(mlist, email, False, False) -def handle_ListDeletedEvent(event): - """Delete a mailing list's members when the list is deleted.""" +def handle_ListDeletingEvent(event): + """Delete a mailing list's members when the list is being deleted.""" - if not isinstance(event, ListDeletedEvent): + if not isinstance(event, ListDeletingEvent): return # Find all the members still associated with the mailing list. members = getUtility(ISubscriptionService).find_members( - fqdn_listname=event.fqdn_listname) + list_id=event.mailing_list.list_id) for member in members: member.unsubscribe() diff --git a/src/mailman/app/templates.py b/src/mailman/app/templates.py index a91231cfc..f29781e58 100644 --- a/src/mailman/app/templates.py +++ b/src/mailman/app/templates.py @@ -31,7 +31,7 @@ from contextlib import closing from urllib import addinfourl from urlparse import urlparse from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer from mailman.utilities.i18n import TemplateNotFoundError, find from mailman.interfaces.languages import ILanguageManager @@ -92,11 +92,10 @@ class MailmanHandler(urllib2.BaseHandler): +@implementer(ITemplateLoader) class TemplateLoader: """Loader of templates, with caching and support for mailman:// URIs.""" - implements(ITemplateLoader) - def __init__(self): opener = urllib2.build_opener(MailmanHandler()) urllib2.install_opener(opener) diff --git a/src/mailman/app/tests/test_membership.py b/src/mailman/app/tests/test_membership.py index 74e15d0e2..00c279910 100644 --- a/src/mailman/app/tests/test_membership.py +++ b/src/mailman/app/tests/test_membership.py @@ -30,13 +30,11 @@ from zope.component import getUtility from mailman.app.lifecycle import create_list from mailman.app.membership import add_member -from mailman.config import config from mailman.core.constants import system_preferences from mailman.interfaces.bans import IBanManager from mailman.interfaces.member import ( AlreadySubscribedError, DeliveryMode, MemberRole, MembershipIsBannedError) from mailman.interfaces.usermanager import IUserManager -from mailman.testing.helpers import reset_the_world from mailman.testing.layers import ConfigLayer @@ -47,9 +45,6 @@ class AddMemberTest(unittest.TestCase): def setUp(self): self._mlist = create_list('test@example.com') - def tearDown(self): - reset_the_world() - def test_add_member_new_user(self): # Test subscribing a user to a mailing list when the email address has # not yet been associated with a user. @@ -57,7 +52,7 @@ class AddMemberTest(unittest.TestCase): 'Anne Person', '123', DeliveryMode.regular, system_preferences.preferred_language) self.assertEqual(member.address.email, 'aperson@example.com') - self.assertEqual(member.mailing_list, 'test@example.com') + self.assertEqual(member.list_id, 'test.example.com') self.assertEqual(member.role, MemberRole.member) def test_add_member_existing_user(self): @@ -69,7 +64,7 @@ class AddMemberTest(unittest.TestCase): 'Anne Person', '123', DeliveryMode.regular, system_preferences.preferred_language) self.assertEqual(member.address.email, 'aperson@example.com') - self.assertEqual(member.mailing_list, 'test@example.com') + self.assertEqual(member.list_id, 'test.example.com') def test_add_member_banned(self): # Test that members who are banned by specific address cannot @@ -132,9 +127,9 @@ class AddMemberTest(unittest.TestCase): system_preferences.preferred_language, MemberRole.moderator) self.assertEqual(member.address.email, 'aperson@example.com') - self.assertEqual(member.mailing_list, 'test@example.com') + self.assertEqual(member.list_id, 'test.example.com') self.assertEqual(member.role, MemberRole.moderator) - + def test_add_member_twice(self): # Adding a member with the same role twice causes an # AlreadySubscribedError to be raised. @@ -164,7 +159,7 @@ class AddMemberTest(unittest.TestCase): 'Anne Person', '123', DeliveryMode.regular, system_preferences.preferred_language, MemberRole.owner) - self.assertEqual(member_1.mailing_list, member_2.mailing_list) + self.assertEqual(member_1.list_id, member_2.list_id) self.assertEqual(member_1.address, member_2.address) self.assertEqual(member_1.user, member_2.user) self.assertNotEqual(member_1.member_id, member_2.member_id) @@ -178,21 +173,10 @@ class AddMemberPasswordTest(unittest.TestCase): def setUp(self): self._mlist = create_list('test@example.com') - # The default ssha scheme introduces a random salt, which is - # inappropriate for unit tests. - config.push('password scheme', """ - [passwords] - password_scheme: sha - """) - - def tearDown(self): - config.pop('password scheme') - reset_the_world() def test_add_member_password(self): # Test that the password stored with the new user is encrypted. member = add_member(self._mlist, 'anne@example.com', 'Anne Person', 'abc', DeliveryMode.regular, system_preferences.preferred_language) - self.assertEqual( - member.user.password, '{SHA}qZk-NkcGgWq6PiVxeFDCbJzQ2J0=') + self.assertEqual(member.user.password, '{plaintext}abc') diff --git a/src/mailman/app/tests/test_moderation.py b/src/mailman/app/tests/test_moderation.py index bc324faea..ef6adf5ed 100644 --- a/src/mailman/app/tests/test_moderation.py +++ b/src/mailman/app/tests/test_moderation.py @@ -37,6 +37,7 @@ from mailman.runners.pipeline import PipelineRunner from mailman.testing.helpers import ( make_testable_runner, specialized_message_from_string) from mailman.testing.layers import SMTPLayer +from mailman.utilities.datetime import now @@ -109,3 +110,14 @@ Message-ID: <alpha> handle_message(self._mlist, request_id, Action.hold) key, data = requests_db.get_request(request_id) self.assertEqual(key, '<alpha>') + + def test_lp_1031391(self): + # LP: #1031391 msgdata['received_time'] gets added by the LMTP server. + # The value is a datetime. If this message gets held, it will break + # pending requests since they require string keys and values. + received_time = now() + msgdata = dict(received_time=received_time) + request_id = hold_message(self._mlist, self._msg, msgdata) + requests_db = IListRequests(self._mlist) + key, data = requests_db.get_request(request_id) + self.assertEqual(data['received_time'], received_time) diff --git a/src/mailman/app/tests/test_subscriptions.py b/src/mailman/app/tests/test_subscriptions.py index a63c9ac04..1c37d4cb9 100644 --- a/src/mailman/app/tests/test_subscriptions.py +++ b/src/mailman/app/tests/test_subscriptions.py @@ -52,7 +52,7 @@ class TestJoin(unittest.TestCase): def test_join_user_with_bogus_id(self): # When `subscriber` is a missing user id, an exception is raised. try: - self._service.join('test@example.com', uuid.UUID(int=99)) + self._service.join('test.example.com', uuid.UUID(int=99)) except MissingUserError as exc: self.assertEqual(exc.user_id, uuid.UUID(int=99)) else: @@ -62,7 +62,7 @@ class TestJoin(unittest.TestCase): # When `subscriber` is a string that is not an email address, an # exception is raised. try: - self._service.join('test@example.com', 'bogus') + self._service.join('test.example.com', 'bogus') except InvalidEmailAddressError as exc: self.assertEqual(exc.email, 'bogus') else: diff --git a/src/mailman/archiving/docs/common.rst b/src/mailman/archiving/docs/common.rst index 7437f4790..86488b26e 100644 --- a/src/mailman/archiving/docs/common.rst +++ b/src/mailman/archiving/docs/common.rst @@ -72,6 +72,8 @@ To archive the message, the archiver actually mails the message to a special address at The Mail Archive. The message gets no header or footer decoration. :: + >>> from mailman.interfaces.archiver import ArchivePolicy + >>> mlist.archive_policy = ArchivePolicy.public >>> archiver.archive_message(mlist, msg) >>> from mailman.runners.outgoing import OutgoingRunner @@ -101,7 +103,7 @@ address at The Mail Archive. The message gets no header or footer decoration. However, if the mailing list is not public, the message will never be archived at this service. - >>> mlist.archive_private = True + >>> mlist.archive_policy = ArchivePolicy.private >>> print archiver.list_url(mlist) None >>> print archiver.permalink(mlist, msg) @@ -114,7 +116,7 @@ Additionally, this archiver can handle malformed ``Message-IDs``. :: >>> from mailman.utilities.email import add_message_hash - >>> mlist.archive_private = False + >>> mlist.archive_policy = ArchivePolicy.public >>> del msg['message-id'] >>> del msg['x-message-id-hash'] >>> msg['Message-ID'] = '12345>' diff --git a/src/mailman/archiving/mailarchive.py b/src/mailman/archiving/mailarchive.py index c72cde11c..e61683a09 100644 --- a/src/mailman/archiving/mailarchive.py +++ b/src/mailman/archiving/mailarchive.py @@ -17,7 +17,7 @@ """The Mail-Archive.com archiver.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -27,35 +27,34 @@ __all__ = [ from urllib import quote from urlparse import urljoin -from zope.interface import implements +from zope.interface import implementer from mailman.config import config -from mailman.interfaces.archiver import IArchiver +from mailman.interfaces.archiver import ArchivePolicy, IArchiver +@implementer(IArchiver) class MailArchive: """Public archiver at the Mail-Archive.com. Messages get archived at http://go.mail-archive.com. """ - implements(IArchiver) - name = 'mail-archive' @staticmethod def list_url(mlist): """See `IArchiver`.""" - if mlist.archive_private: - return None - return urljoin(config.archiver.mail_archive.base_url, - quote(mlist.posting_address)) + if mlist.archive_policy is ArchivePolicy.public: + return urljoin(config.archiver.mail_archive.base_url, + quote(mlist.posting_address)) + return None @staticmethod def permalink(mlist, msg): """See `IArchiver`.""" - if mlist.archive_private: + if mlist.archive_policy is not ArchivePolicy.public: return None # It is the LMTP server's responsibility to ensure that the message # has a X-Message-ID-Hash header. If it doesn't then there's no @@ -68,7 +67,7 @@ class MailArchive: @staticmethod def archive_message(mlist, msg): """See `IArchiver`.""" - if not mlist.archive_private: + if mlist.archive_policy is ArchivePolicy.public: config.switchboards['out'].enqueue( msg, listname=mlist.fqdn_listname, diff --git a/src/mailman/archiving/mhonarc.py b/src/mailman/archiving/mhonarc.py index 0beeed73e..7f0af6cd6 100644 --- a/src/mailman/archiving/mhonarc.py +++ b/src/mailman/archiving/mhonarc.py @@ -17,7 +17,7 @@ """MHonArc archiver.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -29,7 +29,7 @@ import logging import subprocess from urlparse import urljoin -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.interfaces.archiver import IArchiver @@ -40,11 +40,10 @@ log = logging.getLogger('mailman.archiver') +@implementer(IArchiver) class MHonArc: """Local MHonArc archiver.""" - implements(IArchiver) - name = 'mhonarc' @staticmethod diff --git a/src/mailman/archiving/prototype.py b/src/mailman/archiving/prototype.py index 453c6c770..3ce51ddb5 100644 --- a/src/mailman/archiving/prototype.py +++ b/src/mailman/archiving/prototype.py @@ -17,7 +17,7 @@ """Prototypical permalinking archiver.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -34,7 +34,7 @@ from mailbox import Maildir from urlparse import urljoin from flufl.lock import Lock, TimeOutError -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.interfaces.archiver import IArchiver @@ -43,6 +43,7 @@ log = logging.getLogger('mailman.error') +@implementer(IArchiver) class Prototype: """A prototype of a third party archiver. @@ -50,8 +51,6 @@ class Prototype: servers and archivers: <http://wiki.list.org/display/DEV/Stable+URLs>. """ - implements(IArchiver) - name = 'prototype' @staticmethod diff --git a/src/mailman/archiving/tests/test_prototype.py b/src/mailman/archiving/tests/test_prototype.py index 29f6ba1cb..bc1cee8b9 100644 --- a/src/mailman/archiving/tests/test_prototype.py +++ b/src/mailman/archiving/tests/test_prototype.py @@ -37,6 +37,7 @@ from flufl.lock import Lock from mailman.app.lifecycle import create_list from mailman.archiving.prototype import Prototype from mailman.config import config +from mailman.database.transaction import transaction from mailman.testing.helpers import LogFileMark from mailman.testing.helpers import ( specialized_message_from_string as mfs) @@ -61,8 +62,8 @@ X-Message-ID-Hash: MS6QLWERIJLGCRF44J7USBFDELMNT2BW Tests are better than no tests but the water deserves to be swum. """) - self._mlist = create_list('test@example.com') - config.db.commit() + with transaction(): + self._mlist = create_list('test@example.com') # Set up a temporary directory for the prototype archiver so that it's # easier to clean up. self._tempdir = tempfile.mkdtemp() diff --git a/src/mailman/bin/mailman.py b/src/mailman/bin/mailman.py index 94de65255..6b15c9838 100644 --- a/src/mailman/bin/mailman.py +++ b/src/mailman/bin/mailman.py @@ -90,13 +90,9 @@ def main(): # No arguments or subcommands were given. parser.print_help() parser.exit() - # Before actually performing the subcommand, we need to initialize the - # Mailman system, and in particular, we must read the configuration file. - config_file = os.getenv('MAILMAN_CONFIG_FILE') - if config_file is None: - if args.config is not None: - config_file = os.path.abspath(os.path.expanduser(args.config)) - - initialize(config_file) + # Initialize the system. Honor the -C flag if given. + config_path = (None if args.config is None + else os.path.abspath(os.path.expanduser(args.config))) + initialize(config_path) # Perform the subcommand option. args.func(args) diff --git a/src/mailman/bin/set_members.py b/src/mailman/bin/set_members.py deleted file mode 100644 index 6ec66af06..000000000 --- a/src/mailman/bin/set_members.py +++ /dev/null @@ -1,189 +0,0 @@ -# Copyright (C) 2007-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -import csv -import optparse - -from zope.component import getUtility - -from mailman import Utils -from mailman import passwords -from mailman.app.membership import add_member -from mailman.app.notifications import ( - send_admin_subscription_notice, send_welcome_message) -from mailman.configuration import config -from mailman.core.i18n import _ -from mailman.initialize import initialize -from mailman.interfaces.members import DeliveryMode -from mailman.interfaces.usermanager import IUserManager -from mailman.version import MAILMAN_VERSION - - -DELIVERY_MODES = { - 'regular': DeliveryMode.regular, - 'plain': DeliveryMode.plaintext_digests, - 'mime': DeliveryMode.mime_digests, - } - - - -def parseargs(): - parser = optparse.OptionParser(version=MAILMAN_VERSION, - usage=_("""\ -%prog [options] csv-file - -Set the membership of a mailing list to that described in a CSV file. Each -row of the CSV file has the following format. Only the address column is -required. - - - email address - - full name (default: the empty string) - - delivery mode (default: regular delivery) [1] - -[1] The delivery mode is a case insensitive string of the following values: - - regular - regular, i.e. immediate delivery - mime - MIME digest delivery - plain - plain text (RFC 1153) digest delivery - -Any address not included in the CSV file is removed from the list membership. -""")) - parser.add_option('-l', '--listname', - type='string', help=_("""\ -Mailng list to set the membership for.""")) - parser.add_option('-w', '--welcome-msg', - type='string', metavar='<y|n>', help=_("""\ -Set whether or not to send the list members a welcome message, overriding -whatever the list's 'send_welcome_msg' setting is.""")) - parser.add_option('-a', '--admin-notify', - type='string', metavar='<y|n>', help=_("""\ -Set whether or not to send the list administrators a notification on the -success/failure of these subscriptions, overriding whatever the list's -'admin_notify_mchanges' setting is.""")) - parser.add_option('-v', '--verbose', action='store_true', - help=_('Increase verbosity')) - parser.add_option('-C', '--config', - help=_('Alternative configuration file to use')) - opts, args = parser.parse_args() - if opts.welcome_msg is not None: - ch = opts.welcome_msg[0].lower() - if ch == 'y': - opts.welcome_msg = True - elif ch == 'n': - opts.welcome_msg = False - else: - parser.error(_('Illegal value for -w: $opts.welcome_msg')) - if opts.admin_notify is not None: - ch = opts.admin_notify[0].lower() - if ch == 'y': - opts.admin_notify = True - elif ch == 'n': - opts.admin_notify = False - else: - parser.error(_('Illegal value for -a: $opts.admin_notify')) - return parser, opts, args - - - -def parse_file(filename): - members = {} - with open(filename) as fp: - for row in csv.reader(fp): - if len(row) == 0: - continue - elif len(row) == 1: - address = row[0] - real_name = None - delivery_mode = DeliveryMode.regular - elif len(row) == 2: - address, real_name = row - delivery_mode = DeliveryMode.regular - else: - # Ignore extra columns - address, real_name = row[0:2] - delivery_mode = DELIVERY_MODES.get(row[2].lower()) - if delivery_mode is None: - delivery_mode = DeliveryMode.regular - members[address] = real_name, delivery_mode - return members - - - -def main(): - parser, opts, args = parseargs() - initialize(opts.config) - - mlist = config.db.list_manager.get(opts.listname) - if mlist is None: - parser.error(_('No such list: $opts.listname')) - - # Set up defaults. - if opts.welcome_msg is None: - send_welcome_msg = mlist.send_welcome_msg - else: - send_welcome_msg = opts.welcome_msg - if opts.admin_notify is None: - admin_notify = mlist.admin_notify_mchanges - else: - admin_notify = opts.admin_notify - - # Parse the csv files. - member_data = {} - for filename in args: - member_data.update(parse_file(filename)) - - future_members = set(member_data) - current_members = set(obj.address for obj in mlist.members.addresses) - add_members = future_members - current_members - delete_members = current_members - future_members - change_members = current_members & future_members - - with _.using(mlist.preferred_language): - # Start by removing all the delete members. - for address in delete_members: - print _('deleting address: $address') - member = mlist.members.get_member(address) - member.unsubscribe() - # For all members that are in both lists, update their full name and - # delivery mode. - for address in change_members: - print _('updating address: $address') - real_name, delivery_mode = member_data[address] - member = mlist.members.get_member(address) - member.preferences.delivery_mode = delivery_mode - user = getUtility(IUserManager).get_user(address) - user.real_name = real_name - for address in add_members: - print _('adding address: $address') - real_name, delivery_mode = member_data[address] - password = passwords.make_secret( - Utils.MakeRandomPassword(), - passwords.lookup_scheme(config.PASSWORD_SCHEME)) - add_member(mlist, address, real_name, password, delivery_mode, - mlist.preferred_language, send_welcome_msg, - admin_notify) - if send_welcome_msg: - send_welcome_message(mlist, address, language, delivery_mode) - if admin_notify: - send_admin_subscription_notice(mlist, address, real_name) - - config.db.flush() - - - -if __name__ == '__main__': - main() diff --git a/src/mailman/chains/accept.py b/src/mailman/chains/accept.py index 4b326142b..852ca8c06 100644 --- a/src/mailman/chains/accept.py +++ b/src/mailman/chains/accept.py @@ -22,7 +22,6 @@ from __future__ import absolute_import, unicode_literals __metaclass__ = type __all__ = [ 'AcceptChain', - 'AcceptNotification', ] @@ -30,9 +29,10 @@ import logging from zope.event import notify -from mailman.chains.base import ChainNotification, TerminalChainBase +from mailman.chains.base import TerminalChainBase from mailman.config import config from mailman.core.i18n import _ +from mailman.interfaces.chain import AcceptEvent log = logging.getLogger('mailman.vette') @@ -40,11 +40,6 @@ SEMISPACE = '; ' -class AcceptNotification(ChainNotification): - """A notification event signaling that a message is being accepted.""" - - - class AcceptChain(TerminalChainBase): """Accept the message for posting.""" @@ -64,4 +59,4 @@ class AcceptChain(TerminalChainBase): msg['X-Mailman-Rule-Misses'] = SEMISPACE.join(rule_misses) config.switchboards['pipeline'].enqueue(msg, msgdata) log.info('ACCEPT: %s', msg.get('message-id', 'n/a')) - notify(AcceptNotification(mlist, msg, msgdata, self)) + notify(AcceptEvent(mlist, msg, msgdata, self)) diff --git a/src/mailman/chains/base.py b/src/mailman/chains/base.py index 8a2e87ee2..d317bf803 100644 --- a/src/mailman/chains/base.py +++ b/src/mailman/chains/base.py @@ -17,18 +17,17 @@ """Base class for terminal chains.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ 'Chain', - 'ChainNotification', 'Link', 'TerminalChainBase', ] -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.interfaces.chain import ( @@ -36,9 +35,9 @@ from mailman.interfaces.chain import ( +@implementer(IChainLink) class Link: """A chain link.""" - implements(IChainLink) def __init__(self, rule, action=None, chain=None, function=None): self.rule = rule @@ -61,13 +60,12 @@ class Link: +@implementer(IChain, IChainIterator) class TerminalChainBase: """A base chain that always matches and executes a method. The method is called '_process()' and must be provided by the subclass. """ - implements(IChain, IChainIterator) - def _process(self, mlist, msg, msgdata): """Process the message for the given mailing list. @@ -93,9 +91,9 @@ class TerminalChainBase: +@implementer(IMutableChain) class Chain: """Generic chain base class.""" - implements(IMutableChain) def __init__(self, name, description): assert name not in config.chains, ( @@ -125,25 +123,13 @@ class Chain: +@implementer(IChainIterator) class ChainIterator: """Generic chain iterator.""" - implements(IChainIterator) - def __init__(self, chain): self._chain = chain def __iter__(self): """See `IChainIterator`.""" return self._chain.get_iterator() - - - -class ChainNotification: - """Base class for chain notification events.""" - - def __init__(self, mlist, msg, msgdata, chain): - self.mlist = mlist - self.msg = msg - self.msgdata = msgdata - self.chain = chain diff --git a/src/mailman/chains/builtin.py b/src/mailman/chains/builtin.py index 5d51e075d..7ed60dcec 100644 --- a/src/mailman/chains/builtin.py +++ b/src/mailman/chains/builtin.py @@ -17,7 +17,7 @@ """The default built-in starting chain.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -27,7 +27,7 @@ __all__ = [ import logging -from zope.interface import implements +from zope.interface import implementer from mailman.chains.base import Link from mailman.config import config @@ -39,11 +39,10 @@ log = logging.getLogger('mailman.vette') +@implementer(IChain) class BuiltInChain: """Default built-in chain.""" - implements(IChain) - name = 'default-posting-chain' description = _('The built-in moderation chain.') diff --git a/src/mailman/chains/discard.py b/src/mailman/chains/discard.py index 0a4bfe9ac..a52d25d7a 100644 --- a/src/mailman/chains/discard.py +++ b/src/mailman/chains/discard.py @@ -22,26 +22,21 @@ from __future__ import absolute_import, unicode_literals __metaclass__ = type __all__ = [ 'DiscardChain', - 'DiscardNotification', ] import logging from zope.event import notify -from mailman.chains.base import ChainNotification, TerminalChainBase +from mailman.chains.base import TerminalChainBase from mailman.core.i18n import _ +from mailman.interfaces.chain import DiscardEvent log = logging.getLogger('mailman.vette') -class DiscardNotification(ChainNotification): - """A notification event signaling that a message is being discarded.""" - - - class DiscardChain(TerminalChainBase): """Discard a message.""" @@ -55,5 +50,5 @@ class DiscardChain(TerminalChainBase): message away. """ log.info('DISCARD: %s', msg.get('message-id', 'n/a')) - notify(DiscardNotification(mlist, msg, msgdata, self)) + notify(DiscardEvent(mlist, msg, msgdata, self)) # Nothing more needs to happen. diff --git a/src/mailman/chains/docs/moderation.rst b/src/mailman/chains/docs/moderation.rst index fed120147..1e968ee68 100644 --- a/src/mailman/chains/docs/moderation.rst +++ b/src/mailman/chains/docs/moderation.rst @@ -45,9 +45,9 @@ In order to find out whether the message is held or accepted, we can subscribe to Zope events that are triggered on each case. :: - >>> from mailman.chains.base import ChainNotification + >>> from mailman.interfaces.chain import ChainEvent >>> def on_chain(event): - ... if isinstance(event, ChainNotification): + ... if isinstance(event, ChainEvent): ... print event ... print event.chain ... print 'Subject:', event.msg['subject'] @@ -74,7 +74,7 @@ built-in chain. No rules hit and so the message is accepted. >>> from mailman.testing.helpers import event_subscribers >>> with event_subscribers(on_chain): ... process(mlist, msg, {}, 'default-posting-chain') - <mailman.chains.accept.AcceptNotification ...> + <mailman.interfaces.chain.AcceptEvent ...> <mailman.chains.accept.AcceptChain ...> Subject: aardvark Hits: @@ -109,7 +109,7 @@ moderator approval. >>> with event_subscribers(on_chain): ... process(mlist, msg, {}, 'default-posting-chain') - <mailman.chains.hold.HoldNotification ...> + <mailman.interfaces.chain.HoldEvent ...> <mailman.chains.hold.HoldChain ...> Subject: badger Hits: @@ -134,7 +134,7 @@ The list's member moderation action can also be set to `discard`... >>> with event_subscribers(on_chain): ... process(mlist, msg, {}, 'default-posting-chain') - <mailman.chains.discard.DiscardNotification ...> + <mailman.interfaces.chain.DiscardEvent ...> <mailman.chains.discard.DiscardChain ...> Subject: cougar Hits: @@ -158,7 +158,7 @@ The list's member moderation action can also be set to `discard`... >>> with event_subscribers(on_chain): ... process(mlist, msg, {}, 'default-posting-chain') - <mailman.chains.reject.RejectNotification ...> + <mailman.interfaces.chain.RejectEvent ...> <mailman.chains.reject.RejectChain ...> Subject: dingo Hits: @@ -197,7 +197,7 @@ moderator approval. >>> with event_subscribers(on_chain): ... process(mlist, msg, {}, 'default-posting-chain') - <mailman.chains.hold.HoldNotification ...> + <mailman.interfaces.chain.HoldEvent ...> <mailman.chains.hold.HoldChain ...> Subject: elephant Hits: diff --git a/src/mailman/chains/headers.py b/src/mailman/chains/headers.py index d9f8356f8..2dafa07f0 100644 --- a/src/mailman/chains/headers.py +++ b/src/mailman/chains/headers.py @@ -17,7 +17,7 @@ """The header-matching chain.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -28,7 +28,7 @@ __all__ = [ import re import logging -from zope.interface import implements +from zope.interface import implementer from mailman.chains.base import Chain, Link from mailman.config import config @@ -60,9 +60,9 @@ def make_link(header, pattern): +@implementer(IRule) class HeaderMatchRule: """Header matching rule used by header-match chain.""" - implements(IRule) # Sequential rule counter. _count = 1 diff --git a/src/mailman/chains/hold.py b/src/mailman/chains/hold.py index f095bc182..34389f82e 100644 --- a/src/mailman/chains/hold.py +++ b/src/mailman/chains/hold.py @@ -17,12 +17,11 @@ """The terminal 'hold' chain.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ 'HoldChain', - 'HoldNotification', ] @@ -33,15 +32,16 @@ from email.mime.text import MIMEText from email.utils import formatdate, make_msgid from zope.component import getUtility from zope.event import notify -from zope.interface import implements +from zope.interface import implementer from mailman.app.moderator import hold_message from mailman.app.replybot import can_acknowledge -from mailman.chains.base import ChainNotification, TerminalChainBase +from mailman.chains.base import TerminalChainBase from mailman.config import config from mailman.core.i18n import _ from mailman.email.message import UserNotification from mailman.interfaces.autorespond import IAutoResponseSet, Response +from mailman.interfaces.chain import HoldEvent from mailman.interfaces.languages import ILanguageManager from mailman.interfaces.pending import IPendable, IPendings from mailman.interfaces.usermanager import IUserManager @@ -54,15 +54,11 @@ SEMISPACE = '; ' +@implementer(IPendable) class HeldMessagePendable(dict): - implements(IPendable) PEND_KEY = 'held message' -class HoldNotification(ChainNotification): - """A notification event signaling that a message is being held.""" - - def autorespond_to_sender(mlist, sender, language=None): """Should Mailman automatically respond to this sender? @@ -249,4 +245,4 @@ also appear in the first line of the body of the reply.""")), log.info('HOLD: %s post from %s held, message-id=%s: %s', mlist.fqdn_listname, msg.sender, msg.get('message-id', 'n/a'), reason) - notify(HoldNotification(mlist, msg, msgdata, self)) + notify(HoldEvent(mlist, msg, msgdata, self)) diff --git a/src/mailman/chains/moderation.py b/src/mailman/chains/moderation.py index 9f9633346..6c2ed1180 100644 --- a/src/mailman/chains/moderation.py +++ b/src/mailman/chains/moderation.py @@ -34,7 +34,7 @@ made as to the disposition of the message. `defer` is the default for members, while `hold` is the default for nonmembers. """ -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -42,7 +42,7 @@ __all__ = [ ] -from zope.interface import implements +from zope.interface import implementer from mailman.chains.base import Link from mailman.config import config @@ -52,15 +52,13 @@ from mailman.interfaces.chain import IChain, LinkAction +@implementer(IChain) class ModerationChain: """Dynamically produce a link jumping to the appropriate terminal chain. The terminal chain will be one of the Accept, Hold, Discard, or Reject chains, based on the member's or nonmember's moderation action setting. """ - - implements(IChain) - name = 'moderation' description = _('Moderation chain') diff --git a/src/mailman/chains/owner.py b/src/mailman/chains/owner.py index ad0a04cea..22ed4b996 100644 --- a/src/mailman/chains/owner.py +++ b/src/mailman/chains/owner.py @@ -29,20 +29,16 @@ import logging from zope.event import notify -from mailman.chains.base import ChainNotification, TerminalChainBase +from mailman.chains.base import TerminalChainBase from mailman.config import config from mailman.core.i18n import _ +from mailman.interfaces.chain import AcceptOwnerEvent log = logging.getLogger('mailman.vette') -class OwnerNotification(ChainNotification): - """An event signaling that a message is accepted to the -owner address.""" - - - class BuiltInOwnerChain(TerminalChainBase): """Default built-in -owner address chain.""" @@ -53,4 +49,4 @@ class BuiltInOwnerChain(TerminalChainBase): # At least for now, everything posted to -owners goes through. config.switchboards['pipeline'].enqueue(msg, msgdata) log.info('OWNER: %s', msg.get('message-id', 'n/a')) - notify(OwnerNotification(mlist, msg, msgdata, self)) + notify(AcceptOwnerEvent(mlist, msg, msgdata, self)) diff --git a/src/mailman/chains/reject.py b/src/mailman/chains/reject.py index cf1b86fb2..3adc69e3a 100644 --- a/src/mailman/chains/reject.py +++ b/src/mailman/chains/reject.py @@ -22,16 +22,17 @@ from __future__ import absolute_import, unicode_literals __metaclass__ = type __all__ = [ 'RejectChain', - 'RejectNotification', ] import logging + from zope.event import notify from mailman.app.bounces import bounce_message -from mailman.chains.base import ChainNotification, TerminalChainBase +from mailman.chains.base import TerminalChainBase from mailman.core.i18n import _ +from mailman.interfaces.chain import RejectEvent log = logging.getLogger('mailman.vette') @@ -39,11 +40,6 @@ SEMISPACE = '; ' -class RejectNotification(ChainNotification): - """A notification event signaling that a message is being rejected.""" - - - class RejectChain(TerminalChainBase): """Reject/bounce a message.""" @@ -64,4 +60,4 @@ class RejectChain(TerminalChainBase): # XXX Exception/reason bounce_message(mlist, msg) log.info('REJECT: %s', msg.get('message-id', 'n/a')) - notify(RejectNotification(mlist, msg, msgdata, self)) + notify(RejectEvent(mlist, msg, msgdata, self)) diff --git a/src/mailman/chains/tests/test_hold.py b/src/mailman/chains/tests/test_hold.py index f2cd1dabf..515894505 100644 --- a/src/mailman/chains/tests/test_hold.py +++ b/src/mailman/chains/tests/test_hold.py @@ -30,10 +30,9 @@ from zope.component import getUtility from mailman.app.lifecycle import create_list from mailman.chains.hold import autorespond_to_sender -from mailman.config import config from mailman.interfaces.autorespond import IAutoResponseSet, Response from mailman.interfaces.usermanager import IUserManager -from mailman.testing.helpers import get_queue_messages +from mailman.testing.helpers import configuration, get_queue_messages from mailman.testing.layers import ConfigLayer @@ -49,15 +48,12 @@ class TestAutorespond(unittest.TestCase): self.maxDiff = None self.eq = getattr(self, 'assertMultiLineEqual', self.assertEqual) + @configuration('mta', max_autoresponses_per_day=1) def test_max_autoresponses_per_day(self): # The last one we sent was the last one we should send today. Instead # of sending an automatic response, send them the "no more today" - # message. - config.push('max-1', """ - [mta] - max_autoresponses_per_day: 1 - """) - # Simulate a response having been sent to an address already. + # message. Start by simulating a response having been sent to an + # address already. anne = getUtility(IUserManager).create_address('anne@example.com') response_set = IAutoResponseSet(self._mlist) response_set.response_sent(anne, Response.hold) diff --git a/src/mailman/chains/tests/test_owner.py b/src/mailman/chains/tests/test_owner.py index db85d4967..b50c08e31 100644 --- a/src/mailman/chains/tests/test_owner.py +++ b/src/mailman/chains/tests/test_owner.py @@ -28,8 +28,9 @@ __all__ = [ import unittest from mailman.app.lifecycle import create_list -from mailman.chains.owner import BuiltInOwnerChain, OwnerNotification +from mailman.chains.owner import BuiltInOwnerChain from mailman.core.chains import process +from mailman.interfaces.chain import AcceptOwnerEvent from mailman.testing.helpers import ( event_subscribers, get_queue_messages, @@ -60,12 +61,13 @@ Message-ID: <ant> # is processed by the owner chain. events = [] def catch_event(event): - events.append(event) + if isinstance(event, AcceptOwnerEvent): + events.append(event) with event_subscribers(catch_event): process(self._mlist, self._msg, {}, 'default-owner-chain') self.assertEqual(len(events), 1) event = events[0] - self.assertTrue(isinstance(event, OwnerNotification)) + self.assertTrue(isinstance(event, AcceptOwnerEvent)) self.assertEqual(event.mlist, self._mlist) self.assertEqual(event.msg['message-id'], '<ant>') self.assertTrue(isinstance(event.chain, BuiltInOwnerChain)) diff --git a/src/mailman/commands/cli_aliases.py b/src/mailman/commands/cli_aliases.py index 7c1577c9c..d692ba356 100644 --- a/src/mailman/commands/cli_aliases.py +++ b/src/mailman/commands/cli_aliases.py @@ -17,7 +17,7 @@ """Generate Mailman alias files for your MTA.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -29,7 +29,7 @@ import sys from operator import attrgetter from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.core.i18n import _ @@ -41,11 +41,10 @@ from mailman.utilities.modules import call_name +@implementer(ICLISubCommand) class Aliases: """Regenerate the aliases appropriate for your MTA.""" - implements(ICLISubCommand) - name = 'aliases' def add(self, parser, command_parser): @@ -91,11 +90,10 @@ class Aliases: +@implementer(IMailTransportAgentLifecycle) class Dummy: """Dummy aliases implementation for simpler output format.""" - implements(IMailTransportAgentLifecycle) - def create(self, mlist): """See `IMailTransportAgentLifecycle`.""" raise NotImplementedError @@ -132,5 +130,5 @@ class Dummy: for mlist in sorted(by_domain[domain], key=sort_key): utility = getUtility(IMailTransportAgentAliases) for alias in utility.aliases(mlist): - print >> fp, alias - print >> fp + print(alias, file=fp) + print(file=fp) diff --git a/src/mailman/commands/cli_control.py b/src/mailman/commands/cli_control.py index 2013d6745..8349feb60 100644 --- a/src/mailman/commands/cli_control.py +++ b/src/mailman/commands/cli_control.py @@ -17,7 +17,7 @@ """Module stuff.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -34,7 +34,7 @@ import errno import signal import logging -from zope.interface import implements +from zope.interface import implementer from mailman.bin.master import WatcherState, master_state from mailman.config import config @@ -46,11 +46,10 @@ qlog = logging.getLogger('mailman.runner') +@implementer(ICLISubCommand) class Start: """Start the Mailman daemons.""" - implements(ICLISubCommand) - name = 'start' def add(self, parser, command_parser): @@ -107,7 +106,7 @@ class Start: 'cleanly. Try using --force.')) def log(message): if not args.quiet: - print message + print(message) # Daemon process startup according to Stevens, Advanced Programming in # the UNIX Environment, Chapter 13. pid = os.fork() @@ -147,27 +146,26 @@ def kill_watcher(sig): pid = int(fp.read().strip()) except (IOError, ValueError) as error: # For i18n convenience - print >> sys.stderr, _('PID unreadable in: $config.PID_FILE') - print >> sys.stderr, error - print >> sys.stderr, _('Is the master even running?') + print(_('PID unreadable in: $config.PID_FILE'), file=sys.stderr) + print(error, file=sys.stderr) + print(_('Is the master even running?'), file=sys.stderr) return try: os.kill(pid, sig) except OSError as error: if error.errno != errno.ESRCH: raise - print >> sys.stderr, _('No child with pid: $pid') - print >> sys.stderr, error - print >> sys.stderr, _('Stale pid file removed.') + print(_('No child with pid: $pid'), file=sys.stderr) + print(error, file=sys.stderr) + print(_('Stale pid file removed.'), file=sys.stderr) os.unlink(config.PID_FILE) +@implementer(ICLISubCommand) class SignalCommand: """Common base class for simple, signal sending commands.""" - implements(ICLISubCommand) - name = None message = None signal = None @@ -184,7 +182,7 @@ class SignalCommand: def process(self, args): """See `ICLISubCommand`.""" if not args.quiet: - print _(self.message) + print(_(self.message)) kill_watcher(self.signal) @@ -204,11 +202,10 @@ class Reopen(SignalCommand): signal = signal.SIGHUP +@implementer(ICLISubCommand) class Restart(SignalCommand): """Stop the Mailman daemons.""" - implements(ICLISubCommand) - name = 'restart' message = _('Restarting the Mailman runners') signal = signal.SIGUSR1 diff --git a/src/mailman/commands/cli_help.py b/src/mailman/commands/cli_help.py index 538d9c520..a85dcd442 100644 --- a/src/mailman/commands/cli_help.py +++ b/src/mailman/commands/cli_help.py @@ -17,7 +17,7 @@ """The 'help' subcommand.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -25,18 +25,17 @@ __all__ = [ ] -from zope.interface import implements +from zope.interface import implementer from mailman.interfaces.command import ICLISubCommand +@implementer(ICLISubCommand) class Help: # Lowercase, to match argparse's default --help text. """show this help message and exit""" - implements(ICLISubCommand) - name = 'help' def add(self, parser, command_parser): diff --git a/src/mailman/commands/cli_import.py b/src/mailman/commands/cli_import.py index b703f3ffd..f6c016585 100644 --- a/src/mailman/commands/cli_import.py +++ b/src/mailman/commands/cli_import.py @@ -17,7 +17,7 @@ """Importing list data into Mailman 3.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -29,21 +29,20 @@ import sys import cPickle from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer -from mailman.config import config from mailman.core.i18n import _ +from mailman.database.transaction import transactional from mailman.interfaces.command import ICLISubCommand from mailman.interfaces.listmanager import IListManager from mailman.utilities.importer import import_config_pck +@implementer(ICLISubCommand) class Import21: """Import Mailman 2.1 list data.""" - implements(ICLISubCommand) - name = 'import21' def add(self, parser, command_parser): @@ -59,6 +58,7 @@ class Import21: 'pickle_file', metavar='FILENAME', nargs=1, help=_('The path to the config.pck file to import.')) + @transactional def process(self, args): """See `ICLISubCommand`.""" # Could be None or sequence of length 0. @@ -90,10 +90,7 @@ class Import21: return else: if not isinstance(config_dict, dict): - print >> sys.stderr, _( - 'Ignoring non-dictionary: {0!r}').format( - config_dict) + print(_('Ignoring non-dictionary: {0!r}').format( + config_dict), file=sys.stderr) continue import_config_pck(mlist, config_dict) - # Commit the changes to the database. - config.db.commit() diff --git a/src/mailman/commands/cli_info.py b/src/mailman/commands/cli_info.py index 24ccec4fb..cd7269404 100644 --- a/src/mailman/commands/cli_info.py +++ b/src/mailman/commands/cli_info.py @@ -17,7 +17,7 @@ """Information about this Mailman instance.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -27,7 +27,8 @@ __all__ = [ import sys -from zope.interface import implements +from lazr.config import as_boolean +from zope.interface import implementer from mailman.config import config from mailman.core.i18n import _ @@ -37,11 +38,10 @@ from mailman.version import MAILMAN_VERSION_FULL +@implementer(ICLISubCommand) class Info: """Information about this Mailman instance.""" - implements(ICLISubCommand) - name = 'info' def add(self, parser, command_parser): @@ -65,15 +65,19 @@ class Info: # We don't need to close output because that will happen # automatically when the script exits. output = open(args.output, 'w') - print >> output, MAILMAN_VERSION_FULL - print >> output, 'Python', sys.version - print >> output, 'config file:', config.filename - print >> output, 'db url:', config.db.url - print >> output, 'REST root url:', path_to('/') - print >> output, 'REST credentials: {0}:{1}'.format( - config.webservice.admin_user, config.webservice.admin_pass) + print(MAILMAN_VERSION_FULL, file=output) + print('Python', sys.version, file=output) + print('config file:', config.filename, file=output) + print('db url:', config.db.url, file=output) + print('devmode:', + 'ENABLED' if as_boolean(config.devmode.enabled) else 'DISABLED', + file=output) + print('REST root url:', path_to('/'), file=output) + print('REST credentials: {0}:{1}'.format( + config.webservice.admin_user, config.webservice.admin_pass), + file=output) if args.verbose: - print >> output, 'File system paths:' + print('File system paths:', file=output) longest = 0 paths = {} for attribute in dir(config): @@ -81,5 +85,5 @@ class Info: paths[attribute] = getattr(config, attribute) longest = max(longest, len(attribute)) for attribute in sorted(paths): - print ' {0:{2}} = {1}'.format(attribute, paths[attribute], - longest) + print(' {0:{2}} = {1}'.format( + attribute, paths[attribute], longest)) diff --git a/src/mailman/commands/cli_inject.py b/src/mailman/commands/cli_inject.py index 321a92c78..1434fd2a6 100644 --- a/src/mailman/commands/cli_inject.py +++ b/src/mailman/commands/cli_inject.py @@ -17,7 +17,7 @@ """bin/mailman inject""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -28,7 +28,7 @@ __all__ = [ import sys from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer from mailman.app.inject import inject_text from mailman.config import config @@ -38,11 +38,10 @@ from mailman.interfaces.listmanager import IListManager +@implementer(ICLISubCommand) class Inject: """Inject a message from a file into a mailing list's queue.""" - implements(ICLISubCommand) - name = 'inject' def add(self, parser, command_parser): @@ -82,9 +81,9 @@ class Inject: # Process --show first; if given, print output and exit, ignoring all # other command line switches. if args.show: - print 'Available queues:' + print('Available queues:') for switchboard in sorted(config.switchboards): - print ' ', switchboard + print(' ', switchboard) return # Could be None or sequence of length 0. if args.listname is None: @@ -106,7 +105,7 @@ class Inject: try: message_text = sys.stdin.read() except KeyboardInterrupt: - print 'Interrupted' + print('Interrupted') sys.exit(1) else: with open(args.filename) as fp: diff --git a/src/mailman/commands/cli_lists.py b/src/mailman/commands/cli_lists.py index af6afe22d..b91f708de 100644 --- a/src/mailman/commands/cli_lists.py +++ b/src/mailman/commands/cli_lists.py @@ -17,7 +17,7 @@ """The 'lists' subcommand.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -28,12 +28,12 @@ __all__ = [ from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer from mailman.app.lifecycle import create_list, remove_list -from mailman.config import config from mailman.core.constants import system_preferences from mailman.core.i18n import _ +from mailman.database.transaction import transaction, transactional from mailman.email.message import UserNotification from mailman.interfaces.address import ( IEmailValidator, InvalidEmailAddressError) @@ -49,11 +49,10 @@ COMMASPACE = ', ' +@implementer(ICLISubCommand) class Lists: """List all mailing lists""" - implements(ICLISubCommand) - name = 'lists' def add(self, parser, command_parser): @@ -98,11 +97,11 @@ class Lists: # Maybe no mailing lists matched. if len(mailing_lists) == 0: if not args.quiet: - print _('No matching mailing lists found') + print(_('No matching mailing lists found')) return count = len(mailing_lists) if not args.quiet: - print _('$count matching mailing lists found:') + print(_('$count matching mailing lists found:')) # Calculate the longest identifier. longest = 0 output = [] @@ -120,16 +119,15 @@ class Lists: else: format_string = '{0:{2}}' for identifier, description in output: - print format_string.format( - identifier, description, longest, 70 - longest) + print(format_string.format( + identifier, description, longest, 70 - longest)) +@implementer(ICLISubCommand) class Create: """Create a mailing list""" - implements(ICLISubCommand) - name = 'create' def add(self, parser, command_parser): @@ -214,13 +212,13 @@ class Create: self.parser.error(_('Undefined domain: $domain')) return # Find the language associated with the code, then set the mailing - # list's preferred language to that. The changes then must be - # committed to the database. - mlist.preferred_language = getUtility(ILanguageManager)[language_code] - config.db.commit() + # list's preferred language to that. + language_manager = getUtility(ILanguageManager) + with transaction(): + mlist.preferred_language = language_manager[language_code] # Do the notification. if not args.quiet: - print _('Created mailing list: $mlist.fqdn_listname') + print(_('Created mailing list: $mlist.fqdn_listname')) if args.notify: d = dict( listname = mlist.fqdn_listname, @@ -242,11 +240,10 @@ class Create: +@implementer(ICLISubCommand) class Remove: """Remove a mailing list""" - implements(ICLISubCommand) - name = 'remove' def add(self, parser, command_parser): @@ -262,11 +259,12 @@ class Remove: The 'fully qualified list name', i.e. the posting address of the mailing list.""")) + @transactional def process(self, args): """See `ICLISubCommand`.""" def log(message): if not args.quiet: - print message + print(message) assert len(args.listname) == 1, ( 'Unexpected positional arguments: %s' % args.listname) fqdn_listname = args.listname[0] @@ -276,5 +274,4 @@ class Remove: return else: log(_('Removed list: $fqdn_listname')) - remove_list(fqdn_listname, mlist) - config.db.commit() + remove_list(mlist) diff --git a/src/mailman/commands/cli_members.py b/src/mailman/commands/cli_members.py index 2bf6be848..7c5d3b8f3 100644 --- a/src/mailman/commands/cli_members.py +++ b/src/mailman/commands/cli_members.py @@ -29,14 +29,15 @@ import sys import codecs from email.utils import formataddr, parseaddr -from flufl.password import generate from operator import attrgetter +from passlib.utils import generate_password as generate from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer from mailman.app.membership import add_member from mailman.config import config from mailman.core.i18n import _ +from mailman.database.transaction import transactional from mailman.interfaces.command import ICLISubCommand from mailman.interfaces.listmanager import IListManager from mailman.interfaces.member import ( @@ -44,11 +45,10 @@ from mailman.interfaces.member import ( +@implementer(ICLISubCommand) class Members: """Manage list memberships. With no arguments, list all members.""" - implements(ICLISubCommand) - name = 'members' def add(self, parser, command_parser): @@ -177,6 +177,7 @@ class Members: if fp is not sys.stdout: fp.close() + @transactional def add_members(self, mlist, args): """Add the members in a file to a mailing list. @@ -207,9 +208,8 @@ class Members: except AlreadySubscribedError: # It's okay if the address is already subscribed, just # print a warning and continue. - print('Already subscribed (skipping):', + print('Already subscribed (skipping):', email, display_name) finally: if fp is not sys.stdin: fp.close() - config.db.commit() diff --git a/src/mailman/commands/cli_qfile.py b/src/mailman/commands/cli_qfile.py index 78156f08c..b9e0eff02 100644 --- a/src/mailman/commands/cli_qfile.py +++ b/src/mailman/commands/cli_qfile.py @@ -17,7 +17,7 @@ """Getting information out of a qfile.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -28,7 +28,7 @@ __all__ = [ import cPickle from pprint import PrettyPrinter -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.interfaces.command import ICLISubCommand @@ -39,11 +39,10 @@ m = [] +@implementer(ICLISubCommand) class QFile: """Get information out of a queue file.""" - implements(ICLISubCommand) - name = 'qfile' def add(self, parser, command_parser): @@ -79,15 +78,15 @@ class QFile: except EOFError: break if args.doprint: - print _('[----- start pickle -----]') + print(_('[----- start pickle -----]')) for i, obj in enumerate(m): count = i + 1 - print _('<----- start object $count ----->') + print(_('<----- start object $count ----->')) if isinstance(obj, basestring): - print obj + print(obj) else: printer.pprint(obj) - print _('[----- end pickle -----]') + print(_('[----- end pickle -----]')) count = len(m) banner = _("The variable 'm' contains $count objects") if args.interactive: diff --git a/src/mailman/commands/cli_status.py b/src/mailman/commands/cli_status.py index 14b0d976b..9cbaa4f22 100644 --- a/src/mailman/commands/cli_status.py +++ b/src/mailman/commands/cli_status.py @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see <http://www.gnu.org/licenses/>. -"""Module stuff.""" +"""bin/mailman status.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -27,7 +27,7 @@ __all__ = [ import socket -from zope.interface import implements +from zope.interface import implementer from mailman.bin.master import WatcherState, master_state from mailman.core.i18n import _ @@ -35,11 +35,10 @@ from mailman.interfaces.command import ICLISubCommand +@implementer(ICLISubCommand) class Status: """Status of the Mailman system.""" - implements(ICLISubCommand) - name = 'status' def add(self, parser, command_parser): @@ -64,5 +63,5 @@ class Status: 'Invalid enum value: %s' % status) message = _('GNU Mailman is in an unexpected state ' '($hostname != $fqdn_name)') - print message + print(message) return int(status) diff --git a/src/mailman/commands/cli_unshunt.py b/src/mailman/commands/cli_unshunt.py index 4ce711b83..bc50d95ef 100644 --- a/src/mailman/commands/cli_unshunt.py +++ b/src/mailman/commands/cli_unshunt.py @@ -17,7 +17,7 @@ """The 'unshunt' command.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -27,7 +27,7 @@ __all__ = [ import sys -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.core.i18n import _ @@ -35,11 +35,10 @@ from mailman.interfaces.command import ICLISubCommand +@implementer(ICLISubCommand) class Unshunt: """Unshunt messages.""" - implements(ICLISubCommand) - name = 'unshunt' def add(self, parser, command_parser): @@ -64,8 +63,8 @@ class Unshunt: if not args.discard: config.switchboards[which_queue].enqueue(msg, msgdata) except Exception as error: - print >> sys.stderr, _( - 'Cannot unshunt message $filebase, skipping:\n$error') + print(_('Cannot unshunt message $filebase, skipping:\n$error'), + file=sys.stderr) else: # Unlink the .bak file left by dequeue() shunt_queue.finish(filebase) diff --git a/src/mailman/commands/cli_version.py b/src/mailman/commands/cli_version.py index 4090b1173..b5e9b65ff 100644 --- a/src/mailman/commands/cli_version.py +++ b/src/mailman/commands/cli_version.py @@ -17,7 +17,7 @@ """The Mailman version.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -25,18 +25,17 @@ __all__ = [ ] -from zope.interface import implements +from zope.interface import implementer from mailman.interfaces.command import ICLISubCommand from mailman.version import MAILMAN_VERSION_FULL +@implementer(ICLISubCommand) class Version: """Mailman's version.""" - implements(ICLISubCommand) - name = 'version' def add(self, parser, command_parser): @@ -46,4 +45,4 @@ class Version: def process(self, args): """See `ICLISubCommand`.""" - print MAILMAN_VERSION_FULL + print(MAILMAN_VERSION_FULL) diff --git a/src/mailman/commands/cli_withlist.py b/src/mailman/commands/cli_withlist.py index e514c798f..4ccdd8798 100644 --- a/src/mailman/commands/cli_withlist.py +++ b/src/mailman/commands/cli_withlist.py @@ -17,7 +17,7 @@ """bin/mailman withlist""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -31,7 +31,7 @@ import sys from lazr.config import as_boolean from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.core.i18n import _ @@ -47,14 +47,13 @@ r = None +@implementer(ICLISubCommand) class Withlist: """Operate on a mailing list. For detailed help, see --details """ - implements(ICLISubCommand) - name = 'withlist' def add(self, parser, command_parser): @@ -163,7 +162,7 @@ class Withlist: ipshell = InteractiveShellEmbed(banner1=banner, user_ns=overrides) ipshell() except ImportError: - print _('ipython is not available, set use_ipython to no') + print(_('ipython is not available, set use_ipython to no')) def _start_python(self, overrides, banner): # Set the tab completion. @@ -178,25 +177,25 @@ class Withlist: def _details(self): """Print detailed usage.""" # Split this up into paragraphs for easier translation. - print _("""\ + print(_("""\ This script provides you with a general framework for interacting with a -mailing list.""") - print - print _("""\ +mailing list.""")) + print() + print(_("""\ There are two ways to use this script: interactively or programmatically. Using it interactively allows you to play with, examine and modify a mailing list from Python's interactive interpreter. When running interactively, the variable 'm' will be available in the global namespace. It will reference the -mailing list object.""") - print - print _("""\ +mailing list object.""")) + print() + print(_("""\ Programmatically, you can write a function to operate on a mailing list, and this script will take care of the housekeeping (see below for examples). In that case, the general usage syntax is: - % bin/mailman withlist [options] listname [args ...]""") - print - print _("""\ + % bin/mailman withlist [options] listname [args ...]""")) + print() + print(_("""\ Here's an example of how to use the --run option. Say you have a file in the Mailman installation directory called 'listaddr.py', with the following two functions: @@ -205,26 +204,26 @@ functions: print mlist.posting_address def requestaddr(mlist): - print mlist.request_address""") - print - print _("""\ + print mlist.request_address""")) + print() + print(_("""\ You can print the list's posting address by running the following from the command line: % bin/mailman withlist -r listaddr mylist@example.com Importing listaddr ... Running listaddr.listaddr() ... - mylist@example.com""") - print - print _("""\ + mylist@example.com""")) + print() + print(_("""\ And you can print the list's request address by running: % bin/mailman withlist -r listaddr.requestaddr mylist Importing listaddr ... Running listaddr.requestaddr() ... - mylist-request@example.com""") - print - print _("""\ + mylist-request@example.com""")) + print() + print(_("""\ As another example, say you wanted to change the display name for a particular mailing list. You could put the following function in a file called 'change.pw': @@ -236,7 +235,7 @@ mailing list. You could put the following function in a file called and run this from the command line: - % bin/mailman withlist -r change mylist@example.com 'My List'""") + % bin/mailman withlist -r change mylist@example.com 'My List'""")) diff --git a/src/mailman/commands/docs/end.rst b/src/mailman/commands/docs/end.rst index accf91b90..8cd4b2409 100644 --- a/src/mailman/commands/docs/end.rst +++ b/src/mailman/commands/docs/end.rst @@ -13,8 +13,8 @@ processing email messages. The 'end' command takes no arguments. - >>> command.argument_description - '' + >>> print 'DESCRIPTION:', command.argument_description + DESCRIPTION: The command itself is fairly simple; it just stops command processing, and the message isn't even looked at. @@ -31,7 +31,7 @@ The 'stop' command is a synonym for 'end'. stop >>> print command.description An alias for 'end'. - >>> command.argument_description - '' + >>> print 'DESCRIPTION:', command.argument_description + DESCRIPTION: >>> print command.process(mlist, Message(), {}, (), None) ContinueProcessing.no diff --git a/src/mailman/commands/docs/info.rst b/src/mailman/commands/docs/info.rst index 7f69eada5..59801f234 100644 --- a/src/mailman/commands/docs/info.rst +++ b/src/mailman/commands/docs/info.rst @@ -37,6 +37,7 @@ By passing in the ``-o/--output`` option, you can print the info to a file. ... config file: .../test.cfg db url: ... + devmode: DISABLED REST root url: http://localhost:9001/3.0/ REST credentials: restadmin:restpass @@ -50,6 +51,7 @@ system paths that Mailman is using. ... [mailman] ... layout: fhs ... """) + >>> cleanups.append((config.pop, 'fhs')) >>> config.create_paths = True The File System Hierarchy layout is the same every by definition. @@ -73,9 +75,3 @@ The File System Hierarchy layout is the same every by definition. QUEUE_DIR = /var/spool/mailman TEMPLATE_DIR = .../mailman/templates VAR_DIR = /var/lib/mailman - - -Clean up -======== - - >>> config.pop('fhs') diff --git a/src/mailman/commands/eml_confirm.py b/src/mailman/commands/eml_confirm.py index 55619a503..c82dc64c3 100644 --- a/src/mailman/commands/eml_confirm.py +++ b/src/mailman/commands/eml_confirm.py @@ -17,7 +17,7 @@ """Module stuff.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -26,7 +26,7 @@ __all__ = [ from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.interfaces.command import ContinueProcessing, IEmailCommand @@ -34,11 +34,10 @@ from mailman.interfaces.registrar import IRegistrar +@implementer(IEmailCommand) class Confirm: """The email 'confirm' command.""" - implements(IEmailCommand) - name = 'confirm' argument_description = 'token' description = _('Confirm a subscription request.') @@ -48,7 +47,7 @@ class Confirm: """See `IEmailCommand`.""" # The token must be in the arguments. if len(arguments) == 0: - print >> results, _('No confirmation token found') + print(_('No confirmation token found'), file=results) return ContinueProcessing.no # Make sure we don't try to confirm the same token more than once. token = arguments[0] @@ -60,7 +59,7 @@ class Confirm: results.confirms = tokens succeeded = getUtility(IRegistrar).confirm(token) if succeeded: - print >> results, _('Confirmed') + print(_('Confirmed'), file=results) return ContinueProcessing.yes - print >> results, _('Confirmation token did not match') + print(_('Confirmation token did not match'), file=results) return ContinueProcessing.no diff --git a/src/mailman/commands/eml_echo.py b/src/mailman/commands/eml_echo.py index 06d5ee5e7..885edcbae 100644 --- a/src/mailman/commands/eml_echo.py +++ b/src/mailman/commands/eml_echo.py @@ -25,7 +25,7 @@ __all__ = [ ] -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.interfaces.command import ContinueProcessing, IEmailCommand @@ -35,9 +35,9 @@ SPACE = ' ' +@implementer(IEmailCommand) class Echo: """The email 'echo' command.""" - implements(IEmailCommand) name = 'echo' argument_description = '[args]' diff --git a/src/mailman/commands/eml_end.py b/src/mailman/commands/eml_end.py index 3cd70813c..32a024205 100644 --- a/src/mailman/commands/eml_end.py +++ b/src/mailman/commands/eml_end.py @@ -17,6 +17,8 @@ """The email commands 'end' and 'stop'.""" +from __future__ import absolute_import, print_function, unicode_literals + __metaclass__ = type __all__ = [ 'End', @@ -24,16 +26,16 @@ __all__ = [ ] -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.interfaces.command import ContinueProcessing, IEmailCommand +@implementer(IEmailCommand) class End: """The email 'end' command.""" - implements(IEmailCommand) name = 'end' argument_description = '' diff --git a/src/mailman/commands/eml_help.py b/src/mailman/commands/eml_help.py index 6fddb4ef3..a27717179 100644 --- a/src/mailman/commands/eml_help.py +++ b/src/mailman/commands/eml_help.py @@ -25,7 +25,7 @@ __all__ = [ ] -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.core.i18n import _ @@ -37,11 +37,10 @@ SPACE = ' ' +@implementer(IEmailCommand) class Help: """The email 'help' command.""" - implements(IEmailCommand) - name = 'help' argument_description = '[command]' description = _('Get help about available email commands.') @@ -58,14 +57,14 @@ class Help: command = config.commands[command_name] short_description = getattr( command, 'short_description', _('n/a')) - print(format.format(command.name, short_description), + print(format.format(command.name, short_description), file=results) return ContinueProcessing.yes elif len(arguments) == 1: command_name = arguments[0] command = config.commands.get(command_name) if command is None: - print(_('$self.name: no such command: $command_name'), + print(_('$self.name: no such command: $command_name'), file=results) return ContinueProcessing.no print('{0} {1}'.format(command.name, command.argument_description), diff --git a/src/mailman/commands/eml_membership.py b/src/mailman/commands/eml_membership.py index d6f7a47d9..63efbafca 100644 --- a/src/mailman/commands/eml_membership.py +++ b/src/mailman/commands/eml_membership.py @@ -30,7 +30,7 @@ __all__ = [ from email.utils import formataddr, parseaddr from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.interfaces.command import ContinueProcessing, IEmailCommand @@ -41,11 +41,10 @@ from mailman.interfaces.usermanager import IUserManager +@implementer(IEmailCommand) class Join: """The email 'join' command.""" - implements(IEmailCommand) - name = 'join' # XXX 2012-02-29 BAW: DeliveryMode.summary is not yet supported. argument_description = '[digest=<no|mime|plain>]' @@ -85,11 +84,11 @@ used. # Is this person already a member of the list? Search for all # matching memberships. members = getUtility(ISubscriptionService).find_members( - address, mlist.fqdn_listname, MemberRole.member) + address, mlist.list_id, MemberRole.member) if len(members) > 0: print(_('$person is already a member'), file=results) else: - getUtility(IRegistrar).register(mlist, address, + getUtility(IRegistrar).register(mlist, address, display_name, delivery_mode) print(_('Confirmation email sent to $person'), file=results) return ContinueProcessing.yes @@ -131,14 +130,13 @@ class Subscribe(Join): +@implementer(IEmailCommand) class Leave: """The email 'leave' command.""" - implements(IEmailCommand) - name = 'leave' argument_description = '' - description = _("""Leave this mailing list. + description = _("""Leave this mailing list. You may be asked to confirm your request.""") short_description = _('Leave this mailing list.') diff --git a/src/mailman/config/config.py b/src/mailman/config/config.py index 48c849148..f6c39fcec 100644 --- a/src/mailman/config/config.py +++ b/src/mailman/config/config.py @@ -32,13 +32,15 @@ from lazr.config import ConfigSchema, as_boolean from pkg_resources import resource_stream from string import Template from zope.component import getUtility -from zope.interface import Interface, implements +from zope.event import notify +from zope.interface import implementer import mailman.templates from mailman import version +from mailman.interfaces.configuration import ( + ConfigurationUpdatedEvent, IConfiguration) from mailman.interfaces.languages import ILanguageManager -from mailman.interfaces.styles import IStyleManager from mailman.utilities.filesystem import makedirs from mailman.utilities.modules import call_name @@ -47,16 +49,10 @@ SPACE = ' ' -class IConfiguration(Interface): - """Marker interface; used for adaptation in the REST API.""" - - - +@implementer(IConfiguration) class Configuration: """The core global configuration object.""" - implements(IConfiguration) - def __init__(self): self.switchboards = {} self.QFILE_SCHEMA_VERSION = version.QFILE_SCHEMA_VERSION @@ -71,6 +67,7 @@ class Configuration: self.handlers = {} self.pipelines = {} self.commands = {} + self.password_context = None def _clear(self): """Clear the cached configuration variables.""" @@ -119,26 +116,8 @@ class Configuration: """Perform post-processing after loading the configuration files.""" # Expand and set up all directories. self._expand_paths() - # Set up the switchboards. Import this here to avoid circular imports. - from mailman.core.switchboard import Switchboard - Switchboard.initialize() - # Set up all the languages. - languages = self._config.getByCategory('language', []) - language_manager = getUtility(ILanguageManager) - for language in languages: - if language.enabled: - code = language.name.split('.')[1] - language_manager.add( - code, language.charset, language.description) - # The default language must always be available. - assert self._config.mailman.default_language in language_manager, ( - 'System default language code not defined: %s' % - self._config.mailman.default_language) self.ensure_directories_exist() - getUtility(IStyleManager).populate() - # Set the default system language. - from mailman.core.i18n import _ - _.default = self.mailman.default_language + notify(ConfigurationUpdatedEvent(self)) def _expand_paths(self): """Expand all configuration paths.""" @@ -250,3 +229,9 @@ class Configuration: """Iterate over all the style configuration sections.""" for section in self._config.getByCategory('style', []): yield section + + @property + def language_configs(self): + """Iterate over all the language configuration sections.""" + for section in self._config.getByCategory('language', []): + yield section diff --git a/src/mailman/config/configure.zcml b/src/mailman/config/configure.zcml index 8c362c31b..ed85ae1a6 100644 --- a/src/mailman/config/configure.zcml +++ b/src/mailman/config/configure.zcml @@ -22,6 +22,20 @@ factory="mailman.model.requests.ListRequests" /> + <adapter + for="mailman.interfaces.database.IDatabase" + provides="mailman.interfaces.database.ITemporaryDatabase" + factory="mailman.database.sqlite.make_temporary" + name="sqlite" + /> + + <adapter + for="mailman.interfaces.database.IDatabase" + provides="mailman.interfaces.database.ITemporaryDatabase" + factory="mailman.database.postgresql.make_temporary" + name="postgres" + /> + <utility provides="mailman.interfaces.bans.IBanManager" factory="mailman.model.bans.BanManager" @@ -33,6 +47,24 @@ /> <utility + provides="mailman.interfaces.database.IDatabaseFactory" + factory="mailman.database.factory.DatabaseFactory" + name="production" + /> + + <utility + provides="mailman.interfaces.database.IDatabaseFactory" + factory="mailman.database.factory.DatabaseTestingFactory" + name="testing" + /> + + <utility + provides="mailman.interfaces.database.IDatabaseFactory" + factory="mailman.database.factory.DatabaseTemporaryFactory" + name="temporary" + /> + + <utility provides="mailman.interfaces.domain.IDomainManager" factory="mailman.model.domain.DomainManager" /> diff --git a/src/mailman/config/passlib.cfg b/src/mailman/config/passlib.cfg new file mode 100644 index 000000000..805f0fb11 --- /dev/null +++ b/src/mailman/config/passlib.cfg @@ -0,0 +1,10 @@ +[passlib] +# This is the output of passlibs.apps.custom_app_context.to_string(). +# See http://packages.python.org/passlib/index.html for details. +schemes = sha512_crypt, sha256_crypt +default = sha512_crypt +all__vary_rounds = 0.1 +sha256_crypt__min_rounds = 80000 +sha512_crypt__min_rounds = 60000 +admin__sha256_crypt__min_rounds = 160000 +admin__sha512_crypt__min_rounds = 120000 diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg index 00b8d9325..e36d33c10 100644 --- a/src/mailman/config/schema.cfg +++ b/src/mailman/config/schema.cfg @@ -27,11 +27,10 @@ # a human. site_owner: changeme@example.com -# This address is used as the from address whenever a message comes from some -# entity to which there is no natural reply recipient. Set this to a real -# human or to /dev/null. It will be appended with the host name of the list -# involved. This address must not bounce and it must not point to a Mailman -# process. +# This is the local-part of an email address used in the From field whenever a +# message comes from some entity to which there is no natural reply recipient. +# Mailman will append '@' and the host name of the list involved. This +# address must not bounce and it must not point to a Mailman process. noreply_address: noreply # The default language for this server. @@ -154,10 +153,13 @@ wait: 10s [passwords] -# The default scheme to use to encrypt new passwords. Existing passwords -# include the scheme that was used to encrypt them, so it's okay to change -# this after users have been added. -password_scheme: ssha +# Where can we find the passlib configuration file? The path can be either a +# file system path or a Python import path. If the value starts with python: +# then it is a Python import path, otherwise it is a file system path. File +# system paths must be absolute since no guarantees are made about the current +# working directory. Python paths should not include the trailing .cfg, which +# the file must end with. +path: python:mailman.config.passlib # When Mailman generates them, this is the default length of passwords. password_length: 8 diff --git a/src/mailman/config/tests/__init__.py b/src/mailman/config/tests/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/mailman/config/tests/__init__.py diff --git a/src/mailman/config/tests/test_configuration.py b/src/mailman/config/tests/test_configuration.py new file mode 100644 index 000000000..88e00cbb9 --- /dev/null +++ b/src/mailman/config/tests/test_configuration.py @@ -0,0 +1,53 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test the system-wide global configuration.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestConfiguration', + ] + + +import unittest + +from mailman.interfaces.configuration import ConfigurationUpdatedEvent +from mailman.testing.helpers import configuration, event_subscribers +from mailman.testing.layers import ConfigLayer + + + +class TestConfiguration(unittest.TestCase): + layer = ConfigLayer + + def test_push_and_pop_trigger_events(self): + # Pushing a new configuration onto the stack triggers a + # post-processing event. + events = [] + def on_event(event): + if isinstance(event, ConfigurationUpdatedEvent): + # Record both the event and the top overlay. + events.append(event.config.overlays[0].name) + with event_subscribers(on_event): + with configuration('test', _configname='my test'): + pass + # There should be two pushed configuration names on the list now, one + # for the push leaving 'my test' on the top of the stack, and one for + # the pop, leaving the ConfigLayer's 'test config' on top. + self.assertEqual(events, ['my test', 'test config']) diff --git a/src/mailman/core/constants.py b/src/mailman/core/constants.py index 02d46a088..4562f4c74 100644 --- a/src/mailman/core/constants.py +++ b/src/mailman/core/constants.py @@ -17,7 +17,7 @@ """Various constants and enumerations.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -26,7 +26,7 @@ __all__ = [ from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.interfaces.languages import ILanguageManager @@ -35,11 +35,10 @@ from mailman.interfaces.preferences import IPreferences +@implementer(IPreferences) class SystemDefaultPreferences: """The default system preferences.""" - implements(IPreferences) - acknowledge_posts = False hide_address = True receive_list_copy = True diff --git a/src/mailman/core/i18n.py b/src/mailman/core/i18n.py index 2d7c382c7..73453ae65 100644 --- a/src/mailman/core/i18n.py +++ b/src/mailman/core/i18n.py @@ -31,7 +31,7 @@ import time from flufl.i18n import PackageStrategy, registry import mailman.messages - +from mailman.interfaces.configuration import ConfigurationUpdatedEvent _ = None @@ -113,3 +113,9 @@ def ctime(date): wday = daysofweek[wday] mon = months[mon] return _('$wday $mon $day $hh:$mm:$ss $tzname $year') + + + +def handle_ConfigurationUpdatedEvent(event): + if isinstance(event, ConfigurationUpdatedEvent): + _.default = event.config.mailman.default_language diff --git a/src/mailman/core/initialize.py b/src/mailman/core/initialize.py index 389a45f3b..eb8787ad2 100644 --- a/src/mailman/core/initialize.py +++ b/src/mailman/core/initialize.py @@ -24,7 +24,7 @@ line argument parsing, since some of the initialization behavior is controlled by the command line arguments. """ -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -40,13 +40,13 @@ import os import sys from pkg_resources import resource_string +from zope.component import getUtility from zope.configuration import xmlconfig -from zope.interface.verify import verifyObject import mailman.config.config import mailman.core.logging -from mailman.interfaces.database import IDatabase +from mailman.interfaces.database import IDatabaseFactory from mailman.utilities.modules import call_name # The test infrastructure uses this to prevent the search and loading of any @@ -110,6 +110,10 @@ def initialize_1(config_path=None): # o-rwx although I think in most cases it doesn't hurt if other can read # or write the files. os.umask(007) + # Initialize configuration event subscribers. This must be done before + # setting up the configuration system. + from mailman.app.events import initialize as initialize_events + initialize_events() # config_path will be set if the command line argument -C is given. That # case overrides all others. When not given on the command line, the # configuration file is searched for in the file system. @@ -121,9 +125,10 @@ def initialize_1(config_path=None): mailman.config.config.load(config_path) -def initialize_2(debug=False, propagate_logs=None): +def initialize_2(debug=False, propagate_logs=None, testing=False): """Second initialization step. + * Database * Logging * Pre-hook * Rules @@ -144,15 +149,11 @@ def initialize_2(debug=False, propagate_logs=None): call_name(config.mailman.pre_hook) # Instantiate the database class, ensure that it's of the right type, and # initialize it. Then stash the object on our configuration object. - database_class = config.database['class'] - database = call_name(database_class) - verifyObject(IDatabase, database) - database.initialize(debug) - config.db = database + utility_name = ('testing' if testing else 'production') + config.db = getUtility(IDatabaseFactory, utility_name).create() # Initialize the rules and chains. Do the imports here so as to avoid # circular imports. from mailman.app.commands import initialize as initialize_commands - from mailman.app.events import initialize as initialize_events from mailman.core.chains import initialize as initialize_chains from mailman.core.pipelines import initialize as initialize_pipelines from mailman.core.rules import initialize as initialize_rules @@ -161,7 +162,6 @@ def initialize_2(debug=False, propagate_logs=None): initialize_chains() initialize_pipelines() initialize_commands() - initialize_events() def initialize_3(): diff --git a/src/mailman/core/pipelines.py b/src/mailman/core/pipelines.py index 25bb68030..972417c2c 100644 --- a/src/mailman/core/pipelines.py +++ b/src/mailman/core/pipelines.py @@ -32,7 +32,7 @@ __all__ = [ import logging -from zope.interface import implements +from zope.interface import implementer from zope.interface.verify import verifyObject from mailman.app.bounces import bounce_message @@ -75,11 +75,10 @@ def process(mlist, msg, msgdata, pipeline_name='built-in'): +@implementer(IPipeline) class BasePipeline: """Base pipeline implementation.""" - implements(IPipeline) - _default_handlers = () def __init__(self): diff --git a/src/mailman/core/runner.py b/src/mailman/core/runner.py index e86741c41..a79f19fbc 100644 --- a/src/mailman/core/runner.py +++ b/src/mailman/core/runner.py @@ -17,7 +17,7 @@ """The process runner base class.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -32,14 +32,15 @@ import traceback from cStringIO import StringIO from lazr.config import as_boolean, as_timedelta from zope.component import getUtility -from zope.interface import implements +from zope.event import notify +from zope.interface import implementer from mailman.config import config from mailman.core.i18n import _ from mailman.core.switchboard import Switchboard from mailman.interfaces.languages import ILanguageManager from mailman.interfaces.listmanager import IListManager -from mailman.interfaces.runner import IRunner +from mailman.interfaces.runner import IRunner, RunnerCrashEvent from mailman.utilities.string import expand @@ -48,9 +49,8 @@ elog = logging.getLogger('mailman.error') +@implementer(IRunner) class Runner: - implements(IRunner) - intercept_signals = True def __init__(self, name, slice=None): @@ -217,7 +217,12 @@ class Runner: language = mlist.preferred_language with _.using(language.code): msgdata['lang'] = language.code - keepqueued = self._dispose(mlist, msg, msgdata) + try: + keepqueued = self._dispose(mlist, msg, msgdata) + except Exception as error: + # Trigger the Zope event and re-raise + notify(RunnerCrashEvent(self, mlist, msg, msgdata, error)) + raise if keepqueued: self.switchboard.enqueue(msg, msgdata) diff --git a/src/mailman/core/switchboard.py b/src/mailman/core/switchboard.py index 7cab4f4ad..1f16cb5fb 100644 --- a/src/mailman/core/switchboard.py +++ b/src/mailman/core/switchboard.py @@ -24,11 +24,12 @@ written. First, the message is written to the pickle, then the metadata dictionary is written. """ -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ 'Switchboard', + 'handle_ConfigurationUpdatedEvent', ] @@ -40,10 +41,11 @@ import cPickle import hashlib import logging -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.email.message import Message +from mailman.interfaces.configuration import ConfigurationUpdatedEvent from mailman.interfaces.switchboard import ISwitchboard from mailman.utilities.filesystem import makedirs from mailman.utilities.string import expand @@ -63,20 +65,9 @@ elog = logging.getLogger('mailman.error') +@implementer(ISwitchboard) class Switchboard: - implements(ISwitchboard) - - @staticmethod - def initialize(): - """Initialize the global switchboards for input/output.""" - for conf in config.runner_configs: - name = conf.name.split('.')[-1] - assert name not in config.switchboards, ( - 'Duplicate runner name: {0}'.format(name)) - substitutions = config.paths - substitutions['name'] = name - path = expand(conf.path, substitutions) - config.switchboards[name] = Switchboard(name, path) + """See `ISwitchboard`.""" def __init__(self, name, queue_directory, slice=None, numslices=1, recover=False): @@ -263,3 +254,19 @@ class Switchboard: self.finish(filebase, preserve=True) else: os.rename(src, dst) + + + +def handle_ConfigurationUpdatedEvent(event): + """Initialize the global switchboards for input/output.""" + if not isinstance(event, ConfigurationUpdatedEvent): + return + config = event.config + for conf in config.runner_configs: + name = conf.name.split('.')[-1] + assert name not in config.switchboards, ( + 'Duplicate runner name: {0}'.format(name)) + substitutions = config.paths + substitutions['name'] = name + path = expand(conf.path, substitutions) + config.switchboards[name] = Switchboard(name, path) diff --git a/src/mailman/core/system.py b/src/mailman/core/system.py index ce66761a7..b29567827 100644 --- a/src/mailman/core/system.py +++ b/src/mailman/core/system.py @@ -17,7 +17,7 @@ """System information.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -27,15 +27,16 @@ __all__ = [ import sys -from zope.interface import implements +from zope.interface import implementer from mailman import version from mailman.interfaces.system import ISystem +@implementer(ISystem) class System: - implements(ISystem) + """See `ISystem`.""" @property def mailman_version(self): diff --git a/src/mailman/core/tests/test_pipelines.py b/src/mailman/core/tests/test_pipelines.py index 8f851de95..8e76cf033 100644 --- a/src/mailman/core/tests/test_pipelines.py +++ b/src/mailman/core/tests/test_pipelines.py @@ -29,7 +29,7 @@ __all__ = [ import unittest from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer from mailman.app.lifecycle import create_list from mailman.config import config @@ -48,24 +48,24 @@ from mailman.testing.layers import ConfigLayer +@implementer(IHandler) class DiscardingHandler: - implements(IHandler) name = 'discarding' def process(self, mlist, msg, msgdata): raise DiscardMessage('by test handler') +@implementer(IHandler) class RejectHandler: - implements(IHandler) name = 'rejecting' def process(self, mlist, msg, msgdata): raise RejectMessage('by test handler') +@implementer(IPipeline) class DiscardingPipeline: - implements(IPipeline) name = 'test-discarding' description = 'Discarding test pipeline' @@ -73,8 +73,8 @@ class DiscardingPipeline: yield DiscardingHandler() +@implementer(IPipeline) class RejectingPipeline: - implements(IPipeline) name = 'test-rejecting' description = 'Rejectinging test pipeline' diff --git a/src/mailman/core/tests/test_runner.py b/src/mailman/core/tests/test_runner.py new file mode 100644 index 000000000..ad2548adc --- /dev/null +++ b/src/mailman/core/tests/test_runner.py @@ -0,0 +1,89 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test some Runner base class behavior.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestRunner', + ] + + +import unittest + +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.core.runner import Runner +from mailman.interfaces.runner import RunnerCrashEvent +from mailman.testing.helpers import ( + configuration, event_subscribers, get_queue_messages, + make_testable_runner, specialized_message_from_string as mfs) +from mailman.testing.layers import ConfigLayer + + + +class CrashingRunner(Runner): + def _dispose(self, mlist, msg, msgdata): + raise RuntimeError('borked') + + + +class TestRunner(unittest.TestCase): + """Test the Runner base class behavior.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._events = [] + + def _got_event(self, event): + self._events.append(event) + + @configuration('runner.crashing', + **{'class': 'mailman.core.tests.CrashingRunner'}) + def test_crash_event(self): + runner = make_testable_runner(CrashingRunner, 'in') + # When an exception occurs in Runner._process_one_file(), a zope.event + # gets triggered containing the exception object. + msg = mfs("""\ +From: anne@example.com +To: test@example.com +Message-ID: <ant> + +""") + config.switchboards['in'].enqueue(msg, listname='test@example.com') + with event_subscribers(self._got_event): + runner.run() + # We should now have exactly one event, which will contain the + # exception, plus additional metadata containing the mailing list, + # message, and metadata. + self.assertEqual(len(self._events), 1) + event = self._events[0] + self.assertTrue(isinstance(event, RunnerCrashEvent)) + self.assertEqual(event.mailing_list, self._mlist) + self.assertEqual(event.message['message-id'], '<ant>') + self.assertEqual(event.metadata['listname'], 'test@example.com') + self.assertTrue(isinstance(event.error, RuntimeError)) + self.assertEqual(event.error.message, 'borked') + self.assertTrue(isinstance(event.runner, CrashingRunner)) + # The message should also have ended up in the shunt queue. + shunted = get_queue_messages('shunt') + self.assertEqual(len(shunted), 1) + self.assertEqual(shunted[0].msg['message-id'], '<ant>') diff --git a/src/mailman/database/base.py b/src/mailman/database/base.py index a69e99395..694ad9ec3 100644 --- a/src/mailman/database/base.py +++ b/src/mailman/database/base.py @@ -15,7 +15,7 @@ # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see <http://www.gnu.org/licenses/>. -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -32,7 +32,7 @@ from lazr.config import as_boolean from pkg_resources import resource_listdir, resource_string from storm.cache import GenerationalCache from storm.locals import create_database, Store -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.interfaces.database import IDatabase @@ -45,6 +45,7 @@ NL = '\n' +@implementer(IDatabase) class StormBaseDatabase: """The database base class for use with the Storm ORM. @@ -55,8 +56,6 @@ class StormBaseDatabase: # classes. TAG = '' - implements(IDatabase) - def __init__(self): self.url = None self.store = None @@ -143,17 +142,21 @@ class StormBaseDatabase: database.DEBUG = (as_boolean(config.database.debug) if debug is None else debug) self.store = store - self.load_migrations() store.commit() - def load_migrations(self): - """Load all not-yet loaded migrations.""" + def load_migrations(self, until=None): + """Load schema migrations. + + :param until: Load only the migrations up to the specified timestamp. + With default value of None, load all migrations. + :type until: string + """ migrations_path = config.database.migrations_path if '.' in migrations_path: parent, dot, child = migrations_path.rpartition('.') else: parent = migrations_path - child ='' + child = '' # If the database does not yet exist, load the base schema. filenames = sorted(resource_listdir(parent, child)) # Find out which schema migrations have already been loaded. @@ -169,16 +172,40 @@ class StormBaseDatabase: parts = module_fn.split('_') if len(parts) < 2: continue - version = parts[1] + version = parts[1].strip() + if len(version) == 0: + # Not a schema migration file. + continue if version in versions: - # This one is already loaded. + log.debug('already migrated to %s', version) continue + if until is not None and version > until: + # We're done. + break module_path = migrations_path + '.' + module_fn __import__(module_path) upgrade = getattr(sys.modules[module_path], 'upgrade', None) if upgrade is None: continue + log.debug('migrating db to %s: %s', version, module_path) upgrade(self, self.store, version, module_path) + self.commit() + + def load_sql(self, store, sql): + """Load the given SQL into the store. + + :param store: The Storm store to load the schema into. + :type store: storm.locals.Store` + :param sql: The possibly multi-line SQL to load. + :type sql: string + """ + # Discard all blank and comment lines. + lines = (line for line in sql.splitlines() + if line.strip() != '' and line.strip()[:2] != '--') + sql = NL.join(lines) + for statement in sql.split(';'): + if statement.strip() != '': + store.execute(statement + ';') def load_schema(self, store, version, filename, module_path): """Load the schema from a file. @@ -187,7 +214,7 @@ class StormBaseDatabase: :param store: The Storm store to load the schema into. :type store: storm.locals.Store` - :param version: The schema version identifier of the form + :param version: The schema version identifier of the form YYYYMMDDHHMMSS. :type version: string :param filename: The file name containing the schema to load. Pass @@ -200,22 +227,10 @@ class StormBaseDatabase: """ if filename is not None: contents = resource_string('mailman.database.schema', filename) - # Discard all blank and comment lines. - lines = (line for line in contents.splitlines() - if line.strip() != '' and line.strip()[:2] != '--') - sql = NL.join(lines) - for statement in sql.split(';'): - if statement.strip() != '': - store.execute(statement + ';') + self.load_sql(store, contents) # Add a marker that indicates the migration version being applied. store.add(Version(component='schema', version=version)) - # Add a marker so that the module name can be found later. This is - # used by the test suite to reset the database between tests. - store.add(Version(component=version, version=module_path)) - def _reset(self): - """See `IDatabase`.""" - from mailman.database.model import ModelMeta - self.store.rollback() - ModelMeta._reset(self.store) - self.store.commit() + @staticmethod + def _make_temporary(): + raise NotImplementedError diff --git a/src/mailman/database/docs/migration.rst b/src/mailman/database/docs/migration.rst index 9897b1ef2..6216f9bbc 100644 --- a/src/mailman/database/docs/migration.rst +++ b/src/mailman/database/docs/migration.rst @@ -24,17 +24,18 @@ Migrations are applied automatically when Mailman starts up, but can also be applied at any time by calling in the API directly. Once applied, a migration's version string is registered so it will not be applied again. -We see that the base migration is already applied. +We see that the base migration, as well as subsequent standard migrations, are +already applied. >>> from mailman.model.version import Version >>> results = config.db.store.find(Version, component='schema') >>> results.count() - 1 - >>> base = results.one() - >>> print base.component - schema - >>> print base.version + 2 + >>> versions = sorted(result.version for result in results) + >>> for version in versions: + ... print version 00000000000000 + 20120407000000 Migrations @@ -55,6 +56,14 @@ specified in the configuration file. ... migrations_path: migrations ... """) +.. Clean this up at the end of the doctest. + >>> def cleanup(): + ... import shutil + ... from mailman.config import config + ... config.pop('migrations') + ... shutil.rmtree(tempdir) + >>> cleanups.append(cleanup) + Here is an example migrations module. The key part of this interface is the ``upgrade()`` method, which takes four arguments: @@ -69,7 +78,7 @@ This migration module just adds a marker to the `version` table. >>> with open(os.path.join(path, '__init__.py'), 'w') as fp: ... pass - >>> with open(os.path.join(path, 'mm_20120211000000.py'), 'w') as fp: + >>> with open(os.path.join(path, 'mm_20129999000000.py'), 'w') as fp: ... print >> fp, """ ... from __future__ import unicode_literals ... from mailman.model.version import Version @@ -86,14 +95,15 @@ This will load the new migration, since it hasn't been loaded before. >>> for result in sorted(result.version for result in results): ... print result 00000000000000 - 20120211000000 + 20120407000000 + 20129999000000 >>> test = config.db.store.find(Version, component='test').one() >>> print test.version - 20120211000000 + 20129999000000 Migrations will only be loaded once. - >>> with open(os.path.join(path, 'mm_20120211000001.py'), 'w') as fp: + >>> with open(os.path.join(path, 'mm_20129999000001.py'), 'w') as fp: ... print >> fp, """ ... from __future__ import unicode_literals ... from mailman.model.version import Version @@ -115,13 +125,14 @@ The first time we load this new migration, we'll get the 801 marker. >>> for result in sorted(result.version for result in results): ... print result 00000000000000 - 20120211000000 - 20120211000001 + 20120407000000 + 20129999000000 + 20129999000001 >>> test = config.db.store.find(Version, component='test') >>> for marker in sorted(marker.version for marker in test): ... print marker 00000000000801 - 20120211000000 + 20129999000000 We do not get an 802 marker because the migration has already been loaded. @@ -130,17 +141,56 @@ We do not get an 802 marker because the migration has already been loaded. >>> for result in sorted(result.version for result in results): ... print result 00000000000000 - 20120211000000 - 20120211000001 + 20120407000000 + 20129999000000 + 20129999000001 >>> test = config.db.store.find(Version, component='test') >>> for marker in sorted(marker.version for marker in test): ... print marker 00000000000801 - 20120211000000 + 20129999000000 + + +Partial upgrades +================ + +It's possible (mostly for testing purposes) to only do a partial upgrade, by +providing a timestamp to `load_migrations()`. To demonstrate this, we add two +additional migrations, intended to be applied in sequential order. + + >>> from shutil import copyfile + >>> from mailman.testing.helpers import chdir + >>> with chdir(path): + ... copyfile('mm_20129999000000.py', 'mm_20129999000002.py') + ... copyfile('mm_20129999000000.py', 'mm_20129999000003.py') + ... copyfile('mm_20129999000000.py', 'mm_20129999000004.py') + +Now, only migrate to the ...03 timestamp. + + >>> config.db.load_migrations('20129999000003') + +You'll notice that the ...04 version is not present. + + >>> results = config.db.store.find(Version, component='schema') + >>> for result in sorted(result.version for result in results): + ... print result + 00000000000000 + 20120407000000 + 20129999000000 + 20129999000001 + 20129999000002 + 20129999000003 + -.. Clean up the temporary directory:: +.. cleanup: + Because the Version table holds schema migration data, it will not be + cleaned up by the standard test suite. This is generally not a problem + for SQLite since each test gets a new database file, but for PostgreSQL, + this will cause migration.rst to fail on subsequent runs. So let's just + clean up the database explicitly. - >>> config.pop('migrations') - >>> sys.path.remove(tempdir) - >>> import shutil - >>> shutil.rmtree(tempdir) + >>> results = config.db.store.execute(""" + ... DELETE FROM version WHERE version.version >= '201299990000' + ... OR version.component = 'test'; + ... """) + >>> config.db.commit() diff --git a/src/mailman/database/factory.py b/src/mailman/database/factory.py new file mode 100644 index 000000000..127c4aaeb --- /dev/null +++ b/src/mailman/database/factory.py @@ -0,0 +1,101 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Database factory.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'DatabaseFactory', + 'DatabaseTemporaryFactory', + 'DatabaseTestingFactory', + ] + + +import types + +from zope.component import getAdapter +from zope.interface import implementer +from zope.interface.verify import verifyObject + +from mailman.config import config +from mailman.interfaces.database import ( + IDatabase, IDatabaseFactory, ITemporaryDatabase) +from mailman.utilities.modules import call_name + + + +@implementer(IDatabaseFactory) +class DatabaseFactory: + """Create a new database.""" + + @staticmethod + def create(): + """See `IDatabaseFactory`.""" + database_class = config.database['class'] + database = call_name(database_class) + verifyObject(IDatabase, database) + database.initialize() + database.load_migrations() + database.commit() + return database + + + +def _reset(self): + """See `IDatabase`.""" + from mailman.database.model import ModelMeta + self.store.rollback() + self._pre_reset(self.store) + ModelMeta._reset(self.store) + self._post_reset(self.store) + self.store.commit() + + +@implementer(IDatabaseFactory) +class DatabaseTestingFactory: + """Create a new database for testing.""" + + @staticmethod + def create(): + """See `IDatabaseFactory`.""" + database_class = config.database['class'] + database = call_name(database_class) + verifyObject(IDatabase, database) + database.initialize() + database.load_migrations() + database.commit() + # Make _reset() a bound method of the database instance. + database._reset = types.MethodType(_reset, database) + return database + + + +@implementer(IDatabaseFactory) +class DatabaseTemporaryFactory: + """Create a temporary database for some of the migration tests.""" + + @staticmethod + def create(): + """See `IDatabaseFactory`.""" + database_class_name = config.database['class'] + database = call_name(database_class_name) + verifyObject(IDatabase, database) + adapted_database = getAdapter( + database, ITemporaryDatabase, database.TAG) + return adapted_database diff --git a/src/mailman/database/model.py b/src/mailman/database/model.py index c45517c9b..58d5942a4 100644 --- a/src/mailman/database/model.py +++ b/src/mailman/database/model.py @@ -17,7 +17,7 @@ """Base class for all database classes.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -25,8 +25,6 @@ __all__ = [ ] -import sys - from operator import attrgetter from storm.properties import PropertyPublisherMeta @@ -44,46 +42,24 @@ class ModelMeta(PropertyPublisherMeta): self.__storm_table__ = name.lower() super(ModelMeta, self).__init__(name, bases, dict) # Register the model class so that it can be more easily cleared. - # This is required by the test framework. - if name == 'Model': - return - ModelMeta._class_registry.add(self) + # This is required by the test framework so that the corresponding + # table can be reset between tests. + # + # The PRESERVE flag indicates whether the table should be reset or + # not. We have to handle the actual Model base class explicitly + # because it does not correspond to a table in the database. + if not getattr(self, 'PRESERVE', False) and name != 'Model': + ModelMeta._class_registry.add(self) @staticmethod def _reset(store): from mailman.config import config - from mailman.model.version import Version config.db._pre_reset(store) - # Give each schema migration a chance to do its pre-reset. See below - # for calling its post reset too. - versions = sorted(version.version for version in - store.find(Version, component='schema')) - migrations = {} - for version in versions: - # We have to give the migrations module that loaded this version a - # chance to do both pre- and post-reset operations. The following - # find the actual the module path for the migration. See - # StormBaseDatabase.load_schema(). - migration = store.find(Version, component=version).one() - if migration is None: - continue - migrations[version] = module_path = migration.version - module = sys.modules[module_path] - pre_reset = getattr(module, 'pre_reset', None) - if pre_reset is not None: - pre_reset(store) # Make sure this is deterministic, by sorting on the storm table name. classes = sorted(ModelMeta._class_registry, key=attrgetter('__storm_table__')) for model_class in classes: store.find(model_class).remove() - # Now give each migration a chance to do post-reset operations. - for version in versions: - module = sys.modules[migrations[version]] - post_reset = getattr(module, 'post_reset', None) - if post_reset is not None: - post_reset(store) - config.db._post_reset(store) diff --git a/src/mailman/database/postgresql.py b/src/mailman/database/postgresql.py index 988f7a1af..49188148f 100644 --- a/src/mailman/database/postgresql.py +++ b/src/mailman/database/postgresql.py @@ -17,17 +17,23 @@ """PostgreSQL database support.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ 'PostgreSQLDatabase', + 'make_temporary', ] +import types + +from functools import partial from operator import attrgetter +from urlparse import urlsplit, urlunsplit from mailman.database.base import StormBaseDatabase +from mailman.testing.helpers import configuration @@ -40,8 +46,8 @@ class PostgreSQLDatabase(StormBaseDatabase): """See `BaseDatabase`.""" table_query = ('SELECT table_name FROM information_schema.tables ' "WHERE table_schema = 'public'") - table_names = set(item[0] for item in - store.execute(table_query)) + results = store.execute(table_query) + table_names = set(item[0] for item in results) return 'version' in table_names def _post_reset(self, store): @@ -63,3 +69,37 @@ class PostgreSQLDatabase(StormBaseDatabase): max("id") IS NOT null) FROM "{0}"; """.format(model_class.__storm_table__)) + + + +# Test suite adapter for ITemporaryDatabase. + +def _cleanup(self, store, tempdb_name): + from mailman.config import config + store.rollback() + store.close() + # From the original database connection, drop the now unused database. + config.db.store.execute('DROP DATABASE {0}'.format(tempdb_name)) + + +def make_temporary(database): + """Adapts by monkey patching an existing PostgreSQL IDatabase.""" + from mailman.config import config + parts = urlsplit(config.database.url) + assert parts.scheme == 'postgres' + new_parts = list(parts) + new_parts[2] = '/mmtest' + url = urlunsplit(new_parts) + # Use the existing database connection to create a new testing + # database. + config.db.store.execute('ABORT;') + config.db.store.execute('CREATE DATABASE mmtest;') + with configuration('database', url=url): + database.initialize() + database._cleanup = types.MethodType( + partial(_cleanup, store=database.store, tempdb_name='mmtest'), + database) + # bool column values in PostgreSQL. + database.FALSE = 'False' + database.TRUE = 'True' + return database diff --git a/src/mailman/database/schema/mm_00000000000000_base.py b/src/mailman/database/schema/mm_00000000000000_base.py index d703088d6..0dcd28edd 100644 --- a/src/mailman/database/schema/mm_00000000000000_base.py +++ b/src/mailman/database/schema/mm_00000000000000_base.py @@ -22,34 +22,14 @@ from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ 'upgrade', - 'post_reset', - 'pre_reset', ] -_migration_path = None VERSION = '00000000000000' +_helper = None def upgrade(database, store, version, module_path): filename = '{0}.sql'.format(database.TAG) database.load_schema(store, version, filename, module_path) - - -def pre_reset(store): - global _migration_path - # Save the entry in the Version table for the test suite reset. This will - # be restored below. - from mailman.model.version import Version - result = store.find(Version, component=VERSION).one() - # Yes, we abuse this field. - _migration_path = result.version - - -def post_reset(store): - from mailman.model.version import Version - # We need to preserve the Version table entry for this migration, since - # its existence defines the fact that the tables have been loaded. - store.add(Version(component='schema', version=VERSION)) - store.add(Version(component=VERSION, version=_migration_path)) diff --git a/src/mailman/database/schema/mm_20120407000000.py b/src/mailman/database/schema/mm_20120407000000.py new file mode 100644 index 000000000..d6e647c1a --- /dev/null +++ b/src/mailman/database/schema/mm_20120407000000.py @@ -0,0 +1,214 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""3.0b1 -> 3.0b2 schema migrations. + +All column changes are in the `mailinglist` table. + +* Renames: + - news_prefix_subject_too -> nntp_prefix_subject_too + - news_moderation -> newsgroup_moderation + +* Collapsing: + - archive, archive_private -> archive_policy + +* Remove: + - archive_volume_frequency + - generic_nonmember_action + - nntp_host + +* Added: + - list_id + +* Changes: + member.mailing_list holds the list_id not the fqdn_listname + +See https://bugs.launchpad.net/mailman/+bug/971013 for details. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'upgrade', + ] + + +from mailman.interfaces.archiver import ArchivePolicy + + +VERSION = '20120407000000' +_helper = None + + + +def upgrade(database, store, version, module_path): + if database.TAG == 'sqlite': + upgrade_sqlite(database, store, version, module_path) + else: + upgrade_postgres(database, store, version, module_path) + + + +def archive_policy(archive, archive_private): + """Convert archive and archive_private to archive_policy.""" + if archive == 0: + return int(ArchivePolicy.never) + elif archive_private == 1: + return int(ArchivePolicy.private) + else: + return int(ArchivePolicy.public) + + + +def upgrade_sqlite(database, store, version, module_path): + # Load the first part of the migration. This creates a temporary table to + # hold the new mailinglist table columns. The problem is that some of the + # changes must be performed in Python, so after the first part is loaded, + # we do the Python changes, drop the old mailing list table, and then + # rename the temporary table to its place. + database.load_schema( + store, version, 'sqlite_{0}_01.sql'.format(version), module_path) + results = store.execute(""" + SELECT id, include_list_post_header, + news_prefix_subject_too, news_moderation, + archive, archive_private, list_name, mail_host + FROM mailinglist; + """) + for value in results: + (id, list_post, + news_prefix, news_moderation, + archive, archive_private, + list_name, mail_host) = value + # Figure out what the new archive_policy column value should be. + list_id = '{0}.{1}'.format(list_name, mail_host) + fqdn_listname = '{0}@{1}'.format(list_name, mail_host) + store.execute(""" + UPDATE ml_backup SET + allow_list_posts = {0}, + newsgroup_moderation = {1}, + nntp_prefix_subject_too = {2}, + archive_policy = {3}, + list_id = '{4}' + WHERE id = {5}; + """.format( + list_post, + news_moderation, + news_prefix, + archive_policy(archive, archive_private), + list_id, + id)) + # Also update the member.mailing_list column to hold the list_id + # instead of the fqdn_listname. + store.execute(""" + UPDATE member SET + mailing_list = '{0}' + WHERE mailing_list = '{1}'; + """.format(list_id, fqdn_listname)) + # Pivot the backup table to the real thing. + store.execute('DROP TABLE mailinglist;') + store.execute('ALTER TABLE ml_backup RENAME TO mailinglist;') + # Now add some indexes that were previously missing. + store.execute( + 'CREATE INDEX ix_mailinglist_list_id ON mailinglist (list_id);') + store.execute( + 'CREATE INDEX ix_mailinglist_fqdn_listname ' + 'ON mailinglist (list_name, mail_host);') + # Now, do the member table. + results = store.execute('SELECT id, mailing_list FROM member;') + for id, mailing_list in results: + list_name, at, mail_host = mailing_list.partition('@') + if at == '': + list_id = mailing_list + else: + list_id = '{0}.{1}'.format(list_name, mail_host) + store.execute(""" + UPDATE mem_backup SET list_id = '{0}' + WHERE id = {1}; + """.format(list_id, id)) + # Pivot the backup table to the real thing. + store.execute('DROP TABLE member;') + store.execute('ALTER TABLE mem_backup RENAME TO member;') + + + +def upgrade_postgres(database, store, version, module_path): + # Get the old values from the mailinglist table. + results = store.execute(""" + SELECT id, archive, archive_private, list_name, mail_host + FROM mailinglist; + """) + # Do the simple renames first. + store.execute(""" + ALTER TABLE mailinglist + RENAME COLUMN news_prefix_subject_too TO nntp_prefix_subject_too; + """) + store.execute(""" + ALTER TABLE mailinglist + RENAME COLUMN news_moderation TO newsgroup_moderation; + """) + store.execute(""" + ALTER TABLE mailinglist + RENAME COLUMN include_list_post_header TO allow_list_posts; + """) + # Do the easy column drops next. + for column in ('archive_volume_frequency', + 'generic_nonmember_action', + 'nntp_host'): + store.execute( + 'ALTER TABLE mailinglist DROP COLUMN {0};'.format(column)) + # Now do the trickier collapsing of values. Add the new columns. + store.execute('ALTER TABLE mailinglist ADD COLUMN archive_policy INTEGER;') + store.execute('ALTER TABLE mailinglist ADD COLUMN list_id TEXT;') + # Query the database for the old values of archive and archive_private in + # each column. Then loop through all the results and update the new + # archive_policy from the old values. + for value in results: + id, archive, archive_private, list_name, mail_host = value + list_id = '{0}.{1}'.format(list_name, mail_host) + store.execute(""" + UPDATE mailinglist SET + archive_policy = {0}, + list_id = '{1}' + WHERE id = {2}; + """.format(archive_policy(archive, archive_private), list_id, id)) + # Now drop the old columns. + for column in ('archive', 'archive_private'): + store.execute( + 'ALTER TABLE mailinglist DROP COLUMN {0};'.format(column)) + # Now add some indexes that were previously missing. + store.execute( + 'CREATE INDEX ix_mailinglist_list_id ON mailinglist (list_id);') + store.execute( + 'CREATE INDEX ix_mailinglist_fqdn_listname ' + 'ON mailinglist (list_name, mail_host);') + # Now, do the member table. + results = store.execute('SELECT id, mailing_list FROM member;') + store.execute('ALTER TABLE member ADD COLUMN list_id TEXT;') + for id, mailing_list in results: + list_name, at, mail_host = mailing_list.partition('@') + if at == '': + list_id = mailing_list + else: + list_id = '{0}.{1}'.format(list_name, mail_host) + store.execute(""" + UPDATE member SET list_id = '{0}' + WHERE id = {1}; + """.format(list_id, id)) + store.execute('ALTER TABLE member DROP COLUMN mailing_list;') + # Record the migration in the version table. + database.load_schema(store, version, None, module_path) diff --git a/src/mailman/database/schema/postgres.sql b/src/mailman/database/schema/postgres.sql index 2e9ba249f..0e97a4332 100644 --- a/src/mailman/database/schema/postgres.sql +++ b/src/mailman/database/schema/postgres.sql @@ -110,7 +110,8 @@ CREATE TABLE mailinglist ( topics_enabled BOOLEAN, unsubscribe_policy INTEGER, welcome_message_uri TEXT, - moderation_callback TEXT, + -- This was accidentally added by the PostgreSQL porter. + -- moderation_callback TEXT, PRIMARY KEY (id) ); diff --git a/src/mailman/database/schema/sqlite.sql b/src/mailman/database/schema/sqlite.sql index e6211bf53..e2b2d3814 100644 --- a/src/mailman/database/schema/sqlite.sql +++ b/src/mailman/database/schema/sqlite.sql @@ -1,3 +1,6 @@ +-- THIS FILE HAS BEEN FROZEN AS OF 3.0b1 +-- SEE THE SCHEMA MIGRATIONS FOR DIFFERENCES. + PRAGMA foreign_keys = ON; CREATE TABLE _request ( diff --git a/src/mailman/database/schema/sqlite_20120407000000_01.sql b/src/mailman/database/schema/sqlite_20120407000000_01.sql new file mode 100644 index 000000000..b93e214c4 --- /dev/null +++ b/src/mailman/database/schema/sqlite_20120407000000_01.sql @@ -0,0 +1,280 @@ +-- THIS FILE CONTAINS THE SQLITE3 SCHEMA MIGRATION FROM +-- 3.0b1 TO 3.0b2 +-- +-- AFTER 3.0b2 IS RELEASED YOU MAY NOT EDIT THIS FILE. + +-- For SQLite3 migration strategy, see +-- http://sqlite.org/faq.html#q11 + +-- REMOVALS from the mailinglist table. +-- REM archive +-- REM archive_private +-- REM archive_volume_frequency +-- REM include_list_post_header +-- REM news_moderation +-- REM news_prefix_subject_too +-- REM nntp_host +-- +-- ADDS to the mailing list table. +-- ADD allow_list_posts +-- ADD archive_policy +-- ADD list_id +-- ADD newsgroup_moderation +-- ADD nntp_prefix_subject_too + +-- LP: #971013 +-- LP: #967238 + +-- REMOVALS from the member table. +-- REM mailing_list + +-- ADDS to the member table. +-- ADD list_id + +-- LP: #1024509 + + +CREATE TABLE ml_backup( + id INTEGER NOT NULL, + -- List identity + list_name TEXT, + mail_host TEXT, + allow_list_posts BOOLEAN, + include_rfc2369_headers BOOLEAN, + -- Attributes not directly modifiable via the web u/i + created_at TIMESTAMP, + admin_member_chunksize INTEGER, + next_request_id INTEGER, + next_digest_number INTEGER, + digest_last_sent_at TIMESTAMP, + volume INTEGER, + last_post_at TIMESTAMP, + accept_these_nonmembers BLOB, + acceptable_aliases_id INTEGER, + admin_immed_notify BOOLEAN, + admin_notify_mchanges BOOLEAN, + administrivia BOOLEAN, + advertised BOOLEAN, + anonymous_list BOOLEAN, + -- Automatic responses. + autorespond_owner INTEGER, + autoresponse_owner_text TEXT, + autorespond_postings INTEGER, + autoresponse_postings_text TEXT, + autorespond_requests INTEGER, + autoresponse_request_text TEXT, + autoresponse_grace_period TEXT, + -- Bounces. + forward_unrecognized_bounces_to INTEGER, + process_bounces BOOLEAN, + bounce_info_stale_after TEXT, + bounce_matching_headers TEXT, + bounce_notify_owner_on_disable BOOLEAN, + bounce_notify_owner_on_removal BOOLEAN, + bounce_score_threshold INTEGER, + bounce_you_are_disabled_warnings INTEGER, + bounce_you_are_disabled_warnings_interval TEXT, + -- Content filtering. + filter_action INTEGER, + filter_content BOOLEAN, + collapse_alternatives BOOLEAN, + convert_html_to_plaintext BOOLEAN, + default_member_action INTEGER, + default_nonmember_action INTEGER, + description TEXT, + digest_footer_uri TEXT, + digest_header_uri TEXT, + digest_is_default BOOLEAN, + digest_send_periodic BOOLEAN, + digest_size_threshold FLOAT, + digest_volume_frequency INTEGER, + digestable BOOLEAN, + discard_these_nonmembers BLOB, + emergency BOOLEAN, + encode_ascii_prefixes BOOLEAN, + first_strip_reply_to BOOLEAN, + footer_uri TEXT, + forward_auto_discards BOOLEAN, + gateway_to_mail BOOLEAN, + gateway_to_news BOOLEAN, + goodbye_message_uri TEXT, + header_matches BLOB, + header_uri TEXT, + hold_these_nonmembers BLOB, + info TEXT, + linked_newsgroup TEXT, + max_days_to_hold INTEGER, + max_message_size INTEGER, + max_num_recipients INTEGER, + member_moderation_notice TEXT, + mime_is_default_digest BOOLEAN, + moderator_password TEXT, + new_member_options INTEGER, + nondigestable BOOLEAN, + nonmember_rejection_notice TEXT, + obscure_addresses BOOLEAN, + owner_chain TEXT, + owner_pipeline TEXT, + personalize INTEGER, + post_id INTEGER, + posting_chain TEXT, + posting_pipeline TEXT, + preferred_language TEXT, + private_roster BOOLEAN, + display_name TEXT, + reject_these_nonmembers BLOB, + reply_goes_to_list INTEGER, + reply_to_address TEXT, + require_explicit_destination BOOLEAN, + respond_to_post_requests BOOLEAN, + scrub_nondigest BOOLEAN, + send_goodbye_message BOOLEAN, + send_reminders BOOLEAN, + send_welcome_message BOOLEAN, + subject_prefix TEXT, + subscribe_auto_approval BLOB, + subscribe_policy INTEGER, + topics BLOB, + topics_bodylines_limit INTEGER, + topics_enabled BOOLEAN, + unsubscribe_policy INTEGER, + welcome_message_uri TEXT, + PRIMARY KEY (id) + ); + +INSERT INTO ml_backup SELECT + id, + -- List identity + list_name, + mail_host, + include_list_post_header, + include_rfc2369_headers, + -- Attributes not directly modifiable via the web u/i + created_at, + admin_member_chunksize, + next_request_id, + next_digest_number, + digest_last_sent_at, + volume, + last_post_at, + accept_these_nonmembers, + acceptable_aliases_id, + admin_immed_notify, + admin_notify_mchanges, + administrivia, + advertised, + anonymous_list, + -- Automatic responses. + autorespond_owner, + autoresponse_owner_text, + autorespond_postings, + autoresponse_postings_text, + autorespond_requests, + autoresponse_request_text, + autoresponse_grace_period, + -- Bounces. + forward_unrecognized_bounces_to, + process_bounces, + bounce_info_stale_after, + bounce_matching_headers, + bounce_notify_owner_on_disable, + bounce_notify_owner_on_removal, + bounce_score_threshold, + bounce_you_are_disabled_warnings, + bounce_you_are_disabled_warnings_interval, + -- Content filtering. + filter_action, + filter_content, + collapse_alternatives, + convert_html_to_plaintext, + default_member_action, + default_nonmember_action, + description, + digest_footer_uri, + digest_header_uri, + digest_is_default, + digest_send_periodic, + digest_size_threshold, + digest_volume_frequency, + digestable, + discard_these_nonmembers, + emergency, + encode_ascii_prefixes, + first_strip_reply_to, + footer_uri, + forward_auto_discards, + gateway_to_mail, + gateway_to_news, + goodbye_message_uri, + header_matches, + header_uri, + hold_these_nonmembers, + info, + linked_newsgroup, + max_days_to_hold, + max_message_size, + max_num_recipients, + member_moderation_notice, + mime_is_default_digest, + moderator_password, + new_member_options, + nondigestable, + nonmember_rejection_notice, + obscure_addresses, + owner_chain, + owner_pipeline, + personalize, + post_id, + posting_chain, + posting_pipeline, + preferred_language, + private_roster, + display_name, + reject_these_nonmembers, + reply_goes_to_list, + reply_to_address, + require_explicit_destination, + respond_to_post_requests, + scrub_nondigest, + send_goodbye_message, + send_reminders, + send_welcome_message, + subject_prefix, + subscribe_auto_approval, + subscribe_policy, + topics, + topics_bodylines_limit, + topics_enabled, + unsubscribe_policy, + welcome_message_uri + FROM mailinglist; + +CREATE TABLE mem_backup( + id INTEGER NOT NULL, + _member_id TEXT, + role INTEGER, + moderation_action INTEGER, + address_id INTEGER, + preferences_id INTEGER, + user_id INTEGER, + PRIMARY KEY (id) + ); + +INSERT INTO mem_backup SELECT + id, + _member_id, + role, + moderation_action, + address_id, + preferences_id, + user_id + FROM member; + + +-- Add the new columns. They'll get inserted at the Python layer. +ALTER TABLE ml_backup ADD COLUMN archive_policy INTEGER; +ALTER TABLE ml_backup ADD COLUMN list_id TEXT; +ALTER TABLE ml_backup ADD COLUMN nntp_prefix_subject_too INTEGER; +ALTER TABLE ml_backup ADD COLUMN newsgroup_moderation INTEGER; + +ALTER TABLE mem_backup ADD COLUMN list_id TEXT; diff --git a/src/mailman/database/sqlite.py b/src/mailman/database/sqlite.py index 2677d0d71..8415aa1ee 100644 --- a/src/mailman/database/sqlite.py +++ b/src/mailman/database/sqlite.py @@ -17,19 +17,25 @@ """SQLite database support.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ 'SQLiteDatabase', + 'make_temporary', ] import os +import types +import shutil +import tempfile +from functools import partial from urlparse import urlparse from mailman.database.base import StormBaseDatabase +from mailman.testing.helpers import configuration @@ -41,7 +47,7 @@ class SQLiteDatabase(StormBaseDatabase): def _database_exists(self, store): """See `BaseDatabase`.""" table_query = 'select tbl_name from sqlite_master;' - table_names = set(item[0] for item in + table_names = set(item[0] for item in store.execute(table_query)) return 'version' in table_names @@ -54,3 +60,25 @@ class SQLiteDatabase(StormBaseDatabase): # Ignore errors if fd > 0: os.close(fd) + + + +# Test suite adapter for ITemporaryDatabase. + +def _cleanup(self, tempdir): + shutil.rmtree(tempdir) + + +def make_temporary(database): + """Adapts by monkey patching an existing SQLite IDatabase.""" + tempdir = tempfile.mkdtemp() + url = 'sqlite:///' + os.path.join(tempdir, 'mailman.db') + with configuration('database', url=url): + database.initialize() + database._cleanup = types.MethodType( + partial(_cleanup, tempdir=tempdir), + database) + # bool column values in SQLite must be integers. + database.FALSE = 0 + database.TRUE = 1 + return database diff --git a/src/mailman/database/tests/__init__.py b/src/mailman/database/tests/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/mailman/database/tests/__init__.py diff --git a/src/mailman/database/tests/data/__init__.py b/src/mailman/database/tests/data/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/mailman/database/tests/data/__init__.py diff --git a/src/mailman/database/tests/data/mailman_01.db b/src/mailman/database/tests/data/mailman_01.db Binary files differnew file mode 100644 index 000000000..1ff8d8343 --- /dev/null +++ b/src/mailman/database/tests/data/mailman_01.db diff --git a/src/mailman/database/tests/data/migration_postgres_1.sql b/src/mailman/database/tests/data/migration_postgres_1.sql new file mode 100644 index 000000000..b82ecf6e4 --- /dev/null +++ b/src/mailman/database/tests/data/migration_postgres_1.sql @@ -0,0 +1,133 @@ +INSERT INTO "acceptablealias" VALUES(1,'foo@example.com',1); +INSERT INTO "acceptablealias" VALUES(2,'bar@example.com',1); + +INSERT INTO "address" VALUES( + 1,'anne@example.com',NULL,'Anne Person', + '2012-04-19 00:52:24.826432','2012-04-19 00:49:42.373769',1,2); +INSERT INTO "address" VALUES( + 2,'bart@example.com',NULL,'Bart Person', + '2012-04-19 00:53:25.878800','2012-04-19 00:49:52.882050',2,4); + +INSERT INTO "domain" VALUES( + 1,'example.com','http://example.com',NULL,'postmaster@example.com'); + +INSERT INTO "mailinglist" VALUES( + -- id,list_name,mail_host,include_list_post_header,include_rfc2369_headers + 1,'test','example.com',True,True, + -- created_at,admin_member_chunksize,next_request_id,next_digest_number + '2012-04-19 00:46:13.173844',30,1,1, + -- digest_last_sent_at,volume,last_post_at,accept_these_nonmembers + NULL,1,NULL,E'\\x80025D71012E', + -- acceptable_aliases_id,admin_immed_notify,admin_notify_mchanges + NULL,True,False, + -- administrivia,advertised,anonymous_list,archive,archive_private + True,True,False,True,False, + -- archive_volume_frequency + 1, + --autorespond_owner,autoresponse_owner_text + 0,'', + -- autorespond_postings,autoresponse_postings_text + 0,'', + -- autorespond_requests,authoresponse_requests_text + 0,'', + -- autoresponse_grace_period + '90 days, 0:00:00', + -- forward_unrecognized_bounces_to,process_bounces + 1,True, + -- bounce_info_stale_after,bounce_matching_headers + '7 days, 0:00:00',' +# Lines that *start* with a ''#'' are comments. +to: friend@public.com +message-id: relay.comanche.denmark.eu +from: list@listme.com +from: .*@uplinkpro.com +', + -- bounce_notify_owner_on_disable,bounce_notify_owner_on_removal + True,True, + -- bounce_score_threshold,bounce_you_are_disabled_warnings + 5,3, + -- bounce_you_are_disabled_warnings_interval + '7 days, 0:00:00', + -- filter_action,filter_content,collapse_alternatives + 2,False,True, + -- convert_html_to_plaintext,default_member_action,default_nonmember_action + False,4,0, + -- description + '', + -- digest_footer_uri + 'mailman:///$listname/$language/footer-generic.txt', + -- digest_header_uri + NULL, + -- digest_is_default,digest_send_periodic,digest_size_threshold + False,True,30.0, + -- digest_volume_frequency,digestable,discard_these_nonmembers + 1,True,E'\\x80025D71012E', + -- emergency,encode_ascii_prefixes,first_strip_reply_to + False,False,False, + -- footer_uri + 'mailman:///$listname/$language/footer-generic.txt', + -- forward_auto_discards,gateway_to_mail,gateway_to_news + True,False,FAlse, + -- generic_nonmember_action,goodby_message_uri + 1,'', + -- header_matches,header_uri,hold_these_nonmembers,info,linked_newsgroup + E'\\x80025D71012E',NULL,E'\\x80025D71012E','','', + -- max_days_to_hold,max_message_size,max_num_recipients + 0,40,10, + -- member_moderation_notice,mime_is_default_digest,moderator_password + '',False,NULL, + -- new_member_options,news_moderation,news_prefix_subject_too + 256,0,True, + -- nntp_host,nondigestable,nonmember_rejection_notice,obscure_addresses + '',True,'',True, + -- owner_chain,owner_pipeline,personalize,post_id + 'default-owner-chain','default-owner-pipeline',0,1, + -- posting_chain,posting_pipeline,preferred_language,private_roster + 'default-posting-chain','default-posting-pipeline','en',True, + -- display_name,reject_these_nonmembers + 'Test',E'\\x80025D71012E', + -- reply_goes_to_list,reply_to_address + 0,'', + -- require_explicit_destination,respond_to_post_requests + True,True, + -- scrub_nondigest,send_goodbye_message,send_reminders,send_welcome_message + False,True,True,True, + -- subject_prefix,subscribe_auto_approval + '[Test] ',E'\\x80025D71012E', + -- subscribe_policy,topics,topics_bodylines_limit,topics_enabled + 1,E'\\x80025D71012E',5,False, + -- unsubscribe_policy,welcome_message_uri + 0,'mailman:///welcome.txt'); + +INSERT INTO "member" VALUES( + 1,'d1243f4d-e604-4f6b-af52-98d0a7bce0f1',1,'test@example.com',4,NULL,5,1); +INSERT INTO "member" VALUES( + 2,'dccc3851-fdfb-4afa-90cf-bdcbf80ad0fd',2,'test@example.com',3,NULL,6,1); +INSERT INTO "member" VALUES( + 3,'479be431-45f2-473d-bc3c-7eac614030ac',3,'test@example.com',3,NULL,7,2); +INSERT INTO "member" VALUES( + 4,'e2dc604c-d93a-4b91-b5a8-749e3caade36',1,'test@example.com',4,NULL,8,2); + +INSERT INTO "preferences" VALUES(1,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(2,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(3,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(4,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(5,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(6,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(7,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(8,NULL,NULL,NULL,NULL,NULL,NULL,NULL); + +INSERT INTO "user" VALUES( + 1,'Anne Person',NULL,'0adf3caa-6f26-46f8-a11d-5256c8148592', + '2012-04-19 00:49:42.370493',1,1); +INSERT INTO "user" VALUES( + 2,'Bart Person',NULL,'63f5d1a2-e533-4055-afe4-475dec3b1163', + '2012-04-19 00:49:52.868746',2,3); + +INSERT INTO "uid" VALUES(1,'8bf9a615-f23e-4980-b7d1-90ac0203c66f'); +INSERT INTO "uid" VALUES(2,'0adf3caa-6f26-46f8-a11d-5256c8148592'); +INSERT INTO "uid" VALUES(3,'63f5d1a2-e533-4055-afe4-475dec3b1163'); +INSERT INTO "uid" VALUES(4,'d1243f4d-e604-4f6b-af52-98d0a7bce0f1'); +INSERT INTO "uid" VALUES(5,'dccc3851-fdfb-4afa-90cf-bdcbf80ad0fd'); +INSERT INTO "uid" VALUES(6,'479be431-45f2-473d-bc3c-7eac614030ac'); +INSERT INTO "uid" VALUES(7,'e2dc604c-d93a-4b91-b5a8-749e3caade36'); diff --git a/src/mailman/database/tests/data/migration_sqlite_1.sql b/src/mailman/database/tests/data/migration_sqlite_1.sql new file mode 100644 index 000000000..a5ac96dfa --- /dev/null +++ b/src/mailman/database/tests/data/migration_sqlite_1.sql @@ -0,0 +1,133 @@ +INSERT INTO "acceptablealias" VALUES(1,'foo@example.com',1); +INSERT INTO "acceptablealias" VALUES(2,'bar@example.com',1); + +INSERT INTO "address" VALUES( + 1,'anne@example.com',NULL,'Anne Person', + '2012-04-19 00:52:24.826432','2012-04-19 00:49:42.373769',1,2); +INSERT INTO "address" VALUES( + 2,'bart@example.com',NULL,'Bart Person', + '2012-04-19 00:53:25.878800','2012-04-19 00:49:52.882050',2,4); + +INSERT INTO "domain" VALUES( + 1,'example.com','http://example.com',NULL,'postmaster@example.com'); + +INSERT INTO "mailinglist" VALUES( + -- id,list_name,mail_host,include_list_post_header,include_rfc2369_headers + 1,'test','example.com',1,1, + -- created_at,admin_member_chunksize,next_request_id,next_digest_number + '2012-04-19 00:46:13.173844',30,1,1, + -- digest_last_sent_at,volume,last_post_at,accept_these_nonmembers + NULL,1,NULL,X'80025D71012E', + -- acceptable_aliases_id,admin_immed_notify,admin_notify_mchanges + NULL,1,0, + -- administrivia,advertised,anonymous_list,archive,archive_private + 1,1,0,1,0, + -- archive_volume_frequency + 1, + --autorespond_owner,autoresponse_owner_text + 0,'', + -- autorespond_postings,autoresponse_postings_text + 0,'', + -- autorespond_requests,authoresponse_requests_text + 0,'', + -- autoresponse_grace_period + '90 days, 0:00:00', + -- forward_unrecognized_bounces_to,process_bounces + 1,1, + -- bounce_info_stale_after,bounce_matching_headers + '7 days, 0:00:00',' +# Lines that *start* with a ''#'' are comments. +to: friend@public.com +message-id: relay.comanche.denmark.eu +from: list@listme.com +from: .*@uplinkpro.com +', + -- bounce_notify_owner_on_disable,bounce_notify_owner_on_removal + 1,1, + -- bounce_score_threshold,bounce_you_are_disabled_warnings + 5,3, + -- bounce_you_are_disabled_warnings_interval + '7 days, 0:00:00', + -- filter_action,filter_content,collapse_alternatives + 2,0,1, + -- convert_html_to_plaintext,default_member_action,default_nonmember_action + 0,4,0, + -- description + '', + -- digest_footer_uri + 'mailman:///$listname/$language/footer-generic.txt', + -- digest_header_uri + NULL, + -- digest_is_default,digest_send_periodic,digest_size_threshold + 0,1,30.0, + -- digest_volume_frequency,digestable,discard_these_nonmembers + 1,1,X'80025D71012E', + -- emergency,encode_ascii_prefixes,first_strip_reply_to + 0,0,0, + -- footer_uri + 'mailman:///$listname/$language/footer-generic.txt', + -- forward_auto_discards,gateway_to_mail,gateway_to_news + 1,0,0, + -- generic_nonmember_action,goodby_message_uri + 1,'', + -- header_matches,header_uri,hold_these_nonmembers,info,linked_newsgroup + X'80025D71012E',NULL,X'80025D71012E','','', + -- max_days_to_hold,max_message_size,max_num_recipients + 0,40,10, + -- member_moderation_notice,mime_is_default_digest,moderator_password + '',0,NULL, + -- new_member_options,news_moderation,news_prefix_subject_too + 256,0,1, + -- nntp_host,nondigestable,nonmember_rejection_notice,obscure_addresses + '',1,'',1, + -- owner_chain,owner_pipeline,personalize,post_id + 'default-owner-chain','default-owner-pipeline',0,1, + -- posting_chain,posting_pipeline,preferred_language,private_roster + 'default-posting-chain','default-posting-pipeline','en',1, + -- display_name,reject_these_nonmembers + 'Test',X'80025D71012E', + -- reply_goes_to_list,reply_to_address + 0,'', + -- require_explicit_destination,respond_to_post_requests + 1,1, + -- scrub_nondigest,send_goodbye_message,send_reminders,send_welcome_message + 0,1,1,1, + -- subject_prefix,subscribe_auto_approval + '[Test] ',X'80025D71012E', + -- subscribe_policy,topics,topics_bodylines_limit,topics_enabled + 1,X'80025D71012E',5,0, + -- unsubscribe_policy,welcome_message_uri + 0,'mailman:///welcome.txt'); + +INSERT INTO "member" VALUES( + 1,'d1243f4d-e604-4f6b-af52-98d0a7bce0f1',1,'test@example.com',4,NULL,5,1); +INSERT INTO "member" VALUES( + 2,'dccc3851-fdfb-4afa-90cf-bdcbf80ad0fd',2,'test@example.com',3,NULL,6,1); +INSERT INTO "member" VALUES( + 3,'479be431-45f2-473d-bc3c-7eac614030ac',3,'test@example.com',3,NULL,7,2); +INSERT INTO "member" VALUES( + 4,'e2dc604c-d93a-4b91-b5a8-749e3caade36',1,'test@example.com',4,NULL,8,2); + +INSERT INTO "preferences" VALUES(1,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(2,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(3,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(4,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(5,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(6,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(7,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(8,NULL,NULL,NULL,NULL,NULL,NULL,NULL); + +INSERT INTO "user" VALUES( + 1,'Anne Person',NULL,'0adf3caa-6f26-46f8-a11d-5256c8148592', + '2012-04-19 00:49:42.370493',1,1); +INSERT INTO "user" VALUES( + 2,'Bart Person',NULL,'63f5d1a2-e533-4055-afe4-475dec3b1163', + '2012-04-19 00:49:52.868746',2,3); + +INSERT INTO "uid" VALUES(1,'8bf9a615-f23e-4980-b7d1-90ac0203c66f'); +INSERT INTO "uid" VALUES(2,'0adf3caa-6f26-46f8-a11d-5256c8148592'); +INSERT INTO "uid" VALUES(3,'63f5d1a2-e533-4055-afe4-475dec3b1163'); +INSERT INTO "uid" VALUES(4,'d1243f4d-e604-4f6b-af52-98d0a7bce0f1'); +INSERT INTO "uid" VALUES(5,'dccc3851-fdfb-4afa-90cf-bdcbf80ad0fd'); +INSERT INTO "uid" VALUES(6,'479be431-45f2-473d-bc3c-7eac614030ac'); +INSERT INTO "uid" VALUES(7,'e2dc604c-d93a-4b91-b5a8-749e3caade36'); diff --git a/src/mailman/database/tests/test_migrations.py b/src/mailman/database/tests/test_migrations.py new file mode 100644 index 000000000..f50ce35d3 --- /dev/null +++ b/src/mailman/database/tests/test_migrations.py @@ -0,0 +1,369 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test schema migrations.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestMigration20120407MigratedData', + 'TestMigration20120407Schema', + 'TestMigration20120407UnchangedData', + ] + + +import unittest + +from pkg_resources import resource_string +from storm.exceptions import DatabaseError +from zope.component import getUtility + +from mailman.interfaces.database import IDatabaseFactory +from mailman.interfaces.domain import IDomainManager +from mailman.interfaces.archiver import ArchivePolicy +from mailman.interfaces.listmanager import IListManager +from mailman.interfaces.mailinglist import IAcceptableAliasSet +from mailman.interfaces.nntp import NewsgroupModeration +from mailman.interfaces.subscriptions import ISubscriptionService +from mailman.testing.helpers import temporary_db +from mailman.testing.layers import ConfigLayer + + + +class MigrationTestBase(unittest.TestCase): + """Test the dated migration (LP: #971013) + + Circa: 3.0b1 -> 3.0b2 + + table mailinglist: + * news_moderation -> newsgroup_moderation + * news_prefix_subject_too -> nntp_prefix_subject_too + * include_list_post_header -> allow_list_posts + * ADD archive_policy + * ADD list_id + * REMOVE archive + * REMOVE archive_private + * REMOVE archive_volume_frequency + * REMOVE nntp_host + + table member: + * mailing_list -> list_id + """ + + layer = ConfigLayer + + def setUp(self): + self._database = getUtility(IDatabaseFactory, 'temporary').create() + + def tearDown(self): + self._database._cleanup() + + + +class TestMigration20120407Schema(MigrationTestBase): + """Test column migrations.""" + + def test_pre_upgrade_columns_migration(self): + # Test that before the migration, the old table columns are present + # and the new database columns are not. + # + # Load all the migrations to just before the one we're testing. + self._database.load_migrations('20120406999999') + self._database.store.commit() + # Verify that the database has not yet been migrated. + for missing in ('allow_list_posts', + 'archive_policy', + 'list_id', + 'nntp_prefix_subject_too'): + self.assertRaises(DatabaseError, + self._database.store.execute, + 'select {0} from mailinglist;'.format(missing)) + self._database.store.rollback() + self.assertRaises(DatabaseError, + self._database.store.execute, + 'select list_id from member;') + self._database.store.rollback() + for present in ('archive', + 'archive_private', + 'archive_volume_frequency', + 'generic_nonmember_action', + 'include_list_post_header', + 'news_moderation', + 'news_prefix_subject_too', + 'nntp_host'): + # This should not produce an exception. Is there some better test + # that we can perform? + self._database.store.execute( + 'select {0} from mailinglist;'.format(present)) + # Again, this should not produce an exception. + self._database.store.execute('select mailing_list from member;') + + def test_post_upgrade_columns_migration(self): + # Test that after the migration, the old table columns are missing + # and the new database columns are present. + # + # Load all the migrations up to and including the one we're testing. + self._database.load_migrations('20120406999999') + self._database.load_migrations('20120407000000') + # Verify that the database has been migrated. + for present in ('allow_list_posts', + 'archive_policy', + 'list_id', + 'nntp_prefix_subject_too'): + # This should not produce an exception. Is there some better test + # that we can perform? + self._database.store.execute( + 'select {0} from mailinglist;'.format(present)) + self._database.store.execute('select list_id from member;') + for missing in ('archive', + 'archive_private', + 'archive_volume_frequency', + 'generic_nonmember_action', + 'include_list_post_header', + 'news_moderation', + 'news_prefix_subject_too', + 'nntp_host'): + self.assertRaises(DatabaseError, + self._database.store.execute, + 'select {0} from mailinglist;'.format(missing)) + self._database.store.rollback() + self.assertRaises(DatabaseError, + self._database.store.execute, + 'select mailing_list from member;') + + + +class TestMigration20120407UnchangedData(MigrationTestBase): + """Test non-migrated data.""" + + def setUp(self): + MigrationTestBase.setUp(self) + # Load all the migrations to just before the one we're testing. + self._database.load_migrations('20120406999999') + # Load the previous schema's sample data. + sample_data = resource_string( + 'mailman.database.tests.data', + 'migration_{0}_1.sql'.format(self._database.TAG)) + self._database.load_sql(self._database.store, sample_data) + # Update to the current migration we're testing. + self._database.load_migrations('20120407000000') + + def test_migration_domains(self): + # Test that the domains table, which isn't touched, doesn't change. + with temporary_db(self._database): + # Check that the domains survived the migration. This table + # was not touched so it should be fine. + domains = list(getUtility(IDomainManager)) + self.assertEqual(len(domains), 1) + self.assertEqual(domains[0].mail_host, 'example.com') + + def test_migration_mailing_lists(self): + # Test that the mailing lists survive migration. + with temporary_db(self._database): + # There should be exactly one mailing list defined. + mlists = list(getUtility(IListManager).mailing_lists) + self.assertEqual(len(mlists), 1) + self.assertEqual(mlists[0].fqdn_listname, 'test@example.com') + + def test_migration_acceptable_aliases(self): + # Test that the mailing list's acceptable aliases survive migration. + # This proves that foreign key references are migrated properly. + with temporary_db(self._database): + mlist = getUtility(IListManager).get('test@example.com') + aliases_set = IAcceptableAliasSet(mlist) + self.assertEqual(set(aliases_set.aliases), + set(['foo@example.com', 'bar@example.com'])) + + def test_migration_members(self): + # Test that the members of a mailing list all survive migration. + with temporary_db(self._database): + mlist = getUtility(IListManager).get('test@example.com') + # Test that all the members we expect are still there. Start with + # the two list delivery members. + addresses = set(address.email + for address in mlist.members.addresses) + self.assertEqual(addresses, + set(['anne@example.com', 'bart@example.com'])) + # There is one owner. + owners = set(address.email for address in mlist.owners.addresses) + self.assertEqual(len(owners), 1) + self.assertEqual(owners.pop(), 'anne@example.com') + # There is one moderator. + moderators = set(address.email + for address in mlist.moderators.addresses) + self.assertEqual(len(moderators), 1) + self.assertEqual(moderators.pop(), 'bart@example.com') + + + +class TestMigration20120407MigratedData(MigrationTestBase): + """Test affected migration data.""" + + def setUp(self): + MigrationTestBase.setUp(self) + # Load all the migrations to just before the one we're testing. + self._database.load_migrations('20120406999999') + # Load the previous schema's sample data. + sample_data = resource_string( + 'mailman.database.tests.data', + 'migration_{0}_1.sql'.format(self._database.TAG)) + self._database.load_sql(self._database.store, sample_data) + + def _upgrade(self): + # Update to the current migration we're testing. + self._database.load_migrations('20120407000000') + + def test_migration_archive_policy_never_0(self): + # Test that the new archive_policy value is updated correctly. In the + # case of old column archive=0, the archive_private column is + # ignored. This test sets it to 0 to ensure it's ignored. + self._database.store.execute( + 'UPDATE mailinglist SET archive = {0}, archive_private = {0} ' + 'WHERE id = 1;'.format(self._database.FALSE)) + # Complete the migration + self._upgrade() + with temporary_db(self._database): + mlist = getUtility(IListManager).get('test@example.com') + self.assertEqual(mlist.archive_policy, ArchivePolicy.never) + + def test_migration_archive_policy_never_1(self): + # Test that the new archive_policy value is updated correctly. In the + # case of old column archive=0, the archive_private column is + # ignored. This test sets it to 1 to ensure it's ignored. + self._database.store.execute( + 'UPDATE mailinglist SET archive = {0}, archive_private = {1} ' + 'WHERE id = 1;'.format(self._database.FALSE, + self._database.TRUE)) + # Complete the migration + self._upgrade() + with temporary_db(self._database): + mlist = getUtility(IListManager).get('test@example.com') + self.assertEqual(mlist.archive_policy, ArchivePolicy.never) + + def test_archive_policy_private(self): + # Test that the new archive_policy value is updated correctly for + # private archives. + self._database.store.execute( + 'UPDATE mailinglist SET archive = {0}, archive_private = {0} ' + 'WHERE id = 1;'.format(self._database.TRUE)) + # Complete the migration + self._upgrade() + with temporary_db(self._database): + mlist = getUtility(IListManager).get('test@example.com') + self.assertEqual(mlist.archive_policy, ArchivePolicy.private) + + def test_archive_policy_public(self): + # Test that the new archive_policy value is updated correctly for + # public archives. + self._database.store.execute( + 'UPDATE mailinglist SET archive = {1}, archive_private = {0} ' + 'WHERE id = 1;'.format(self._database.FALSE, + self._database.TRUE)) + # Complete the migration + self._upgrade() + with temporary_db(self._database): + mlist = getUtility(IListManager).get('test@example.com') + self.assertEqual(mlist.archive_policy, ArchivePolicy.public) + + def test_list_id(self): + # Test that the mailinglist table gets a list_id column. + self._upgrade() + with temporary_db(self._database): + mlist = getUtility(IListManager).get('test@example.com') + self.assertEqual(mlist.list_id, 'test.example.com') + + def test_list_id_member(self): + # Test that the member table's mailing_list column becomes list_id. + self._upgrade() + with temporary_db(self._database): + service = getUtility(ISubscriptionService) + members = list(service.find_members(list_id='test.example.com')) + self.assertEqual(len(members), 4) + + def test_news_moderation_none(self): + # Test that news_moderation becomes newsgroup_moderation. + self._database.store.execute( + 'UPDATE mailinglist SET news_moderation = 0 ' + 'WHERE id = 1;') + self._upgrade() + with temporary_db(self._database): + mlist = getUtility(IListManager).get('test@example.com') + self.assertEqual(mlist.newsgroup_moderation, + NewsgroupModeration.none) + + def test_news_moderation_open_moderated(self): + # Test that news_moderation becomes newsgroup_moderation. + self._database.store.execute( + 'UPDATE mailinglist SET news_moderation = 1 ' + 'WHERE id = 1;') + self._upgrade() + with temporary_db(self._database): + mlist = getUtility(IListManager).get('test@example.com') + self.assertEqual(mlist.newsgroup_moderation, + NewsgroupModeration.open_moderated) + + def test_news_moderation_moderated(self): + # Test that news_moderation becomes newsgroup_moderation. + self._database.store.execute( + 'UPDATE mailinglist SET news_moderation = 2 ' + 'WHERE id = 1;') + self._upgrade() + with temporary_db(self._database): + mlist = getUtility(IListManager).get('test@example.com') + self.assertEqual(mlist.newsgroup_moderation, + NewsgroupModeration.moderated) + + def test_nntp_prefix_subject_too_false(self): + # Test that news_prefix_subject_too becomes nntp_prefix_subject_too. + self._database.store.execute( + 'UPDATE mailinglist SET news_prefix_subject_too = {0} ' + 'WHERE id = 1;'.format(self._database.FALSE)) + self._upgrade() + with temporary_db(self._database): + mlist = getUtility(IListManager).get('test@example.com') + self.assertFalse(mlist.nntp_prefix_subject_too) + + def test_nntp_prefix_subject_too_true(self): + # Test that news_prefix_subject_too becomes nntp_prefix_subject_too. + self._database.store.execute( + 'UPDATE mailinglist SET news_prefix_subject_too = {0} ' + 'WHERE id = 1;'.format(self._database.TRUE)) + self._upgrade() + with temporary_db(self._database): + mlist = getUtility(IListManager).get('test@example.com') + self.assertTrue(mlist.nntp_prefix_subject_too) + + def test_allow_list_posts_false(self): + # Test that include_list_post_header -> allow_list_posts. + self._database.store.execute( + 'UPDATE mailinglist SET include_list_post_header = {0} ' + 'WHERE id = 1;'.format(self._database.FALSE)) + self._upgrade() + with temporary_db(self._database): + mlist = getUtility(IListManager).get('test@example.com') + self.assertFalse(mlist.allow_list_posts) + + def test_allow_list_posts_true(self): + # Test that include_list_post_header -> allow_list_posts. + self._database.store.execute( + 'UPDATE mailinglist SET include_list_post_header = {0} ' + 'WHERE id = 1;'.format(self._database.TRUE)) + self._upgrade() + with temporary_db(self._database): + mlist = getUtility(IListManager).get('test@example.com') + self.assertTrue(mlist.allow_list_posts) diff --git a/src/mailman/database/transaction.py b/src/mailman/database/transaction.py index 47e58d3e2..295f3d567 100644 --- a/src/mailman/database/transaction.py +++ b/src/mailman/database/transaction.py @@ -17,19 +17,36 @@ """Transactional support.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ - 'txn', + 'dbconnection', + 'transaction', + 'transactional', ] +from contextlib import contextmanager + from mailman.config import config -class txn(object): +@contextmanager +def transaction(): + """Context manager for ensuring the transaction is complete.""" + try: + yield + except: + config.db.abort() + raise + else: + config.db.commit() + + + +def transactional(function): """Decorator for transactional support. When the function this decorator wraps exits cleanly, the current @@ -38,16 +55,25 @@ class txn(object): Either way, the current transaction is completed. """ - def __init__(self, function): - self._function = function + def wrapper(*args, **kws): + try: + rtn = function(*args, **kws) + config.db.commit() + return rtn + except: + config.db.abort() + raise + return wrapper - def __get__(self, obj, type=None): - def wrapper(*args, **kws): - try: - rtn = self._function(obj, *args, **kws) - config.db.commit() - return rtn - except: - config.db.abort() - raise - return wrapper + + +def dbconnection(function): + """Decorator for getting at the database connection. + + Use this to avoid having to access the global `config.db.store` + attribute. This calls the function with `store` as the first argument. + """ + def wrapper(*args, **kws): + # args[0] is self. + return function(args[0], config.db.store, *args[1:], **kws) + return wrapper diff --git a/src/mailman/docs/ACKNOWLEDGMENTS.rst b/src/mailman/docs/ACKNOWLEDGMENTS.rst index b6c6cc16c..10d4bcd93 100644 --- a/src/mailman/docs/ACKNOWLEDGMENTS.rst +++ b/src/mailman/docs/ACKNOWLEDGMENTS.rst @@ -58,6 +58,7 @@ code, and have assigned copyright for contributions to the FSF: * Simone Piunno * Claudia Schmidt * Andreas Schosser +* Stephen J. Turnbull * Richard Wackerbarth @@ -247,6 +248,7 @@ left off the list! * Greg Stein * Dale Stimson * Students of HIT <mailman-cn@mail.cs.hit.edu.cn> +* Alexander Sulfrian * Szabolcs Szigeti * Vizi Szilard * David T-G diff --git a/src/mailman/docs/DATABASE.rst b/src/mailman/docs/DATABASE.rst index f5fe39849..762d60a11 100644 --- a/src/mailman/docs/DATABASE.rst +++ b/src/mailman/docs/DATABASE.rst @@ -8,10 +8,10 @@ however, Storm is compatible with PostgreSQL_ and MySQL, among possibly others. Currently, Mailman is known to work with either the default SQLite3 database, -or PostgreSQL. (Volunteers to port it to MySQL are welcome!). If you want to -use SQLite3, you generally don't need to change anything, but if you want -Mailman to use PostgreSQL, you'll need to set that up first, and then change a -configuration variable in your `/etc/mailman.cfg` file. +or PostgreSQL. (Volunteers to port it to other databases are welcome!). If +you want to use SQLite3, you generally don't need to change anything, but if +you want Mailman to use PostgreSQL, you'll need to set that up first, and then +change a configuration variable in your `/etc/mailman.cfg` file. Two configuration variables control which database Mailman uses. The first names the class implementing the database interface. The second names the diff --git a/src/mailman/docs/INTRODUCTION.rst b/src/mailman/docs/INTRODUCTION.rst index fd64efb42..11d13c239 100644 --- a/src/mailman/docs/INTRODUCTION.rst +++ b/src/mailman/docs/INTRODUCTION.rst @@ -53,8 +53,8 @@ Warsaw, Ken Manheimer, Scott Cotton, Harald Meland, and John Viega. Version 1.0 and beyond have been primarily maintained by Barry Warsaw with contributions from many; see the ACKNOWLEDGMENTS file for details. Jeremy Hylton helped considerably with the Pipermail code in Mailman 2.0. Mailman -2.1 is primarily maintained by Mark Sapiro and Tokio Kikuchi. Barry Warsaw is -the lead developer on Mailman 3. +2.1 is primarily maintained by Mark Sapiro, with previous help by Tokio +Kikuchi. Barry Warsaw is the lead developer on Mailman 3. Help diff --git a/src/mailman/docs/MTA.rst b/src/mailman/docs/MTA.rst index f541d3838..c6d2230c4 100644 --- a/src/mailman/docs/MTA.rst +++ b/src/mailman/docs/MTA.rst @@ -2,9 +2,10 @@ Hooking up your mail server =========================== -Mailman needs to be hooked up to your mail server both to accept incoming mail -and to deliver outgoing mail. Mailman itself never delivers messages to the -end user; it lets its immediate upstream mail server do that. +Mailman needs to be hooked up to your mail server (a.k.a. *mail transport +agent* or *MTA*) both to accept incoming mail and to deliver outgoing mail. +Mailman itself never delivers messages to the end user; it lets its immediate +upstream mail server do that. The preferred way to allow Mailman to accept incoming messages from your mail server is to use the `Local Mail Transfer Protocol`_ (LMTP_) interface. Most @@ -32,6 +33,9 @@ Contributions are welcome! Postfix ======= +Postfix_ is an open source mail server by Wietse Venema. + + Mailman settings ---------------- @@ -58,9 +62,9 @@ as shown above. Basic Postfix connections ------------------------- -There are several ways to hook Postfix_ up to Mailman, so here are the -simplest instructions. The following settings should be added to Postfix's -`main.cf` file. +There are several ways to hook Postfix up to Mailman, so here are the simplest +instructions. The following settings should be added to Postfix's `main.cf` +file. Mailman supports a technique called `Variable Envelope Return Path`_ (VERP) to disambiguate and accurately record bounces. By default Mailman's VERP diff --git a/src/mailman/docs/NEWS.rst b/src/mailman/docs/NEWS.rst index b92d57068..5020b89e3 100644 --- a/src/mailman/docs/NEWS.rst +++ b/src/mailman/docs/NEWS.rst @@ -10,10 +10,26 @@ Here is a history of user visible changes to Mailman. 3.0 beta 2 -- "Freeze" ====================== -(20XX-XX-XX) +(2012-09-05) Architecture ------------ + * The link between members and the mailing lists they are subscribed to, is + now via the RFC 2369 `list_id` instead of the fqdn listname (i.e. posting + address). This is because while the posting address can change if the + mailing list is moved to a new server, the list id is fixed. + (LP: #1024509) + + - IListManager.get_by_list_id() added. + - IListManager.list_ids added. + - IMailingList.list_id added. + - Several internal APIs that accepted fqdn list names now require list ids, + e.g. ISubscriptionService.join() and .find_members(). + - IMember.list_id attribute added; .mailing_list is now an alias that + retrieves and returns the IMailingList. + + * `passlib`_ is now used for all password hashing instead of flufl.password. + The default hash is `sha512_crypt`. (LP: #1015758) * Internally, all datetimes are kept in the UTC timezone, however because of LP: #280708, they are stored in the database in naive format. * `received_time` is now added to the message metadata by the LMTP runner @@ -31,6 +47,52 @@ Architecture or unverified. (LP: #975698) * A `PasswordChangeEvent` is triggered when an `IUser`'s password changes. (LP: #975700) + * When a queue runner gets an exception in its _dispose() method, a + `RunnerCrashEvent` is triggered, which contains references to the queue + runner, mailing list, message, metadata, and exception. Interested parties + can subscribe to that `zope.event` for notification. + * Events renamed and moved: + * `mailman.chains.accept.AcceptNotification` + * `mailman.chains.base.ChainNotification` + * `mailman.chains.discard.DiscardNotification` + * `mailman.chains.hold.HoldNotification` + * `mailman.chains.owner.OwnerNotification` + * `mailman.chains.reject.RejectNotification` + changed to (respectively): + * `mailman.interfaces.chains.AcceptEvent` + * `mailman.interfaces.chains.ChainEvent` + * `mailman.interfaces.chains.DiscardEvent` + * `mailman.interfaces.chains.HoldEvent` + * `mailman.interfaces.chains.AcceptOwnerEvent` + * `mailman.interfaces.chains.RejectEvent` + * A `ConfigurationUpdatedEvent` is triggered when the system-wide global + configuration stack is pushed or popped. + * The policy for archiving has now been collapsed into a single enum, called + ArchivePolicy. This describes the three states of never archive, archive + privately, and archive_publicly. (LP: #967238) + +Database +-------- + * Schema migrations (LP: #971013) + + - mailinglist.include_list_post_header -> allow_list_posts + - mailinglist.news_prefix_subject_too -> nntp_prefix_subject_too + - mailinglist.news_moderation -> newsgroup_moderation + - mailinglist.archive and mailinglist.archive_private have been collapsed + into archive_policy. + - mailinglist.nntp_host has been removed. + - mailinglist.generic_nonmember_action has been removed (LP: #975696) + + * Schema migrations (LP: #1024509) + - member.mailing_list -> list_id + * The PostgreSQL port of the schema accidentally added a moderation_callback + column to the mailinglist table. Since this is unused in Mailman, it was + simply commented out of the base schema for PostgreSQL. + +REST +---- + * Expose `archive_policy` in the REST API. Contributed by Alexander + Sulfrian. (LP: #1039129) Configuration ------------- @@ -38,9 +100,12 @@ Configuration every `[archiver.<name>]` section. These are used to determine under what circumstances a message destined for a specific archiver should have its `Date:` header clobbered. (LP: #963612) + * With the switch to `passlib`_, `[passwords]password_scheme` has been + removed. Instead use `[passwords]path` to specify where to find the + `passlib.cfg` file. See the comments in `schema.cfg` for details. * Configuration schema variable changes: - [nntp]username -> [nntp]user - [nntp]port (added) + * [nntp]username -> [nntp]user + * [nntp]port (added) * Header check specifications in the `mailman.cfg` file have changed quite bit. The previous `[spam.header.foo]` sections have been removed. Instead, there's a new `[antispam]` section that contains a `header_checks` @@ -61,6 +126,13 @@ Bug fixes given by Mark Sapiro. (LP: #949924) * Fixed a typo when returning the configuration file's header match checks. (LP: #953497) + * List-Post should be NO when posting is not allowed. (LP: #987563) + * Non-unicode values in msgdata broke pending requests. (LP: #1031391) + * Show devmode in `bin/mailman info` output. (LP: #1035028) + * Fix residual references to the old `IMailingList` archive variables. + (LP: #1031393) + +.. _`passlib`: http://packages.python.org/passlib/index.html 3.0 beta 1 -- "The Twilight Zone" diff --git a/src/mailman/docs/START.rst b/src/mailman/docs/START.rst index 57df48e30..da76feae0 100644 --- a/src/mailman/docs/START.rst +++ b/src/mailman/docs/START.rst @@ -9,11 +9,8 @@ Beta Release ============ This is a beta release. The developers believe it has sufficient -functionality to provide full services to a mailing list, but it is not ready -for production yet. Interfaces and administration may differ substantially -from the alpha series, but changes should be incremental going forward from -beta 1. Changes from the alpha series will be described in notes to the main -text. +functionality to provide full mailing list services, but it is not yet ready +for production use. The Mailman 3 beta releases are being provided to give developers and other interested people an early look at the next major version, and site @@ -40,9 +37,9 @@ $PATH or it can be accessible via the ``python2.6`` or ``python2.7`` binary. If your operating system does not include Python, see http://www.python.org downloading and installing it from source. Python 3 is not yet supported. -In this documentation, a bare ``python`` refers to the python used to invoke -``bootstrap.py``, which might be ``python2.6`` or ``python2.7``, as well as -the system ``python`` or an absolute path. +In this documentation, a bare ``python`` refers to the Python executable used +to invoke ``bootstrap.py``, which might be ``python2.6`` or ``python2.7``, as +well as the system ``python`` or an absolute path. Mailman 3 is now based on the `zc.buildout`_ infrastructure, which greatly simplifies building and testing Mailman. diff --git a/src/mailman/docs/STYLEGUIDE.rst b/src/mailman/docs/STYLEGUIDE.rst index 32b2da72f..b744c6557 100644 --- a/src/mailman/docs/STYLEGUIDE.rst +++ b/src/mailman/docs/STYLEGUIDE.rst @@ -2,7 +2,7 @@ GNU Mailman Coding Style Guide ============================== -Copyright (C) 2002-2011 Barry A. Warsaw +Copyright (C) 2002-2012 Barry A. Warsaw Python coding style guide for GNU Mailman diff --git a/src/mailman/email/validate.py b/src/mailman/email/validate.py index 1861a8121..021e4c073 100644 --- a/src/mailman/email/validate.py +++ b/src/mailman/email/validate.py @@ -27,7 +27,7 @@ __all__ = [ import re -from zope.interface import implements +from zope.interface import implementer from mailman.interfaces.address import ( IEmailValidator, InvalidEmailAddressError) @@ -39,11 +39,10 @@ _badchars = re.compile(r'[][()<>|;^,\000-\037\177-\377]') +@implementer(IEmailValidator) class Validator: """An email address validator.""" - implements(IEmailValidator) - def is_valid(self, email): """See `IEmailValidator`.""" if not email or ' ' in email: diff --git a/src/mailman/handlers/acknowledge.py b/src/mailman/handlers/acknowledge.py index 0e0916337..0366f8ce6 100644 --- a/src/mailman/handlers/acknowledge.py +++ b/src/mailman/handlers/acknowledge.py @@ -20,7 +20,7 @@ This only happens if the sender has set their AcknowledgePosts attribute. """ -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -29,7 +29,7 @@ __all__ = [ from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.email.message import UserNotification @@ -40,9 +40,9 @@ from mailman.utilities.string import oneline +@implementer(IHandler) class Acknowledge: """Send an acknowledgment.""" - implements(IHandler) name = 'acknowledge' description = _("""Send an acknowledgment of a posting.""") diff --git a/src/mailman/handlers/after_delivery.py b/src/mailman/handlers/after_delivery.py index a964804b5..0a3ba2c75 100644 --- a/src/mailman/handlers/after_delivery.py +++ b/src/mailman/handlers/after_delivery.py @@ -17,7 +17,7 @@ """Perform some bookkeeping after a successful post.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -25,7 +25,7 @@ __all__ = [ ] -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler @@ -33,11 +33,10 @@ from mailman.utilities.datetime import now +@implementer(IHandler) class AfterDelivery: """Perform some bookkeeping after a successful post.""" - implements(IHandler) - name = 'after-delivery' description = _('Perform some bookkeeping after a successful post.') diff --git a/src/mailman/handlers/avoid_duplicates.py b/src/mailman/handlers/avoid_duplicates.py index ffbc80c85..de1939822 100644 --- a/src/mailman/handlers/avoid_duplicates.py +++ b/src/mailman/handlers/avoid_duplicates.py @@ -23,7 +23,7 @@ has already received a copy, we either drop the message, add a duplicate warning header, or pass it through, depending on the user's preferences. """ -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -32,7 +32,7 @@ __all__ = [ from email.utils import getaddresses, formataddr -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler @@ -42,11 +42,10 @@ COMMASPACE = ', ' +@implementer(IHandler) class AvoidDuplicates: """If the user wishes it, do not send duplicates of the same message.""" - implements(IHandler) - name = 'avoid-duplicates' description = _('Suppress some duplicates of the same message.') diff --git a/src/mailman/handlers/cleanse.py b/src/mailman/handlers/cleanse.py index 605b843d0..32d3455ca 100644 --- a/src/mailman/handlers/cleanse.py +++ b/src/mailman/handlers/cleanse.py @@ -17,7 +17,7 @@ """Cleanse certain headers from all messages.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -28,7 +28,7 @@ __all__ = [ import logging from email.utils import formataddr -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.handlers.cook_headers import uheader @@ -39,11 +39,10 @@ log = logging.getLogger('mailman.post') +@implementer(IHandler) class Cleanse: """Cleanse certain headers from all messages.""" - implements(IHandler) - name = 'cleanse' description = _('Cleanse certain headers from all messages.') diff --git a/src/mailman/handlers/cleanse_dkim.py b/src/mailman/handlers/cleanse_dkim.py index d2cd32636..bc23980b7 100644 --- a/src/mailman/handlers/cleanse_dkim.py +++ b/src/mailman/handlers/cleanse_dkim.py @@ -25,7 +25,7 @@ and it will also give the MTA the opportunity to regenerate valid keys originating at the Mailman server for the outgoing message. """ -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -34,7 +34,7 @@ __all__ = [ from lazr.config import as_boolean -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.core.i18n import _ @@ -42,11 +42,10 @@ from mailman.interfaces.handler import IHandler +@implementer(IHandler) class CleanseDKIM: """Remove DomainKeys headers.""" - implements(IHandler) - name = 'cleanse-dkim' description = _('Remove DomainKeys headers.') diff --git a/src/mailman/handlers/cook_headers.py b/src/mailman/handlers/cook_headers.py index 5d1e416a6..535155ab7 100644 --- a/src/mailman/handlers/cook_headers.py +++ b/src/mailman/handlers/cook_headers.py @@ -17,7 +17,7 @@ """Cook a message's headers.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -30,7 +30,7 @@ import re from email.errors import HeaderParseError from email.header import Header, decode_header, make_header from email.utils import parseaddr, formataddr, getaddresses -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler @@ -279,11 +279,10 @@ def ch_oneline(headerstr): +@implementer(IHandler) class CookHeaders: """Modify message headers.""" - implements(IHandler) - name = 'cook-headers' description = _('Modify message headers.') diff --git a/src/mailman/handlers/decorate.py b/src/mailman/handlers/decorate.py index d6d156048..c5fad2891 100644 --- a/src/mailman/handlers/decorate.py +++ b/src/mailman/handlers/decorate.py @@ -31,7 +31,7 @@ import logging from email.mime.text import MIMEText from urllib2 import URLError from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.email.message import Message @@ -57,7 +57,7 @@ def process(mlist, msg, msgdata): d['user_address'] = recipient d['user_delivered_to'] = member.address.original_email d['user_language'] = member.preferred_language.description - d['user_name'] = (member.user.display_name + d['user_name'] = (member.user.display_name if member.user.display_name else member.address.original_email) d['user_optionsurl'] = member.options_url @@ -232,11 +232,10 @@ def decorate(mlist, uri, extradict=None): +@implementer(IHandler) class Decorate: """Decorate a message with headers and footers.""" - implements(IHandler) - name = 'decorate' description = _('Decorate a message with headers and footers.') diff --git a/src/mailman/handlers/docs/archives.rst b/src/mailman/handlers/docs/archives.rst index 323d121e8..abaad9f52 100644 --- a/src/mailman/handlers/docs/archives.rst +++ b/src/mailman/handlers/docs/archives.rst @@ -26,7 +26,8 @@ should *not* get archived. For example, no digests should ever get archived. - >>> mlist.archive = True + >>> from mailman.interfaces.archiver import ArchivePolicy + >>> mlist.archive_policy = ArchivePolicy.public >>> msg = message_from_string("""\ ... Subject: A sample message ... @@ -39,7 +40,7 @@ For example, no digests should ever get archived. If the mailing list is not configured to archive, then even regular deliveries won't be archived. - >>> mlist.archive = False + >>> mlist.archive_policy = ArchivePolicy.never >>> handler.process(mlist, msg, {}) >>> switchboard.files [] @@ -49,7 +50,7 @@ want to be archived. We've seen both in the wild so both are supported. The ``X-No-Archive:`` header can be used to indicate that the message should not be archived. Confusingly, this header's value is actually ignored. - >>> mlist.archive = True + >>> mlist.archive_policy = ArchivePolicy.public >>> msg = message_from_string("""\ ... Subject: A sample message ... X-No-Archive: YES diff --git a/src/mailman/handlers/docs/rfc-2369.rst b/src/mailman/handlers/docs/rfc-2369.rst index 0461f27ba..7064de6bd 100644 --- a/src/mailman/handlers/docs/rfc-2369.rst +++ b/src/mailman/handlers/docs/rfc-2369.rst @@ -7,7 +7,8 @@ headers generally start with the `List-` prefix. >>> mlist = create_list('test@example.com') >>> mlist.preferred_language = 'en' - >>> mlist.archive = False + >>> from mailman.interfaces.archiver import ArchivePolicy + >>> mlist.archive_policy = ArchivePolicy.never .. This is a helper function for the following section. @@ -59,13 +60,14 @@ about hiding them. A list owner can turn these headers off. Messages which Mailman generates itself, such as user or owner notifications, have a reduced set of `List-` headers. Specifically, there is no `List-Post`, `List-Archive` or `Archived-At` header. +.. >>> mlist.include_rfc2369_headers = True - >>> mlist.include_list_post_header = False >>> msg = message_from_string("""\ ... From: aperson@example.com ... ... """) + >>> process(mlist, msg, dict(reduced_list_headers=True)) >>> list_headers(msg) ---start--- @@ -84,7 +86,8 @@ List-Post header Discussion lists, to which any subscriber can post, also have a `List-Post` header which contains the `mailto:` URL used to send messages to the list. - >>> mlist.include_list_post_header = True + >>> mlist.include_rfc2369_headers = True + >>> mlist.allow_list_posts = True >>> msg = message_from_string("""\ ... From: aperson@example.com ... @@ -101,6 +104,28 @@ header which contains the `mailto:` URL used to send messages to the list. <mailto:test-leave@example.com> ---end--- +Some mailing lists are announce, or one-way lists, not discussion lists. +Because the general membership cannot post to these mailing lists, the list +owner can set a flag which adds a special `List-Post` header value, according +to RFC 2369. + + >>> mlist.allow_list_posts = False + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... + ... """) + >>> process(mlist, msg, {}) + >>> list_headers(msg) + ---start--- + list-help: <mailto:test-request@example.com?subject=help> + list-id: <test.example.com> + list-post: NO + list-subscribe: <http://lists.example.com/listinfo/test@example.com>, + <mailto:test-join@example.com> + list-unsubscribe: <http://lists.example.com/listinfo/test@example.com>, + <mailto:test-leave@example.com> + ---end--- + List-Id header ============== @@ -108,6 +133,7 @@ List-Id header If the mailing list has a description, then it is included in the ``List-Id`` header. + >>> mlist.allow_list_posts = True >>> mlist.description = 'My test mailing list' >>> msg = message_from_string("""\ ... From: aperson@example.com @@ -146,7 +172,7 @@ Archive headers When the mailing list is configured to enable archiving, a `List-Archive` header will be added. - >>> mlist.archive = True + >>> mlist.archive_policy = ArchivePolicy.public `RFC 5064`_ defines the `Archived-At` header which contains the url to the individual message in the archives. Archivers which don't support @@ -183,7 +209,7 @@ If the mailing list isn't being archived, neither the `List-Archive` nor `Archived-At` headers will be added. >>> config.pop('prototype') - >>> mlist.archive = False + >>> mlist.archive_policy = ArchivePolicy.never >>> msg = message_from_string("""\ ... From: aperson@example.com ... diff --git a/src/mailman/handlers/file_recipients.py b/src/mailman/handlers/file_recipients.py index d087ff2bb..750357a90 100644 --- a/src/mailman/handlers/file_recipients.py +++ b/src/mailman/handlers/file_recipients.py @@ -17,7 +17,7 @@ """Get the normal delivery recipients from a Sendmail style :include: file.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -28,18 +28,17 @@ __all__ = [ import os import errno -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler +@implementer(IHandler) class FileRecipients: """Get the normal delivery recipients from an include file.""" - implements(IHandler) - name = 'file-recipients' description = _('Get the normal delivery recipients from an include file.') diff --git a/src/mailman/handlers/member_recipients.py b/src/mailman/handlers/member_recipients.py index 956ea6adc..ec8ed77b1 100644 --- a/src/mailman/handlers/member_recipients.py +++ b/src/mailman/handlers/member_recipients.py @@ -31,7 +31,7 @@ __all__ = [ ] -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.core import errors @@ -42,11 +42,10 @@ from mailman.utilities.string import wrap +@implementer(IHandler) class MemberRecipients: """Calculate the regular (i.e. non-digest) recipients of the message.""" - implements(IHandler) - name = 'member-recipients' description = _('Calculate the regular recipients of the message.') diff --git a/src/mailman/handlers/mime_delete.py b/src/mailman/handlers/mime_delete.py index c9c1eb408..52fcc99fa 100644 --- a/src/mailman/handlers/mime_delete.py +++ b/src/mailman/handlers/mime_delete.py @@ -24,7 +24,7 @@ wrapping only single sections after other processing are replaced by their contents. """ -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -42,7 +42,7 @@ from email.mime.message import MIMEMessage from email.mime.text import MIMEText from lazr.config import as_boolean from os.path import splitext -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.core import errors @@ -285,11 +285,10 @@ def get_file_ext(m): +@implementer(IHandler) class MIMEDelete: """Filter the MIME content of messages.""" - implements(IHandler) - name = 'mime-delete' description = _('Filter the MIME content of messages.') diff --git a/src/mailman/handlers/owner_recipients.py b/src/mailman/handlers/owner_recipients.py index e431d00cf..66c380635 100644 --- a/src/mailman/handlers/owner_recipients.py +++ b/src/mailman/handlers/owner_recipients.py @@ -25,7 +25,7 @@ __all__ = [ ] -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.core.i18n import _ @@ -34,11 +34,10 @@ from mailman.interfaces.member import DeliveryStatus +@implementer(IHandler) class OwnerRecipients: """Calculate the owner (and moderator) recipients for -owner postings.""" - implements(IHandler) - name = 'owner-recipients' description = _('Calculate the owner and moderator recipients.') diff --git a/src/mailman/handlers/replybot.py b/src/mailman/handlers/replybot.py index 83aa40214..a25f4f30b 100644 --- a/src/mailman/handlers/replybot.py +++ b/src/mailman/handlers/replybot.py @@ -17,7 +17,7 @@ """Handler for automatic responses.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -28,7 +28,7 @@ __all__ = [ import logging from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.email.message import UserNotification @@ -44,11 +44,10 @@ log = logging.getLogger('mailman.error') +@implementer(IHandler) class Replybot: """Send automatic responses.""" - implements(IHandler) - name = 'replybot' description = _('Send automatic responses.') diff --git a/src/mailman/handlers/rfc_2369.py b/src/mailman/handlers/rfc_2369.py index ece4e83cb..47dd9dad2 100644 --- a/src/mailman/handlers/rfc_2369.py +++ b/src/mailman/handlers/rfc_2369.py @@ -17,7 +17,7 @@ """RFC 2369 List-* and related headers.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -26,11 +26,12 @@ __all__ = [ from email.utils import formataddr -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.core.i18n import _ from mailman.handlers.cook_headers import uheader +from mailman.interfaces.archiver import ArchivePolicy from mailman.interfaces.handler import IHandler @@ -74,11 +75,15 @@ def process(mlist, msg, msgdata): 'List-Subscribe' : subfieldfmt.format(listinfo, mlist.join_address), }) if not msgdata.get('reduced_list_headers'): - # List-Post: is controlled by a separate attribute - if mlist.include_list_post_header: - headers['List-Post'] = '<mailto:{0}>'.format(mlist.posting_address) + # List-Post: is controlled by a separate attribute, which is somewhat + # misnamed. RFC 2369 requires a value of NO if posting is not + # allowed, i.e. for an announce-only list. + list_post = ('<mailto:{0}>'.format(mlist.posting_address) + if mlist.allow_list_posts + else 'NO') + headers['List-Post'] = list_post # Add RFC 2369 and 5064 archiving headers, if archiving is enabled. - if mlist.archive: + if mlist.archive_policy is not ArchivePolicy.never: for archiver in config.archivers: headers['List-Archive'] = '<{0}>'.format( archiver.list_url(mlist)) @@ -100,11 +105,10 @@ def process(mlist, msg, msgdata): +@implementer(IHandler) class RFC2369: """Add the RFC 2369 List-* headers.""" - implements(IHandler) - name = 'rfc-2369' description = _('Add the RFC 2369 List-* headers.') diff --git a/src/mailman/handlers/tagger.py b/src/mailman/handlers/tagger.py index 49e004a12..9d78372e6 100644 --- a/src/mailman/handlers/tagger.py +++ b/src/mailman/handlers/tagger.py @@ -17,7 +17,7 @@ """Extract topics from the original mail message.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -29,7 +29,7 @@ import re import email.iterators import email.parser -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler @@ -178,11 +178,10 @@ class _ForgivingParser(email.parser.HeaderParser): +@implementer(IHandler) class Tagger: """Tag messages with topic matches.""" - implements(IHandler) - name = 'tagger' description = _('Tag messages with topic matches.') diff --git a/src/mailman/handlers/to_archive.py b/src/mailman/handlers/to_archive.py index fd5259a14..6b82aef61 100644 --- a/src/mailman/handlers/to_archive.py +++ b/src/mailman/handlers/to_archive.py @@ -17,7 +17,7 @@ """Add the message to the archives.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -25,26 +25,27 @@ __all__ = [ ] -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.core.i18n import _ +from mailman.interfaces.archiver import ArchivePolicy from mailman.interfaces.handler import IHandler +@implementer(IHandler) class ToArchive: """Add the message to the archives.""" - implements(IHandler) - name = 'to-archive' description = _('Add the message to the archives.') def process(self, mlist, msg, msgdata): """See `IHandler`.""" # Short circuits. - if msgdata.get('isdigest') or not mlist.archive: + if (msgdata.get('isdigest') or + mlist.archive_policy is ArchivePolicy.never): return # Common practice seems to favor "X-No-Archive: yes". No other value # for this header seems to make sense, so we'll just test for it's diff --git a/src/mailman/handlers/to_digest.py b/src/mailman/handlers/to_digest.py index 71511f136..8067e2c0c 100644 --- a/src/mailman/handlers/to_digest.py +++ b/src/mailman/handlers/to_digest.py @@ -17,7 +17,7 @@ """Add the message to the list's current digest.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -27,7 +27,7 @@ __all__ = [ import os -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.core.i18n import _ @@ -39,11 +39,10 @@ from mailman.utilities.mailbox import Mailbox +@implementer(IHandler) class ToDigest: """Add the message to the digest, possibly sending it.""" - implements(IHandler) - name = 'to-digest' description = _('Add the message to the digest, possibly sending it.') diff --git a/src/mailman/handlers/to_outgoing.py b/src/mailman/handlers/to_outgoing.py index 971f87757..a212485c7 100644 --- a/src/mailman/handlers/to_outgoing.py +++ b/src/mailman/handlers/to_outgoing.py @@ -22,7 +22,7 @@ posted to the list membership. Anything else that needs to go out to some recipient should just be placed in the out queue directly. """ -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -30,7 +30,7 @@ __all__ = [ ] -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.core.i18n import _ @@ -38,11 +38,10 @@ from mailman.interfaces.handler import IHandler +@implementer(IHandler) class ToOutgoing: """Send the message to the outgoing queue.""" - implements(IHandler) - name = 'to-outgoing' description = _('Send the message to the outgoing queue.') diff --git a/src/mailman/handlers/to_usenet.py b/src/mailman/handlers/to_usenet.py index 021f8f9e5..79f4c9b1b 100644 --- a/src/mailman/handlers/to_usenet.py +++ b/src/mailman/handlers/to_usenet.py @@ -27,7 +27,7 @@ __all__ = [ import logging -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.core.i18n import _ @@ -39,11 +39,10 @@ log = logging.getLogger('mailman.error') +@implementer(IHandler) class ToUsenet: """Move the message to the outgoing news queue.""" - implements(IHandler) - name = 'to-usenet' description = _('Move the message to the outgoing news queue.') diff --git a/src/mailman/interfaces/archiver.py b/src/mailman/interfaces/archiver.py index f3edc7719..d9ca45514 100644 --- a/src/mailman/interfaces/archiver.py +++ b/src/mailman/interfaces/archiver.py @@ -17,10 +17,11 @@ """Interface for archiving schemes.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ + 'ArchivePolicy', 'ClobberDate', 'IArchiver', ] @@ -31,6 +32,13 @@ from zope.interface import Interface, Attribute +class ArchivePolicy(Enum): + never = 0 + private = 1 + public = 2 + + + class ClobberDate(Enum): never = 1 maybe = 2 diff --git a/src/mailman/interfaces/chain.py b/src/mailman/interfaces/chain.py index 8858f874c..8862026f8 100644 --- a/src/mailman/interfaces/chain.py +++ b/src/mailman/interfaces/chain.py @@ -21,11 +21,17 @@ from __future__ import absolute_import, unicode_literals __metaclass__ = type __all__ = [ + 'AcceptEvent', + 'AcceptOwnerEvent', + 'ChainEvent', + 'DiscardEvent', + 'HoldEvent', 'IChain', 'IChainIterator', 'IChainLink', 'IMutableChain', 'LinkAction', + 'RejectEvent', ] @@ -34,6 +40,37 @@ from zope.interface import Interface, Attribute +class ChainEvent: + """Base class for chain notification events.""" + + def __init__(self, mlist, msg, msgdata, chain): + self.mlist = mlist + self.msg = msg + self.msgdata = msgdata + self.chain = chain + + +class AcceptEvent(ChainEvent): + """A notification event signaling that a message is being accepted.""" + + +class AcceptOwnerEvent(ChainEvent): + """An event signaling that a message is accepted to the -owner address.""" + + +class DiscardEvent(ChainEvent): + """A notification event signaling that a message is being discarded.""" + + +class HoldEvent(ChainEvent): + """A notification event signaling that a message is being held.""" + + +class RejectEvent(ChainEvent): + """A notification event signaling that a message is being rejected.""" + + + class LinkAction(Enum): # Jump to another chain. jump = 0 diff --git a/src/mailman/interfaces/configuration.py b/src/mailman/interfaces/configuration.py new file mode 100644 index 000000000..8c4fb52a6 --- /dev/null +++ b/src/mailman/interfaces/configuration.py @@ -0,0 +1,41 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Configuration system interface.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'ConfigurationUpdatedEvent', + 'IConfiguration', + ] + + +from zope.interface import Interface + + + +class IConfiguration(Interface): + """Marker interface; used for adaptation in the REST API.""" + + + +class ConfigurationUpdatedEvent: + """The system-wide global configuration was updated.""" + def __init__(self, config): + self.config = config diff --git a/src/mailman/interfaces/database.py b/src/mailman/interfaces/database.py index 0530f83b9..1f39daee7 100644 --- a/src/mailman/interfaces/database.py +++ b/src/mailman/interfaces/database.py @@ -17,16 +17,18 @@ """Interfaces for database interaction.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ 'DatabaseError', 'IDatabase', + 'IDatabaseFactory', + 'ITemporaryDatabase', ] -from zope.interface import Interface +from zope.interface import Attribute, Interface from mailman.interfaces.errors import MailmanError @@ -49,12 +51,6 @@ class IDatabase(Interface): configuration file setting. """ - def _reset(): - """Reset the database to its pristine state. - - This is only used by the test framework. - """ - def begin(): """Begin the current transaction.""" @@ -63,3 +59,25 @@ class IDatabase(Interface): def abort(): """Abort the current transaction.""" + + store = Attribute( + """The underlying Storm store on which you can do queries.""") + + + +class ITemporaryDatabase(Interface): + """Marker interface for test suite adaptation.""" + + + +class IDatabaseFactory(Interface): + "Interface for creating new databases.""" + + def create(): + """Return a new `IDatabase`. + + The database will be initialized and all migrations will be loaded. + + :return: A new database. + :rtype: IDatabase + """ diff --git a/src/mailman/interfaces/listmanager.py b/src/mailman/interfaces/listmanager.py index 6f43edf3f..573ba11df 100644 --- a/src/mailman/interfaces/listmanager.py +++ b/src/mailman/interfaces/listmanager.py @@ -113,6 +113,15 @@ class IListManager(Interface): not exist. """ + def get_by_list_id(list_id): + """Return the mailing list with the given list id, if it exists. + + :type fqdn_listname: Unicode. + :param fqdn_listname: The fully qualified name of the mailing list. + :return: the matching `IMailingList` or None if the named list does + not exist. + """ + def delete(mlist): """Remove the mailing list from the database. @@ -134,6 +143,10 @@ class IListManager(Interface): """An iterator over the fully qualified list names of all mailing lists managed by this list manager.""") + list_ids = Attribute( + """An iterator over the list ids of all mailing lists managed by this + list manager.""") + name_components = Attribute( """An iterator over the 2-tuple of (list_name, mail_host) for all mailing lists managed by this list manager.""") diff --git a/src/mailman/interfaces/mailinglist.py b/src/mailman/interfaces/mailinglist.py index bced070d3..8a4436a21 100644 --- a/src/mailman/interfaces/mailinglist.py +++ b/src/mailman/interfaces/mailinglist.py @@ -17,7 +17,7 @@ """Interface for a mailing list.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -81,6 +81,11 @@ class IMailingList(Interface): mail_host is 'example.com'. """) + list_id = Attribute("""\ + The identity of the mailing list. This value will never change. It + is defined in RFC 2369. + """) + fqdn_listname = Attribute("""\ The read-only fully qualified name of the mailing list. This is the guaranteed unique id for the mailing list, and it is always the @@ -103,9 +108,13 @@ class IMailingList(Interface): mailing lists, or in headers, and so forth. It should be as succinct as you can get it, while still identifying what the list is.""") - include_list_post_header = Attribute( - """Flag specifying whether to include the RFC 2369 List-Post header. - This is usually set to True, except for announce-only lists.""") + allow_list_posts = Attribute( + """Flag specifying posts to the list are generally allowed. + + This controls the value of the RFC 2369 List-Post header. This is + usually set to True, except for announce-only lists. When False, the + List-Post is set to NO as per the RFC. + """) include_rfc2369_headers = Attribute( """Flag specifying whether to include any RFC 2369 header, including @@ -250,6 +259,13 @@ class IMailingList(Interface): # Delivery. + archive_policy = Attribute( + """The policy for archiving messages to this mailing list. + + The value is an `ArchivePolicy` enum. Use this to archive the mailing + list publicly, privately, or not at all. + """) + last_post_at = Attribute( """The date and time a message was last posted to the mailing list.""") @@ -511,6 +527,9 @@ class IMailingList(Interface): without any other checks. """) + newsgroup_moderation = Attribute( + """The moderation policy for the linked newsgroup, if there is one.""") + # Bounces. forward_unrecognized_bounces_to = Attribute( diff --git a/src/mailman/interfaces/member.py b/src/mailman/interfaces/member.py index 52bacc72d..997338835 100644 --- a/src/mailman/interfaces/member.py +++ b/src/mailman/interfaces/member.py @@ -136,8 +136,11 @@ class IMember(Interface): member_id = Attribute( """The member's unique, random identifier as a UUID.""") + list_id = Attribute( + """The list id of the mailing list the member is subscribed to.""") + mailing_list = Attribute( - """The mailing list subscribed to.""") + """The `IMailingList` that the member is subscribed to.""") address = Attribute( """The email address that's subscribed to the list.""") diff --git a/src/mailman/interfaces/nntp.py b/src/mailman/interfaces/nntp.py index d5d08d3f0..22b8b1754 100644 --- a/src/mailman/interfaces/nntp.py +++ b/src/mailman/interfaces/nntp.py @@ -15,9 +15,13 @@ # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see <http://www.gnu.org/licenses/>. +"""NNTP and newsgroup interfaces.""" + +from __future__ import absolute_import, print_function, unicode_literals + __metaclass__ = type __all__ = [ - 'NewsModeration', + 'NewsgroupModeration', ] @@ -25,7 +29,7 @@ from flufl.enum import Enum -class NewsModeration(Enum): +class NewsgroupModeration(Enum): # The newsgroup is not moderated. none = 0 # The newsgroup is moderated, but allows for an open posting policy. diff --git a/src/mailman/interfaces/runner.py b/src/mailman/interfaces/runner.py index 4611fa3a7..9a3c9baa4 100644 --- a/src/mailman/interfaces/runner.py +++ b/src/mailman/interfaces/runner.py @@ -22,6 +22,7 @@ from __future__ import absolute_import, unicode_literals __metaclass__ = type __all__ = [ 'IRunner', + 'RunnerCrashEvent', ] @@ -29,6 +30,18 @@ from zope.interface import Interface, Attribute +class RunnerCrashEvent: + """Triggered when a runner encounters an exception in _dispose().""" + + def __init__(self, runner, mlist, msg, metadata, error): + self.runner = runner + self.mailing_list = mlist + self.message = msg + self.metadata = metadata + self.error = error + + + class IRunner(Interface): """The runner.""" diff --git a/src/mailman/interfaces/subscriptions.py b/src/mailman/interfaces/subscriptions.py index 85f333cf8..cb4900053 100644 --- a/src/mailman/interfaces/subscriptions.py +++ b/src/mailman/interfaces/subscriptions.py @@ -69,7 +69,7 @@ class ISubscriptionService(Interface): :rtype: `IMember` """ - def find_members(subscriber=None, fqdn_listname=None, role=None): + def find_members(subscriber=None, list_id=None, role=None): """Search for and return a specific member. The members are sorted first by fully-qualified mailing list name, @@ -80,9 +80,9 @@ class ISubscriptionService(Interface): :param subscriber: The email address or user id of the user getting subscribed. :type subscriber: string or int - :param fqdn_listname: The posting address of the mailing list to - search for the subscriber's memberships on. - :type fqdn_listname: string + :param list_id: The list id of the mailing list to search for the + subscriber's memberships on. + :type list_id: string :param role: The member role. :type role: `MemberRole` :return: The list of all memberships, which may be empty. @@ -92,8 +92,8 @@ class ISubscriptionService(Interface): def __iter__(): """See `get_members()`.""" - def join(fqdn_listname, subscriber, display_name=None, - delivery_mode=DeliveryMode.regular, + def join(list_id, subscriber, display_name=None, + delivery_mode=DeliveryMode.regular, role=MemberRole.member): """Subscribe to a mailing list. @@ -103,9 +103,9 @@ class ISubscriptionService(Interface): the subscription request is still dependent on the policy of the mailing list. - :param fqdn_listname: The posting address of the mailing list to - subscribe the user to. - :type fqdn_listname: string + :param list_id: The list id of the mailing list the user is + subscribing to. + :type list_id: string :param subscriber: The email address or user id of the user getting subscribed. :type subscriber: string or int @@ -130,12 +130,12 @@ class ISubscriptionService(Interface): :raises ValueError: when `delivery_mode` is invalid. """ - def leave(fqdn_listname, email): + def leave(list_id, email): """Unsubscribe from a mailing list. - :param fqdn_listname: The posting address of the mailing list to - unsubscribe the user from. - :type fqdn_listname: string + :param list_id: The list id of the mailing list the user is + unsubscribing from. + :type list_id: string :param email: The email address of the user getting unsubscribed. :type email: string :raises InvalidEmailAddressError: if the email address is not valid. diff --git a/src/mailman/languages/language.py b/src/mailman/languages/language.py index 04f9c6639..effcf9c1b 100644 --- a/src/mailman/languages/language.py +++ b/src/mailman/languages/language.py @@ -18,7 +18,7 @@ """The representation of a language.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -26,16 +26,16 @@ __all__ = [ ] -from zope.interface import implements +from zope.interface import implementer + from mailman.interfaces.languages import ILanguage +@implementer(ILanguage) class Language: """The representation of a language.""" - implements(ILanguage) - def __init__(self, code, charset, description): self.code = code self.charset = charset diff --git a/src/mailman/languages/manager.py b/src/mailman/languages/manager.py index ca7001102..87b56dbda 100644 --- a/src/mailman/languages/manager.py +++ b/src/mailman/languages/manager.py @@ -17,25 +17,27 @@ """Language manager.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ 'LanguageManager', ] -from zope.interface import implements +from zope.component import getUtility +from zope.interface import implementer + +from mailman.interfaces.configuration import ConfigurationUpdatedEvent from mailman.interfaces.languages import ILanguageManager from mailman.languages.language import Language +@implementer(ILanguageManager) class LanguageManager: """Language manager.""" - implements(ILanguageManager) - def __init__(self): # Mapping from 2-letter code to Language instance. self._languages = {} @@ -73,3 +75,18 @@ class LanguageManager: def clear(self): """See `ILanguageManager`.""" self._languages.clear() + + + +def handle_ConfigurationUpdatedEvent(event): + if not isinstance(event, ConfigurationUpdatedEvent): + return + manager = getUtility(ILanguageManager) + for language in event.config.language_configs: + if language.enabled: + code = language.name.split('.')[1] + manager.add(code, language.charset, language.description) + # the default language must always be available. + assert event.config.mailman.default_language in manager, ( + 'system default language code not defined: {0}'.format( + event.config.mailman.default_language)) diff --git a/src/mailman/model/address.py b/src/mailman/model/address.py index a12a993a8..d8ab65a80 100644 --- a/src/mailman/model/address.py +++ b/src/mailman/model/address.py @@ -17,7 +17,7 @@ """Model for addresses.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -28,7 +28,7 @@ __all__ = [ from email.utils import formataddr from storm.locals import DateTime, Int, Reference, Unicode from zope.event import notify -from zope.interface import implements +from zope.interface import implementer from mailman.database.model import Model from mailman.interfaces.address import AddressVerificationEvent, IAddress @@ -36,8 +36,9 @@ from mailman.utilities.datetime import now +@implementer(IAddress) class Address(Model): - implements(IAddress) + """See `IAddress`.""" id = Int(primary=True) email = Unicode() diff --git a/src/mailman/model/autorespond.py b/src/mailman/model/autorespond.py index 7b42205b4..567dcd19e 100644 --- a/src/mailman/model/autorespond.py +++ b/src/mailman/model/autorespond.py @@ -17,7 +17,7 @@ """Module stuff.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -27,10 +27,10 @@ __all__ = [ from storm.locals import And, Date, Desc, Int, Reference -from zope.interface import implements +from zope.interface import implementer -from mailman.config import config from mailman.database.model import Model +from mailman.database.transaction import dbconnection from mailman.database.types import Enum from mailman.interfaces.autorespond import ( IAutoResponseRecord, IAutoResponseSet, Response) @@ -38,8 +38,9 @@ from mailman.utilities.datetime import today +@implementer(IAutoResponseRecord) class AutoResponseRecord(Model): - implements(IAutoResponseRecord) + """See `IAutoResponseRecord`.""" id = Int(primary=True) @@ -60,33 +61,37 @@ class AutoResponseRecord(Model): +@implementer(IAutoResponseSet) class AutoResponseSet: - implements(IAutoResponseSet) + """See `IAutoResponseSet`.""" def __init__(self, mailing_list): self._mailing_list = mailing_list - def todays_count(self, address, response_type): + @dbconnection + def todays_count(self, store, address, response_type): """See `IAutoResponseSet`.""" - return config.db.store.find( + return store.find( AutoResponseRecord, And(AutoResponseRecord.address == address, AutoResponseRecord.mailing_list == self._mailing_list, AutoResponseRecord.response_type == response_type, AutoResponseRecord.date_sent == today())).count() - def response_sent(self, address, response_type): + @dbconnection + def response_sent(self, store, address, response_type): """See `IAutoResponseSet`.""" response = AutoResponseRecord( self._mailing_list, address, response_type) - config.db.store.add(response) + store.add(response) - def last_response(self, address, response_type): + @dbconnection + def last_response(self, store, address, response_type): """See `IAutoResponseSet`.""" - results = config.db.store.find( + results = store.find( AutoResponseRecord, And(AutoResponseRecord.address == address, AutoResponseRecord.mailing_list == self._mailing_list, AutoResponseRecord.response_type == response_type) - ).order_by(Desc(AutoResponseRecord.date_sent)) + ).order_by(Desc(AutoResponseRecord.date_sent)) return (None if results.count() == 0 else results.first()) diff --git a/src/mailman/model/bans.py b/src/mailman/model/bans.py index 9dc0c51ba..b6de9336f 100644 --- a/src/mailman/model/bans.py +++ b/src/mailman/model/bans.py @@ -17,7 +17,7 @@ """Ban manager.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -28,16 +28,17 @@ __all__ = [ import re from storm.locals import Int, Unicode -from zope.interface import implements +from zope.interface import implementer -from mailman.config import config from mailman.database.model import Model +from mailman.database.transaction import dbconnection from mailman.interfaces.bans import IBan, IBanManager +@implementer(IBan) class Ban(Model): - implements(IBan) + """See `IBan`.""" id = Int(primary=True) email = Unicode() @@ -50,46 +51,47 @@ class Ban(Model): +@implementer(IBanManager) class BanManager: - implements(IBanManager) + """See `IBanManager`.""" - def ban(self, email, mailing_list=None): + @dbconnection + def ban(self, store, email, mailing_list=None): """See `IBanManager`.""" - bans = config.db.store.find( - Ban, email=email, mailing_list=mailing_list) + bans = store.find(Ban, email=email, mailing_list=mailing_list) if bans.count() == 0: ban = Ban(email, mailing_list) - config.db.store.add(ban) + store.add(ban) - def unban(self, email, mailing_list=None): + @dbconnection + def unban(self, store, email, mailing_list=None): """See `IBanManager`.""" - ban = config.db.store.find( - Ban, email=email, mailing_list=mailing_list).one() + ban = store.find(Ban, email=email, mailing_list=mailing_list).one() if ban is not None: - config.db.store.remove(ban) + store.remove(ban) - def is_banned(self, email, mailing_list=None): + @dbconnection + def is_banned(self, store, email, mailing_list=None): """See `IBanManager`.""" # A specific mailing list ban is being checked, however the email # address could be banned specifically, or globally. if mailing_list is not None: # Try specific bans first. - bans = config.db.store.find( - Ban, email=email, mailing_list=mailing_list) + bans = store.find(Ban, email=email, mailing_list=mailing_list) if bans.count() > 0: return True # Try global bans next. - bans = config.db.store.find(Ban, email=email, mailing_list=None) + bans = store.find(Ban, email=email, mailing_list=None) if bans.count() > 0: return True # Now try specific mailing list bans, but with a pattern. - bans = config.db.store.find(Ban, mailing_list=mailing_list) + bans = store.find(Ban, mailing_list=mailing_list) for ban in bans: if (ban.email.startswith('^') and re.match(ban.email, email, re.IGNORECASE) is not None): return True # And now try global pattern bans. - bans = config.db.store.find(Ban, mailing_list=None) + bans = store.find(Ban, mailing_list=None) for ban in bans: if (ban.email.startswith('^') and re.match(ban.email, email, re.IGNORECASE) is not None): @@ -97,12 +99,11 @@ class BanManager: else: # The client is asking for global bans. Look up bans on the # specific email address first. - bans = config.db.store.find( - Ban, email=email, mailing_list=None) + bans = store.find(Ban, email=email, mailing_list=None) if bans.count() > 0: return True # And now look for global pattern bans. - bans = config.db.store.find(Ban, mailing_list=None) + bans = store.find(Ban, mailing_list=None) for ban in bans: if (ban.email.startswith('^') and re.match(ban.email, email, re.IGNORECASE) is not None): diff --git a/src/mailman/model/bounce.py b/src/mailman/model/bounce.py index 8c55e3d16..628e076bf 100644 --- a/src/mailman/model/bounce.py +++ b/src/mailman/model/bounce.py @@ -17,7 +17,7 @@ """Bounce support.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -27,10 +27,10 @@ __all__ = [ from storm.locals import Bool, Int, DateTime, Unicode -from zope.interface import implements +from zope.interface import implementer -from mailman.config import config from mailman.database.model import Model +from mailman.database.transaction import dbconnection from mailman.database.types import Enum from mailman.interfaces.bounce import ( BounceContext, IBounceEvent, IBounceProcessor) @@ -38,8 +38,9 @@ from mailman.utilities.datetime import now +@implementer(IBounceEvent) class BounceEvent(Model): - implements(IBounceEvent) + """See `IBounceEvent`.""" id = Int(primary=True) list_name = Unicode() @@ -59,24 +60,27 @@ class BounceEvent(Model): +@implementer(IBounceProcessor) class BounceProcessor: - implements(IBounceProcessor) + """See `IBounceProcessor`.""" - def register(self, mlist, email, msg, where=None): + @dbconnection + def register(self, store, mlist, email, msg, where=None): """See `IBounceProcessor`.""" event = BounceEvent(mlist.fqdn_listname, email, msg, where) - config.db.store.add(event) + store.add(event) return event @property - def events(self): + @dbconnection + def events(self, store): """See `IBounceProcessor`.""" - for event in config.db.store.find(BounceEvent): + for event in store.find(BounceEvent): yield event @property - def unprocessed(self): + @dbconnection + def unprocessed(self, store): """See `IBounceProcessor`.""" - for event in config.db.store.find(BounceEvent, - BounceEvent.processed == False): + for event in store.find(BounceEvent, BounceEvent.processed == False): yield event diff --git a/src/mailman/model/digests.py b/src/mailman/model/digests.py index d7805ebf6..1d422ce8b 100644 --- a/src/mailman/model/digests.py +++ b/src/mailman/model/digests.py @@ -17,7 +17,7 @@ """One last digest.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -26,7 +26,7 @@ __all__ = [ from storm.locals import Int, Reference -from zope.interface import implements +from zope.interface import implementer from mailman.database.model import Model from mailman.database.types import Enum @@ -35,8 +35,9 @@ from mailman.interfaces.member import DeliveryMode +@implementer(IOneLastDigest) class OneLastDigest(Model): - implements(IOneLastDigest) + """See `IOneLastDigest`.""" id = Int(primary=True) diff --git a/src/mailman/model/docs/listmanager.rst b/src/mailman/model/docs/listmanager.rst index 9c72b18e7..380fe7704 100644 --- a/src/mailman/model/docs/listmanager.rst +++ b/src/mailman/model/docs/listmanager.rst @@ -16,28 +16,31 @@ Creating a mailing list Creating the list returns the newly created IMailList object. >>> from mailman.interfaces.mailinglist import IMailingList - >>> mlist = list_manager.create('_xtest@example.com') + >>> mlist = list_manager.create('test@example.com') >>> IMailingList.providedBy(mlist) True -All lists with identities have a short name, a host name, and a fully -qualified listname. This latter is what uniquely distinguishes the mailing -list to the system. +All lists with identities have a short name, a host name, a fully qualified +listname, and an `RFC 2369`_ list id. This latter will not change even if the +mailing list moves to a different host, so it is what uniquely distinguishes +the mailing list to the system. >>> print mlist.list_name - _xtest + test >>> print mlist.mail_host example.com >>> print mlist.fqdn_listname - _xtest@example.com + test@example.com + >>> print mlist.list_id + test.example.com If you try to create a mailing list with the same name as an existing list, you will get an exception. - >>> list_manager.create('_xtest@example.com') + >>> list_manager.create('test@example.com') Traceback (most recent call last): ... - ListAlreadyExistsError: _xtest@example.com + ListAlreadyExistsError: test@example.com It is an error to create a mailing list that isn't a fully qualified list name (i.e. posting address). @@ -59,9 +62,9 @@ Use the list manager to delete a mailing list. After deleting the list, you can create it again. - >>> mlist = list_manager.create('_xtest@example.com') + >>> mlist = list_manager.create('test@example.com') >>> print mlist.fqdn_listname - _xtest@example.com + test@example.com Retrieving a mailing list @@ -70,13 +73,21 @@ Retrieving a mailing list When a mailing list exists, you can ask the list manager for it and you will always get the same object back. - >>> mlist_2 = list_manager.get('_xtest@example.com') + >>> mlist_2 = list_manager.get('test@example.com') + >>> mlist_2 is mlist + True + +You can also get a mailing list by it's list id. + + >>> mlist_2 = list_manager.get_by_list_id('test.example.com') >>> mlist_2 is mlist True If you try to get a list that doesn't existing yet, you get ``None``. - >>> print list_manager.get('_xtest_2@example.com') + >>> print list_manager.get('test_2@example.com') + None + >>> print list_manager.get_by_list_id('test_2.example.com') None You also get ``None`` if the list name is invalid. @@ -93,25 +104,34 @@ iterate over the mailing list objects, the list posting addresses, or the list address components. :: - >>> mlist_3 = list_manager.create('_xtest_3@example.com') - >>> mlist_4 = list_manager.create('_xtest_4@example.com') + >>> mlist_3 = list_manager.create('test_3@example.com') + >>> mlist_4 = list_manager.create('test_4@example.com') >>> for name in sorted(list_manager.names): ... print name - _xtest@example.com - _xtest_3@example.com - _xtest_4@example.com + test@example.com + test_3@example.com + test_4@example.com + + >>> for list_id in sorted(list_manager.list_ids): + ... print list_id + test.example.com + test_3.example.com + test_4.example.com >>> for fqdn_listname in sorted(m.fqdn_listname ... for m in list_manager.mailing_lists): ... print fqdn_listname - _xtest@example.com - _xtest_3@example.com - _xtest_4@example.com + test@example.com + test_3@example.com + test_4@example.com >>> for list_name, mail_host in sorted(list_manager.name_components, ... key=lambda (name, host): name): ... print list_name, '@', mail_host - _xtest @ example.com - _xtest_3 @ example.com - _xtest_4 @ example.com + test @ example.com + test_3 @ example.com + test_4 @ example.com + + +.. _`RFC 2369`: http://www.faqs.org/rfcs/rfc2369.html diff --git a/src/mailman/model/docs/mailinglist.rst b/src/mailman/model/docs/mailinglist.rst index 895068e52..21c2f0fd8 100644 --- a/src/mailman/model/docs/mailinglist.rst +++ b/src/mailman/model/docs/mailinglist.rst @@ -5,11 +5,13 @@ Mailing lists .. XXX 2010-06-18 BAW: This documentation needs a lot more detail. The mailing list is a core object in Mailman. It is uniquely identified in -the system by its posting address, i.e. the email address you would send a -message to in order to post a message to the mailing list. This must be fully -qualified. +the system by its *list-id* which is derived from its posting address, +i.e. the email address you would send a message to in order to post a message +to the mailing list. The list id is defined in `RFC 2369`_. >>> mlist = create_list('aardvark@example.com') + >>> print mlist.list_id + aardvark.example.com >>> print mlist.fqdn_listname aardvark@example.com @@ -163,3 +165,6 @@ A user cannot subscribe to a mailing list without a preferred address. ... MissingPreferredAddressError: User must have a preferred address: <User "Elly Person" (2) at ...> + + +.. _`RFC 2369`: http://www.faqs.org/rfcs/rfc2369.html diff --git a/src/mailman/model/docs/membership.rst b/src/mailman/model/docs/membership.rst index f070b4d40..3286bfe6e 100644 --- a/src/mailman/model/docs/membership.rst +++ b/src/mailman/model/docs/membership.rst @@ -283,8 +283,8 @@ though that the address their changing to must be verified. >>> gwen_address = list(gwen.addresses)[0] >>> gwen_member = bee.subscribe(gwen_address) >>> for m in bee.members.members: - ... print m.member_id.int, m.mailing_list, m.address.email - 7 bee@example.com gwen@example.com + ... print m.member_id.int, m.mailing_list.list_id, m.address.email + 7 bee.example.com gwen@example.com Gwen gets a email address. @@ -301,8 +301,8 @@ address, but the address is not yet verified. Her membership has not changed. >>> for m in bee.members.members: - ... print m.member_id.int, m.mailing_list, m.address.email - 7 bee@example.com gwen@example.com + ... print m.member_id.int, m.mailing_list.list_id, m.address.email + 7 bee.example.com gwen@example.com Gwen verifies her email address, and updates her membership. @@ -313,5 +313,5 @@ Gwen verifies her email address, and updates her membership. Now her membership reflects the new address. >>> for m in bee.members.members: - ... print m.member_id.int, m.mailing_list, m.address.email - 7 bee@example.com gperson@example.com + ... print m.member_id.int, m.mailing_list.list_id, m.address.email + 7 bee.example.com gperson@example.com diff --git a/src/mailman/model/docs/pending.rst b/src/mailman/model/docs/pending.rst index 1bf1ee0e9..3d33dd5da 100644 --- a/src/mailman/model/docs/pending.rst +++ b/src/mailman/model/docs/pending.rst @@ -15,11 +15,14 @@ In order to pend an event, you first need a pending database. The pending database can add any ``IPendable`` to the database, returning a token that can be used in urls and such. +:: - >>> from zope.interface import implements + >>> from zope.interface import implementer >>> from mailman.interfaces.pending import IPendable - >>> class SimplePendable(dict): - ... implements(IPendable) + >>> @implementer(IPendable) + ... class SimplePendable(dict): + ... pass + >>> subscription = SimplePendable( ... type='subscription', ... address='aperson@example.com', diff --git a/src/mailman/model/docs/registration.rst b/src/mailman/model/docs/registration.rst index eecb3a8cd..58e9d7a86 100644 --- a/src/mailman/model/docs/registration.rst +++ b/src/mailman/model/docs/registration.rst @@ -318,12 +318,15 @@ confirm method will just return False. Likewise, if you try to confirm, through the `IUserRegistrar` interface, a token that doesn't match a registration event, you will get ``None``. However, the pending event matched with that token will still be removed. +:: >>> from mailman.interfaces.pending import IPendable - >>> from zope.interface import implements + >>> from zope.interface import implementer + + >>> @implementer(IPendable) + ... class SimplePendable(dict): + ... pass - >>> class SimplePendable(dict): - ... implements(IPendable) >>> pendable = SimplePendable(type='foo', bar='baz') >>> token = pendingdb.add(pendable) >>> registrar.confirm(token) diff --git a/src/mailman/model/docs/requests.rst b/src/mailman/model/docs/requests.rst index a20823a91..a51cbc099 100644 --- a/src/mailman/model/docs/requests.rst +++ b/src/mailman/model/docs/requests.rst @@ -696,7 +696,7 @@ Frank Person is now a member of the mailing list. >>> print member.user.display_name Frank Person >>> print member.user.password - {CLEARTEXT}abcxyz + {plaintext}abcxyz Holding unsubscription requests diff --git a/src/mailman/model/docs/usermanager.rst b/src/mailman/model/docs/usermanager.rst index 727f82835..cf7672b27 100644 --- a/src/mailman/model/docs/usermanager.rst +++ b/src/mailman/model/docs/usermanager.rst @@ -173,8 +173,9 @@ There are now four members in the system. Sort them by address then role. ... return (member.address.email, member.role.name) >>> members = sorted(user_manager.members, key=sort_key) >>> for member in members: - ... print member.mailing_list, member.address.email, member.role - test@example.com bperson@example.com MemberRole.member - test@example.com bperson@example.com MemberRole.owner - test@example.com eperson@example.com MemberRole.member - test@example.com fperson@example.com MemberRole.member + ... print member.mailing_list.list_id, member.address.email, \ + ... member.role + test.example.com bperson@example.com MemberRole.member + test.example.com bperson@example.com MemberRole.owner + test.example.com eperson@example.com MemberRole.member + test.example.com fperson@example.com MemberRole.member diff --git a/src/mailman/model/docs/users.rst b/src/mailman/model/docs/users.rst index 95e08a8d7..997f983b2 100644 --- a/src/mailman/model/docs/users.rst +++ b/src/mailman/model/docs/users.rst @@ -335,11 +335,12 @@ membership role. ... return (member.address.email, member.mailing_list, ... int(member.role)) >>> for member in sorted(members, key=sortkey): - ... print member.address.email, member.mailing_list, member.role - zperson@example.com xtest_1@example.com MemberRole.member - zperson@example.net xtest_3@example.com MemberRole.moderator - zperson@example.org xtest_2@example.com MemberRole.member - zperson@example.org xtest_2@example.com MemberRole.owner + ... print member.address.email, member.mailing_list.list_id, \ + ... member.role + zperson@example.com xtest_1.example.com MemberRole.member + zperson@example.net xtest_3.example.com MemberRole.moderator + zperson@example.org xtest_2.example.com MemberRole.member + zperson@example.org xtest_2.example.com MemberRole.owner .. _`usermanager.txt`: usermanager.html diff --git a/src/mailman/model/domain.py b/src/mailman/model/domain.py index 49c935740..de6a9005a 100644 --- a/src/mailman/model/domain.py +++ b/src/mailman/model/domain.py @@ -17,7 +17,7 @@ """Domains.""" -from __future__ import unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -29,10 +29,10 @@ __all__ = [ from urlparse import urljoin, urlparse from storm.locals import Int, Unicode from zope.event import notify -from zope.interface import implements +from zope.interface import implementer -from mailman.config import config from mailman.database.model import Model +from mailman.database.transaction import dbconnection from mailman.interfaces.domain import ( BadDomainSpecificationError, DomainCreatedEvent, DomainCreatingEvent, DomainDeletedEvent, DomainDeletingEvent, IDomain, IDomainManager) @@ -40,11 +40,10 @@ from mailman.model.mailinglist import MailingList +@implementer(IDomain) class Domain(Model): """Domains.""" - implements(IDomain) - id = Int(primary=True) mail_host = Unicode() @@ -90,9 +89,10 @@ class Domain(Model): return urlparse(self.base_url).scheme @property - def mailing_lists(self): + @dbconnection + def mailing_lists(self, store): """See `IDomain`.""" - mailing_lists = config.db.store.find( + mailing_lists = store.find( MailingList, MailingList.mail_host == self.mail_host) for mlist in mailing_lists: @@ -114,12 +114,13 @@ class Domain(Model): +@implementer(IDomainManager) class DomainManager: """Domain manager.""" - implements(IDomainManager) - - def add(self, mail_host, + @dbconnection + def add(self, store, + mail_host, description=None, base_url=None, contact_address=None): @@ -131,20 +132,22 @@ class DomainManager: 'Duplicate email host: %s' % mail_host) notify(DomainCreatingEvent(mail_host)) domain = Domain(mail_host, description, base_url, contact_address) - config.db.store.add(domain) + store.add(domain) notify(DomainCreatedEvent(domain)) return domain - def remove(self, mail_host): + @dbconnection + def remove(self, store, mail_host): domain = self[mail_host] notify(DomainDeletingEvent(domain)) - config.db.store.remove(domain) + store.remove(domain) notify(DomainDeletedEvent(mail_host)) return domain - def get(self, mail_host, default=None): + @dbconnection + def get(self, store, mail_host, default=None): """See `IDomainManager`.""" - domains = config.db.store.find(Domain, mail_host=mail_host) + domains = store.find(Domain, mail_host=mail_host) if domains.count() < 1: return default assert domains.count() == 1, ( @@ -159,14 +162,17 @@ class DomainManager: raise KeyError(mail_host) return domain - def __len__(self): - return config.db.store.find(Domain).count() + @dbconnection + def __len__(self, store): + return store.find(Domain).count() - def __iter__(self): + @dbconnection + def __iter__(self, store): """See `IDomainManager`.""" - for domain in config.db.store.find(Domain): + for domain in store.find(Domain): yield domain - def __contains__(self, mail_host): + @dbconnection + def __contains__(self, store, mail_host): """See `IDomainManager`.""" - return config.db.store.find(Domain, mail_host=mail_host).count() > 0 + return store.find(Domain, mail_host=mail_host).count() > 0 diff --git a/src/mailman/model/language.py b/src/mailman/model/language.py index da86b326c..b593721df 100644 --- a/src/mailman/model/language.py +++ b/src/mailman/model/language.py @@ -17,7 +17,7 @@ """Model for languages.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -26,15 +26,16 @@ __all__ = [ from storm.locals import Int, Unicode -from zope.interface import implements +from zope.interface import implementer from mailman.database import Model from mailman.interfaces import ILanguage +@implementer(ILanguage) class Language(Model): - implements(ILanguage) + """See `ILanguage`.""" id = Int(primary=True) code = Unicode() diff --git a/src/mailman/model/listmanager.py b/src/mailman/model/listmanager.py index 0ea87a082..ce94047dd 100644 --- a/src/mailman/model/listmanager.py +++ b/src/mailman/model/listmanager.py @@ -17,7 +17,7 @@ """A mailing list manager.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -26,9 +26,9 @@ __all__ = [ from zope.event import notify -from zope.interface import implements +from zope.interface import implementer -from mailman.config import config +from mailman.database.transaction import dbconnection from mailman.interfaces.address import InvalidEmailAddressError from mailman.interfaces.listmanager import ( IListManager, ListAlreadyExistsError, ListCreatedEvent, ListCreatingEvent, @@ -38,66 +38,84 @@ from mailman.utilities.datetime import now +@implementer(IListManager) class ListManager: """An implementation of the `IListManager` interface.""" - implements(IListManager) - - def create(self, fqdn_listname): + @dbconnection + def create(self, store, fqdn_listname): """See `IListManager`.""" listname, at, hostname = fqdn_listname.partition('@') if len(hostname) == 0: raise InvalidEmailAddressError(fqdn_listname) + list_id = '{0}.{1}'.format(listname, hostname) notify(ListCreatingEvent(fqdn_listname)) - mlist = config.db.store.find( + mlist = store.find( MailingList, - MailingList.list_name == listname, - MailingList.mail_host == hostname).one() + MailingList._list_id == list_id).one() if mlist: raise ListAlreadyExistsError(fqdn_listname) mlist = MailingList(fqdn_listname) mlist.created_at = now() - config.db.store.add(mlist) + store.add(mlist) notify(ListCreatedEvent(mlist)) return mlist - def get(self, fqdn_listname): + @dbconnection + def get(self, store, fqdn_listname): """See `IListManager`.""" listname, at, hostname = fqdn_listname.partition('@') - return config.db.store.find(MailingList, - list_name=listname, - mail_host=hostname).one() + list_id = '{0}.{1}'.format(listname, hostname) + return store.find(MailingList, MailingList._list_id == list_id).one() + + @dbconnection + def get_by_list_id(self, store, list_id): + """See `IListManager`.""" + return store.find(MailingList, MailingList._list_id == list_id).one() - def delete(self, mlist): + @dbconnection + def delete(self, store, mlist): """See `IListManager`.""" fqdn_listname = mlist.fqdn_listname notify(ListDeletingEvent(mlist)) - config.db.store.remove(mlist) + store.remove(mlist) notify(ListDeletedEvent(fqdn_listname)) @property - def mailing_lists(self): + @dbconnection + def mailing_lists(self, store): """See `IListManager`.""" - for mlist in config.db.store.find(MailingList): + for mlist in store.find(MailingList): yield mlist - def __iter__(self): + @dbconnection + def __iter__(self, store): """See `IListManager`.""" - for mlist in config.db.store.find(MailingList): + for mlist in store.find(MailingList): yield mlist @property - def names(self): + @dbconnection + def names(self, store): """See `IListManager`.""" - result_set = config.db.store.find(MailingList) - for mail_host, list_name in result_set.values(MailingList.mail_host, + result_set = store.find(MailingList) + for mail_host, list_name in result_set.values(MailingList.mail_host, MailingList.list_name): yield '{0}@{1}'.format(list_name, mail_host) @property - def name_components(self): + @dbconnection + def list_ids(self, store): + """See `IListManager`.""" + result_set = store.find(MailingList) + for list_id in result_set.values(MailingList._list_id): + yield list_id + + @property + @dbconnection + def name_components(self, store): """See `IListManager`.""" - result_set = config.db.store.find(MailingList) - for mail_host, list_name in result_set.values(MailingList.mail_host, + result_set = store.find(MailingList) + for mail_host, list_name in result_set.values(MailingList.mail_host, MailingList.list_name): yield list_name, mail_host diff --git a/src/mailman/model/mailinglist.py b/src/mailman/model/mailinglist.py index d51c89514..2c55540be 100644 --- a/src/mailman/model/mailinglist.py +++ b/src/mailman/model/mailinglist.py @@ -17,7 +17,7 @@ """Model for mailing lists.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -33,13 +33,14 @@ from storm.locals import ( TimeDelta, Unicode) from urlparse import urljoin from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.database.model import Model from mailman.database.types import Enum from mailman.interfaces.action import Action, FilterAction from mailman.interfaces.address import IAddress +from mailman.interfaces.archiver import ArchivePolicy from mailman.interfaces.autorespond import ResponseAction from mailman.interfaces.bounce import UnrecognizedBounceDisposition from mailman.interfaces.digests import DigestFrequency @@ -51,7 +52,7 @@ from mailman.interfaces.mailinglist import ( from mailman.interfaces.member import ( AlreadySubscribedError, MemberRole, MissingPreferredAddressError) from mailman.interfaces.mime import FilterType -from mailman.interfaces.nntp import NewsModeration +from mailman.interfaces.nntp import NewsgroupModeration from mailman.interfaces.user import IUser from mailman.model import roster from mailman.model.digests import OneLastDigest @@ -67,8 +68,9 @@ UNDERSCORE = '_' +@implementer(IMailingList) class MailingList(Model): - implements(IMailingList) + """See `IMailingList`.""" id = Int(primary=True) @@ -78,7 +80,8 @@ class MailingList(Model): # List identity list_name = Unicode() mail_host = Unicode() - include_list_post_header = Bool() + _list_id = Unicode(name='list_id') + allow_list_posts = Bool() include_rfc2369_headers = Bool() advertised = Bool() anonymous_list = Bool() @@ -103,9 +106,7 @@ class MailingList(Model): admin_immed_notify = Bool() admin_notify_mchanges = Bool() administrivia = Bool() - archive = Bool() # XXX - archive_private = Bool() # XXX - archive_volume_frequency = Int() # XXX + archive_policy = Enum(ArchivePolicy) # Automatic responses. autoresponse_grace_period = TimeDelta() autorespond_owner = Enum(ResponseAction) @@ -148,7 +149,6 @@ class MailingList(Model): forward_auto_discards = Bool() gateway_to_mail = Bool() gateway_to_news = Bool() - generic_nonmember_action = Int() goodbye_message_uri = Unicode() header_matches = Pickle() header_uri = Unicode() @@ -162,9 +162,8 @@ class MailingList(Model): mime_is_default_digest = Bool() moderator_password = RawStr() new_member_options = Int() - news_moderation = Enum(NewsModeration) - news_prefix_subject_too = Bool() - nntp_host = Unicode() + newsgroup_moderation = Enum(NewsgroupModeration) + nntp_prefix_subject_too = Bool() nondigestable = Bool() nonmember_rejection_notice = Unicode() obscure_addresses = Bool() @@ -201,6 +200,7 @@ class MailingList(Model): assert hostname, 'Bad list name: {0}'.format(fqdn_listname) self.list_name = listname self.mail_host = hostname + self._list_id = '{0}.{1}'.format(listname, hostname) # For the pending database self.next_request_id = 1 # We need to set up the rosters. Normally, this method will get @@ -233,6 +233,11 @@ class MailingList(Model): return '{0}@{1}'.format(self.list_name, self.mail_host) @property + def list_id(self): + """See `IMailingList`.""" + return self._list_id + + @property def domain(self): """See `IMailingList`.""" return getUtility(IDomainManager)[self.mail_host] @@ -465,7 +470,7 @@ class MailingList(Model): member = store.find( Member, Member.role == role, - Member.mailing_list == self.fqdn_listname, + Member.list_id == self._list_id, Member._address == subscriber).one() if member: raise AlreadySubscribedError( @@ -476,7 +481,7 @@ class MailingList(Model): member = store.find( Member, Member.role == role, - Member.mailing_list == self.fqdn_listname, + Member.list_id == self._list_id, Member._user == subscriber).one() if member: raise AlreadySubscribedError( @@ -484,7 +489,7 @@ class MailingList(Model): else: raise ValueError('subscriber must be an address or user') member = Member(role=role, - mailing_list=self.fqdn_listname, + list_id=self._list_id, subscriber=subscriber) member.preferences = Preferences() store.add(member) @@ -492,8 +497,9 @@ class MailingList(Model): +@implementer(IAcceptableAlias) class AcceptableAlias(Model): - implements(IAcceptableAlias) + """See `IAcceptableAlias`.""" id = Int(primary=True) @@ -507,8 +513,10 @@ class AcceptableAlias(Model): self.alias = alias + +@implementer(IAcceptableAliasSet) class AcceptableAliasSet: - implements(IAcceptableAliasSet) + """See `IAcceptableAliasSet`.""" def __init__(self, mailing_list): self._mailing_list = mailing_list diff --git a/src/mailman/model/member.py b/src/mailman/model/member.py index ae83fb388..76fe2f992 100644 --- a/src/mailman/model/member.py +++ b/src/mailman/model/member.py @@ -17,7 +17,7 @@ """Model for members.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -27,11 +27,11 @@ __all__ = [ from storm.locals import Int, Reference, Unicode from storm.properties import UUID from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer -from mailman.config import config from mailman.core.constants import system_preferences from mailman.database.model import Model +from mailman.database.transaction import dbconnection from mailman.database.types import Enum from mailman.interfaces.action import Action from mailman.interfaces.address import IAddress @@ -46,13 +46,14 @@ uid_factory = UniqueIDFactory(context='members') +@implementer(IMember) class Member(Model): - implements(IMember) + """See `IMember`.""" id = Int(primary=True) _member_id = UUID() role = Enum(MemberRole) - mailing_list = Unicode() + list_id = Unicode() moderation_action = Enum(Action) address_id = Int() @@ -62,10 +63,10 @@ class Member(Model): user_id = Int() _user = Reference(user_id, 'User.id') - def __init__(self, role, mailing_list, subscriber): + def __init__(self, role, list_id, subscriber): self._member_id = uid_factory.new_uid() self.role = role - self.mailing_list = mailing_list + self.list_id = list_id if IAddress.providedBy(subscriber): self._address = subscriber # Look this up dynamically. @@ -79,17 +80,23 @@ class Member(Model): if role in (MemberRole.owner, MemberRole.moderator): self.moderation_action = Action.accept elif role is MemberRole.member: - self.moderation_action = getUtility(IListManager).get( - mailing_list).default_member_action + self.moderation_action = getUtility(IListManager).get_by_list_id( + list_id).default_member_action else: assert role is MemberRole.nonmember, ( 'Invalid MemberRole: {0}'.format(role)) - self.moderation_action = getUtility(IListManager).get( - mailing_list).default_nonmember_action + self.moderation_action = getUtility(IListManager).get_by_list_id( + list_id).default_nonmember_action def __repr__(self): return '<Member: {0} on {1} as {2}>'.format( - self.address, self.mailing_list, self.role) + self.address, self.mailing_list.fqdn_listname, self.role) + + @property + def mailing_list(self): + """See `IMember`.""" + list_manager = getUtility(IListManager) + return list_manager.get_by_list_id(self.list_id) @property def member_id(self): @@ -176,7 +183,8 @@ class Member(Model): # XXX Um, this is definitely wrong return 'http://example.com/' + self.address.email - def unsubscribe(self): + @dbconnection + def unsubscribe(self, store): """See `IMember`.""" - config.db.store.remove(self.preferences) - config.db.store.remove(self) + store.remove(self.preferences) + store.remove(self) diff --git a/src/mailman/model/message.py b/src/mailman/model/message.py index 3345c64c9..190b4055c 100644 --- a/src/mailman/model/message.py +++ b/src/mailman/model/message.py @@ -17,8 +17,7 @@ """Model for messages.""" - -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -26,28 +25,28 @@ __all__ = [ ] from storm.locals import AutoReload, Int, RawStr, Unicode -from zope.interface import implements +from zope.interface import implementer -from mailman.config import config from mailman.database.model import Model +from mailman.database.transaction import dbconnection from mailman.interfaces.messages import IMessage +@implementer(IMessage) class Message(Model): """A message in the message store.""" - implements(IMessage) - id = Int(primary=True, default=AutoReload) message_id = Unicode() message_id_hash = RawStr() path = RawStr() # This is a Messge-ID field representation, not a database row id. - def __init__(self, message_id, message_id_hash, path): + @dbconnection + def __init__(self, store, message_id, message_id_hash, path): super(Message, self).__init__() self.message_id = message_id self.message_id_hash = message_id_hash self.path = path - config.db.store.add(self) + store.add(self) diff --git a/src/mailman/model/messagestore.py b/src/mailman/model/messagestore.py index 59490993b..156375e6f 100644 --- a/src/mailman/model/messagestore.py +++ b/src/mailman/model/messagestore.py @@ -17,8 +17,7 @@ """Model for message stores.""" - -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -31,9 +30,10 @@ import base64 import hashlib import cPickle as pickle -from zope.interface import implements +from zope.interface import implementer from mailman.config import config +from mailman.database.transaction import dbconnection from mailman.interfaces.messages import IMessageStore from mailman.model.message import Message from mailman.utilities.filesystem import makedirs @@ -46,10 +46,12 @@ EMPTYSTRING = '' +@implementer(IMessageStore) class MessageStore: - implements(IMessageStore) + """See `IMessageStore`.""" - def add(self, message): + @dbconnection + def add(self, store, message): # Ensure that the message has the requisite headers. message_ids = message.get_all('message-id', []) if len(message_ids) <> 1: @@ -57,8 +59,7 @@ class MessageStore: # Calculate and insert the X-Message-ID-Hash. message_id = message_ids[0] # Complain if the Message-ID already exists in the storage. - existing = config.db.store.find(Message, - Message.message_id == message_id).one() + existing = store.find(Message, Message.message_id == message_id).one() if existing is not None: raise ValueError( 'Message ID already exists in message store: {0}'.format( @@ -104,34 +105,37 @@ class MessageStore: with open(path) as fp: return pickle.load(fp) - def get_message_by_id(self, message_id): - row = config.db.store.find(Message, message_id=message_id).one() + @dbconnection + def get_message_by_id(self, store, message_id): + row = store.find(Message, message_id=message_id).one() if row is None: return None return self._get_message(row) - def get_message_by_hash(self, message_id_hash): + @dbconnection + def get_message_by_hash(self, store, message_id_hash): # It's possible the hash came from a message header, in which case it # will be a Unicode. However when coming from source code, it may be # an 8-string. Coerce to the latter if necessary; it must be # US-ASCII. if isinstance(message_id_hash, unicode): message_id_hash = message_id_hash.encode('ascii') - row = config.db.store.find(Message, - message_id_hash=message_id_hash).one() + row = store.find(Message, message_id_hash=message_id_hash).one() if row is None: return None return self._get_message(row) @property - def messages(self): - for row in config.db.store.find(Message): + @dbconnection + def messages(self, store): + for row in store.find(Message): yield self._get_message(row) - def delete_message(self, message_id): - row = config.db.store.find(Message, message_id=message_id).one() + @dbconnection + def delete_message(self, store, message_id): + row = store.find(Message, message_id=message_id).one() if row is None: raise LookupError(message_id) path = os.path.join(config.MESSAGES_DIR, row.path) os.remove(path) - config.db.store.remove(row) + store.remove(row) diff --git a/src/mailman/model/mime.py b/src/mailman/model/mime.py index c611aab89..462bb9016 100644 --- a/src/mailman/model/mime.py +++ b/src/mailman/model/mime.py @@ -15,9 +15,9 @@ # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see <http://www.gnu.org/licenses/>. -"""Module stuff.""" +"""The content filter.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -26,7 +26,7 @@ __all__ = [ from storm.locals import Int, Reference, Unicode -from zope.interface import implements +from zope.interface import implementer from mailman.database.model import Model from mailman.database.types import Enum @@ -34,9 +34,9 @@ from mailman.interfaces.mime import IContentFilter, FilterType +@implementer(IContentFilter) class ContentFilter(Model): """A single filter criteria.""" - implements(IContentFilter) id = Int(primary=True) diff --git a/src/mailman/model/pending.py b/src/mailman/model/pending.py index 557361c6f..727e4f754 100644 --- a/src/mailman/model/pending.py +++ b/src/mailman/model/pending.py @@ -17,7 +17,7 @@ """Implementations of the IPendable and IPending interfaces.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -32,11 +32,12 @@ import hashlib from lazr.config import as_timedelta from storm.locals import DateTime, Int, RawStr, ReferenceSet, Unicode -from zope.interface import implements +from zope.interface import implementer from zope.interface.verify import verifyObject from mailman.config import config from mailman.database.model import Model +from mailman.database.transaction import dbconnection from mailman.interfaces.pending import ( IPendable, IPended, IPendedKeyValue, IPendings) from mailman.utilities.datetime import now @@ -44,11 +45,10 @@ from mailman.utilities.modules import call_name +@implementer(IPendedKeyValue) class PendedKeyValue(Model): """A pended key/value pair, tied to a token.""" - implements(IPendedKeyValue) - def __init__(self, key, value): self.key = key self.value = value @@ -59,11 +59,11 @@ class PendedKeyValue(Model): pended_id = Int() + +@implementer(IPended) class Pended(Model): """A pended event, tied to a token.""" - implements(IPended) - def __init__(self, token, expiration_date): super(Pended, self).__init__() self.token = token @@ -76,17 +76,18 @@ class Pended(Model): +@implementer(IPendable) class UnpendedPendable(dict): - implements(IPendable) + pass +@implementer(IPendings) class Pendings: """Implementation of the IPending interface.""" - implements(IPendings) - - def add(self, pendable, lifetime=None): + @dbconnection + def add(self, store, pendable, lifetime=None): verifyObject(IPendable, pendable) # Calculate the token and the lifetime. if lifetime is None: @@ -104,7 +105,7 @@ class Pendings: token = hashlib.sha1(repr(x)).hexdigest() # In practice, we'll never get a duplicate, but we'll be anal # about checking anyway. - if config.db.store.find(Pended, token=token).count() == 0: + if store.find(Pended, token=token).count() == 0: break else: raise AssertionError('Could not find a valid pendings token') @@ -129,11 +130,11 @@ class Pendings: '\2'.join(value)) keyval = PendedKeyValue(key=key, value=value) pending.key_values.add(keyval) - config.db.store.add(pending) + store.add(pending) return token - def confirm(self, token, expunge=True): - store = config.db.store + @dbconnection + def confirm(self, store, token, expunge=True): # Token can come in as a unicode, but it's stored in the database as # bytes. They must be ascii. pendings = store.find(Pended, token=str(token)) @@ -158,8 +159,8 @@ class Pendings: store.remove(pending) return pendable - def evict(self): - store = config.db.store + @dbconnection + def evict(self, store): right_now = now() for pending in store.find(Pended): if pending.expiration_date < right_now: diff --git a/src/mailman/model/preferences.py b/src/mailman/model/preferences.py index 234c7399e..fdc30a94d 100644 --- a/src/mailman/model/preferences.py +++ b/src/mailman/model/preferences.py @@ -17,7 +17,7 @@ """Model for preferences.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -27,7 +27,7 @@ __all__ = [ from storm.locals import Bool, Int, Unicode from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer from mailman.database.model import Model from mailman.database.types import Enum @@ -37,8 +37,9 @@ from mailman.interfaces.preferences import IPreferences +@implementer(IPreferences) class Preferences(Model): - implements(IPreferences) + """See `IPreferences`.""" id = Int(primary=True) acknowledge_posts = Bool() diff --git a/src/mailman/model/requests.py b/src/mailman/model/requests.py index 4a3efa67f..5eb940233 100644 --- a/src/mailman/model/requests.py +++ b/src/mailman/model/requests.py @@ -17,81 +17,95 @@ """Implementations of the pending requests interfaces.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ ] +from cPickle import dumps, loads from datetime import timedelta from storm.locals import AutoReload, Int, RawStr, Reference, Unicode from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer -from mailman.config import config from mailman.database.model import Model +from mailman.database.transaction import dbconnection from mailman.database.types import Enum from mailman.interfaces.pending import IPendable, IPendings from mailman.interfaces.requests import IListRequests, RequestType +@implementer(IPendable) class DataPendable(dict): - implements(IPendable) + def update(self, mapping): + # Keys and values must be strings (unicodes, but bytes values are + # accepted for now). Any other types for keys are a programming + # error. If we find a non-Unicode value, pickle it and encode it in + # such a way that it will be properly reconstituted when unpended. + clean_mapping = {} + for key, value in mapping.items(): + assert isinstance(key, basestring) + if not isinstance(value, unicode): + key = '_pck_' + key + value = dumps(value).decode('raw-unicode-escape') + clean_mapping[key] = value + super(DataPendable, self).update(clean_mapping) +@implementer(IListRequests) class ListRequests: - implements(IListRequests) def __init__(self, mailing_list): self.mailing_list = mailing_list @property - def count(self): - return config.db.store.find( - _Request, mailing_list=self.mailing_list).count() + @dbconnection + def count(self, store): + return store.find(_Request, mailing_list=self.mailing_list).count() - def count_of(self, request_type): - return config.db.store.find( + @dbconnection + def count_of(self, store, request_type): + return store.find( _Request, mailing_list=self.mailing_list, request_type=request_type).count() @property - def held_requests(self): - results = config.db.store.find( - _Request, mailing_list=self.mailing_list) + @dbconnection + def held_requests(self, store): + results = store.find(_Request, mailing_list=self.mailing_list) for request in results: yield request - def of_type(self, request_type): - results = config.db.store.find( + @dbconnection + def of_type(self, store, request_type): + results = store.find( _Request, mailing_list=self.mailing_list, request_type=request_type) for request in results: yield request - def hold_request(self, request_type, key, data=None): + @dbconnection + def hold_request(self, store, request_type, key, data=None): if request_type not in RequestType: raise TypeError(request_type) if data is None: data_hash = None else: - # We're abusing the pending database as a way of storing arbitrary - # key/value pairs, where both are strings. This isn't ideal but - # it lets us get auxiliary data almost for free. We may need to - # lock this down more later. pendable = DataPendable() pendable.update(data) token = getUtility(IPendings).add(pendable, timedelta(days=5000)) data_hash = token request = _Request(key, request_type, self.mailing_list, data_hash) - config.db.store.add(request) + store.add(request) return request.id - def get_request(self, request_id, request_type=None): - result = config.db.store.get(_Request, request_id) + @dbconnection + def get_request(self, store, request_id, request_type=None): + result = store.get(_Request, request_id) if result is None: return None if request_type is not None and result.request_type != request_type: @@ -101,16 +115,22 @@ class ListRequests: pendable = getUtility(IPendings).confirm( result.data_hash, expunge=False) data = dict() - data.update(pendable) + # Unpickle any non-Unicode values. + for key, value in pendable.items(): + if key.startswith('_pck_'): + data[key[5:]] = loads(value.encode('raw-unicode-escape')) + else: + data[key] = value return result.key, data - def delete_request(self, request_id): - request = config.db.store.get(_Request, request_id) + @dbconnection + def delete_request(self, store, request_id): + request = store.get(_Request, request_id) if request is None: raise KeyError(request_id) # Throw away the pended data. getUtility(IPendings).confirm(request.data_hash) - config.db.store.remove(request) + store.remove(request) diff --git a/src/mailman/model/roster.py b/src/mailman/model/roster.py index 48d434ab1..f6f86fbeb 100644 --- a/src/mailman/model/roster.py +++ b/src/mailman/model/roster.py @@ -22,7 +22,7 @@ the ones that fit a particular role. These are used as the member, owner, moderator, and administrator roster filters. """ -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -38,9 +38,9 @@ __all__ = [ from storm.expr import And, Or -from zope.interface import implements +from zope.interface import implementer -from mailman.config import config +from mailman.database.transaction import dbconnection from mailman.interfaces.member import DeliveryMode, MemberRole from mailman.interfaces.roster import IRoster from mailman.model.address import Address @@ -48,6 +48,7 @@ from mailman.model.member import Member +@implementer(IRoster) class AbstractRoster: """An abstract IRoster class. @@ -57,18 +58,17 @@ class AbstractRoster: This requires that subclasses implement the 'members' property. """ - implements(IRoster) - role = None def __init__(self, mlist): self._mlist = mlist - def _query(self): - return config.db.store.find( + @dbconnection + def _query(self, store): + return store.find( Member, - mailing_list=self._mlist.fqdn_listname, - role=self.role) + Member.list_id == self._mlist.list_id, + Member.role == self.role) @property def members(self): @@ -101,11 +101,12 @@ class AbstractRoster: for member in self.members: yield member.address - def get_member(self, address): + @dbconnection + def get_member(self, store, address): """See `IRoster`.""" - results = config.db.store.find( + results = store.find( Member, - Member.mailing_list == self._mlist.fqdn_listname, + Member.list_id == self._mlist.list_id, Member.role == self.role, Address.email == address, Member.address_id == Address.id) @@ -157,18 +158,20 @@ class AdministratorRoster(AbstractRoster): name = 'administrator' - def _query(self): - return config.db.store.find( + @dbconnection + def _query(self, store): + return store.find( Member, - Member.mailing_list == self._mlist.fqdn_listname, + Member.list_id == self._mlist.list_id, Or(Member.role == MemberRole.owner, Member.role == MemberRole.moderator)) - def get_member(self, address): + @dbconnection + def get_member(self, store, address): """See `IRoster`.""" - results = config.db.store.find( + results = store.find( Member, - Member.mailing_list == self._mlist.fqdn_listname, + Member.list_id == self._mlist.list_id, Or(Member.role == MemberRole.moderator, Member.role == MemberRole.owner), Address.email == address, @@ -194,7 +197,8 @@ class DeliveryMemberRoster(AbstractRoster): # checking the delivery mode to a query parameter. return len(tuple(self.members)) - def _get_members(self, *delivery_modes): + @dbconnection + def _get_members(self, store, *delivery_modes): """The set of members for a mailing list, filter by delivery mode. :param delivery_modes: The modes to filter on. @@ -202,9 +206,9 @@ class DeliveryMemberRoster(AbstractRoster): :return: A generator of members. :rtype: generator """ - results = config.db.store.find( + results = store.find( Member, - And(Member.mailing_list == self._mlist.fqdn_listname, + And(Member.list_id == self._mlist.list_id, Member.role == MemberRole.member)) for member in results: if member.delivery_mode in delivery_modes: @@ -244,25 +248,24 @@ class Subscribers(AbstractRoster): name = 'subscribers' - def _query(self): - return config.db.store.find( - Member, - mailing_list=self._mlist.fqdn_listname) + @dbconnection + def _query(self, store): + return store.find(Member, Member.list_id == self._mlist.list_id) +@implementer(IRoster) class Memberships: """A roster of a single user's memberships.""" - implements(IRoster) - name = 'memberships' def __init__(self, user): self._user = user - def _query(self): - results = config.db.store.find( + @dbconnection + def _query(self, store): + results = store.find( Member, Or(Member.user_id == self._user.id, And(Address.user_id == self._user.id, @@ -291,9 +294,10 @@ class Memberships: for address in self._user.addresses: yield address - def get_member(self, address): + @dbconnection + def get_member(self, store, address): """See `IRoster`.""" - results = config.db.store.find( + results = store.find( Member, Member.address_id == Address.id, Address.user_id == self._user.id) diff --git a/src/mailman/model/tests/test_bounce.py b/src/mailman/model/tests/test_bounce.py index da2b661ea..377fab4cc 100644 --- a/src/mailman/model/tests/test_bounce.py +++ b/src/mailman/model/tests/test_bounce.py @@ -17,7 +17,7 @@ """Test bounce model objects.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -30,7 +30,7 @@ from datetime import datetime from zope.component import getUtility from mailman.app.lifecycle import create_list -from mailman.config import config +from mailman.database.transaction import transaction from mailman.interfaces.bounce import BounceContext, IBounceProcessor from mailman.testing.helpers import ( specialized_message_from_string as message_from_string) @@ -52,8 +52,9 @@ Message-Id: <first> """) def test_events_iterator(self): - self._processor.register(self._mlist, 'anne@example.com', self._msg) - config.db.commit() + with transaction(): + self._processor.register( + self._mlist, 'anne@example.com', self._msg) events = list(self._processor.events) self.assertEqual(len(events), 1) event = events[0] @@ -75,23 +76,25 @@ Message-Id: <first> self.assertEqual(event.processed, False) def test_unprocessed_events_iterator(self): - self._processor.register(self._mlist, 'anne@example.com', self._msg) - self._processor.register(self._mlist, 'bart@example.com', self._msg) - config.db.commit() + with transaction(): + self._processor.register( + self._mlist, 'anne@example.com', self._msg) + self._processor.register( + self._mlist, 'bart@example.com', self._msg) events = list(self._processor.events) self.assertEqual(len(events), 2) unprocessed = list(self._processor.unprocessed) # The unprocessed list will be exactly the same right now. self.assertEqual(len(unprocessed), 2) # Process one of the events. - events[0].processed = True - config.db.commit() + with transaction(): + events[0].processed = True # Now there will be only one unprocessed event. unprocessed = list(self._processor.unprocessed) self.assertEqual(len(unprocessed), 1) # Process the other event. - events[1].processed = True - config.db.commit() + with transaction(): + events[1].processed = True # Now there will be no unprocessed events. unprocessed = list(self._processor.unprocessed) self.assertEqual(len(unprocessed), 0) diff --git a/src/mailman/model/uid.py b/src/mailman/model/uid.py index c3564aa40..08eae8aff 100644 --- a/src/mailman/model/uid.py +++ b/src/mailman/model/uid.py @@ -17,7 +17,7 @@ """Unique IDs.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -28,8 +28,8 @@ __all__ = [ from storm.locals import Int from storm.properties import UUID -from mailman.config import config from mailman.database.model import Model +from mailman.database.transaction import dbconnection @@ -48,23 +48,29 @@ class UID(Model): id = Int(primary=True) uid = UUID() - def __init__(self, uid): + @dbconnection + def __init__(self, store, uid): super(UID, self).__init__() self.uid = uid - config.db.store.add(self) + store.add(self) def __repr__(self): return '<UID {0} at {1}>'.format(self.uid, id(self)) @staticmethod - def record(uid): + @dbconnection + # Note that the parameter order is deliberate reversed here. Normally, + # `store` is the first parameter after `self`, but since this is a + # staticmethod and there is no self, the decorator will see the uid in + # arg[0]. + def record(uid, store): """Record the uid in the database. :param uid: The unique id. :type uid: unicode :raises ValueError: if the id is not unique. """ - existing = config.db.store.find(UID, uid=uid) + existing = store.find(UID, uid=uid) if existing.count() != 0: raise ValueError(uid) return UID(uid) diff --git a/src/mailman/model/user.py b/src/mailman/model/user.py index 9ca9b5aea..a723df44e 100644 --- a/src/mailman/model/user.py +++ b/src/mailman/model/user.py @@ -17,7 +17,7 @@ """Model for users.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -28,10 +28,10 @@ from storm.locals import ( DateTime, Int, RawStr, Reference, ReferenceSet, Unicode) from storm.properties import UUID from zope.event import notify -from zope.interface import implements +from zope.interface import implementer -from mailman.config import config from mailman.database.model import Model +from mailman.database.transaction import dbconnection from mailman.interfaces.address import ( AddressAlreadyLinkedError, AddressNotLinkedError) from mailman.interfaces.user import ( @@ -47,11 +47,10 @@ uid_factory = UniqueIDFactory(context='users') +@implementer(IUser) class User(Model): """Mailman users.""" - implements(IUser) - id = Int(primary=True) display_name = Unicode() _password = RawStr(name='password') @@ -64,16 +63,17 @@ class User(Model): preferences_id = Int() preferences = Reference(preferences_id, 'Preferences.id') - def __init__(self, display_name=None, preferences=None): + @dbconnection + def __init__(self, store, display_name=None, preferences=None): super(User, self).__init__() self._created_on = date_factory.now() user_id = uid_factory.new_uid() - assert config.db.store.find(User, _user_id=user_id).count() == 0, ( + assert store.find(User, _user_id=user_id).count() == 0, ( 'Duplicate user id {0}'.format(user_id)) self._user_id = user_id self.display_name = ('' if display_name is None else display_name) self.preferences = preferences - config.db.store.add(self) + store.add(self) def __repr__(self): short_user_id = self.user_id.int @@ -135,18 +135,20 @@ class User(Model): """See `IUser`.""" self._preferred_address = None - def controls(self, email): + @dbconnection + def controls(self, store, email): """See `IUser`.""" - found = config.db.store.find(Address, email=email) + found = store.find(Address, email=email) if found.count() == 0: return False assert found.count() == 1, 'Unexpected count' return found[0].user is self - def register(self, email, display_name=None): + @dbconnection + def register(self, store, email, display_name=None): """See `IUser`.""" # First, see if the address already exists - address = config.db.store.find(Address, email=email).one() + address = store.find(Address, email=email).one() if address is None: if display_name is None: display_name = '' diff --git a/src/mailman/model/usermanager.py b/src/mailman/model/usermanager.py index c8a5c65a2..4c7daaa59 100644 --- a/src/mailman/model/usermanager.py +++ b/src/mailman/model/usermanager.py @@ -17,7 +17,7 @@ """A user manager.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -25,9 +25,9 @@ __all__ = [ ] -from zope.interface import implements +from zope.interface import implementer -from mailman.config import config +from mailman.database.transaction import dbconnection from mailman.interfaces.address import ExistingAddressError from mailman.interfaces.usermanager import IUserManager from mailman.model.address import Address @@ -37,8 +37,9 @@ from mailman.model.user import User +@implementer(IUserManager) class UserManager: - implements(IUserManager) + """See `IUserManager`.""" def create_user(self, email=None, display_name=None): """See `IUserManager`.""" @@ -48,33 +49,38 @@ class UserManager: user.link(address) return user - def delete_user(self, user): + @dbconnection + def delete_user(self, store, user): """See `IUserManager`.""" - config.db.store.remove(user) + store.remove(user) - def get_user(self, email): + @dbconnection + def get_user(self, store, email): """See `IUserManager`.""" - addresses = config.db.store.find(Address, email=email.lower()) + addresses = store.find(Address, email=email.lower()) if addresses.count() == 0: return None return addresses.one().user - def get_user_by_id(self, user_id): + @dbconnection + def get_user_by_id(self, store, user_id): """See `IUserManager`.""" - users = config.db.store.find(User, _user_id=user_id) + users = store.find(User, _user_id=user_id) if users.count() == 0: return None return users.one() @property - def users(self): + @dbconnection + def users(self, store): """See `IUserManager`.""" - for user in config.db.store.find(User): + for user in store.find(User): yield user - def create_address(self, email, display_name=None): + @dbconnection + def create_address(self, store, email, display_name=None): """See `IUserManager`.""" - addresses = config.db.store.find(Address, email=email.lower()) + addresses = store.find(Address, email=email.lower()) if addresses.count() == 1: found = addresses[0] raise ExistingAddressError(found.original_email) @@ -85,32 +91,36 @@ class UserManager: # constructor will do the right thing. address = Address(email, display_name) address.preferences = Preferences() - config.db.store.add(address) + store.add(address) return address - def delete_address(self, address): + @dbconnection + def delete_address(self, store, address): """See `IUserManager`.""" # If there's a user controlling this address, it has to first be # unlinked before the address can be deleted. if address.user: address.user.unlink(address) - config.db.store.remove(address) + store.remove(address) - def get_address(self, email): + @dbconnection + def get_address(self, store, email): """See `IUserManager`.""" - addresses = config.db.store.find(Address, email=email.lower()) + addresses = store.find(Address, email=email.lower()) if addresses.count() == 0: return None return addresses.one() @property - def addresses(self): + @dbconnection + def addresses(self, store): """See `IUserManager`.""" - for address in config.db.store.find(Address): + for address in store.find(Address): yield address @property - def members(self): + @dbconnection + def members(self, store): """See `IUserManager.""" - for member in config.db.store.find(Member): + for member in store.find(Member): yield member diff --git a/src/mailman/model/version.py b/src/mailman/model/version.py index d6a4f3938..8b4dcae89 100644 --- a/src/mailman/model/version.py +++ b/src/mailman/model/version.py @@ -17,7 +17,7 @@ """Model class for version numbers.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -34,6 +34,10 @@ class Version(Model): component = Unicode() version = Unicode() + # The testing machinery will generally reset all tables, however because + # this table tracks schema migrations, we do not want to reset it. + PRESERVE = True + def __init__(self, component, version): super(Version, self).__init__() self.component = component diff --git a/src/mailman/mta/aliases.py b/src/mailman/mta/aliases.py index 573160d3c..e87f5880f 100644 --- a/src/mailman/mta/aliases.py +++ b/src/mailman/mta/aliases.py @@ -17,7 +17,7 @@ """Utility for generating all the aliases of a mailing list.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -25,7 +25,7 @@ __all__ = [ ] -from zope.interface import implements +from zope.interface import implementer from mailman.interfaces.mta import IMailTransportAgentAliases @@ -43,11 +43,10 @@ SUBDESTINATIONS = ( +@implementer(IMailTransportAgentAliases) class MailTransportAgentAliases: """Utility for generating all the aliases of a mailing list.""" - implements(IMailTransportAgentAliases) - def aliases(self, mlist): """See `IMailTransportAgentAliases`.""" # Always return diff --git a/src/mailman/mta/base.py b/src/mailman/mta/base.py index 8068eaec1..e11c37ff6 100644 --- a/src/mailman/mta/base.py +++ b/src/mailman/mta/base.py @@ -17,7 +17,7 @@ """Base delivery class.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -31,7 +31,7 @@ import socket import logging import smtplib -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.interfaces.mta import IMailTransportAgentDelivery @@ -42,11 +42,10 @@ log = logging.getLogger('mailman.smtp') +@implementer(IMailTransportAgentDelivery) class BaseDelivery: """Base delivery class.""" - implements(IMailTransportAgentDelivery) - def __init__(self): """Create a basic deliverer.""" username = (config.mta.smtp_user if config.mta.smtp_user else None) diff --git a/src/mailman/mta/null.py b/src/mailman/mta/null.py index 6b2f76a1c..c94fa1015 100644 --- a/src/mailman/mta/null.py +++ b/src/mailman/mta/null.py @@ -20,7 +20,7 @@ Exim one example of an MTA that Just Works. """ -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -28,16 +28,16 @@ __all__ = [ ] -from zope.interface import implements +from zope.interface import implementer + from mailman.interfaces.mta import IMailTransportAgentLifecycle +@implementer(IMailTransportAgentLifecycle) class NullMTA: """Null MTA that just satisfies the interface.""" - implements(IMailTransportAgentLifecycle) - def create(self, mlist): """See `IMailTransportAgentLifecycle`.""" pass diff --git a/src/mailman/mta/postfix.py b/src/mailman/mta/postfix.py index 32bdb8268..c04e38f02 100644 --- a/src/mailman/mta/postfix.py +++ b/src/mailman/mta/postfix.py @@ -17,7 +17,7 @@ """Creation/deletion hooks for the Postfix MTA.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -31,7 +31,7 @@ import logging from flufl.lock import Lock from operator import attrgetter from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.interfaces.listmanager import IListManager @@ -55,11 +55,10 @@ class _FakeList: +@implementer(IMailTransportAgentLifecycle) class LMTP: """Connect Mailman to Postfix via LMTP.""" - implements(IMailTransportAgentLifecycle) - def create(self, mlist): """See `IMailTransportAgentLifecycle`.""" # We can ignore the mlist argument because for LMTP delivery, we just @@ -117,23 +116,24 @@ class LMTP: for list_name, mail_host in list_manager.name_components: mlist = _FakeList(list_name, mail_host) by_domain.setdefault(mlist.mail_host, []).append(mlist) - print >> fp, """\ + print("""\ # AUTOMATICALLY GENERATED BY MAILMAN ON {0} # # This file is generated by Mailman, and is kept in sync with the binary hash # file. YOU SHOULD NOT MANUALLY EDIT THIS FILE unless you know what you're # doing, and can keep the two files properly in sync. If you screw it up, # you're on your own. -""".format(now().replace(microsecond=0)) +""".format(now().replace(microsecond=0)), file=fp) sort_key = attrgetter('list_name') for domain in sorted(by_domain): - print >> fp, """\ -# Aliases which are visible only in the @{0} domain.""".format(domain) + print("""\ +# Aliases which are visible only in the @{0} domain.""".format(domain), + file=fp) for mlist in sorted(by_domain[domain], key=sort_key): utility = getUtility(IMailTransportAgentAliases) aliases = list(utility.aliases(mlist)) width = max(len(alias) for alias in aliases) + 3 - print >> fp, ALIASTMPL.format(aliases.pop(0), config, width) + print(ALIASTMPL.format(aliases.pop(0), config, width), file=fp) for alias in aliases: - print >> fp, ALIASTMPL.format(alias, config, width) - print >> fp + print(ALIASTMPL.format(alias, config, width), file=fp) + print(file=fp) diff --git a/src/mailman/rest/addresses.py b/src/mailman/rest/addresses.py index e791dcfbd..2e81cb030 100644 --- a/src/mailman/rest/addresses.py +++ b/src/mailman/rest/addresses.py @@ -139,7 +139,7 @@ class UserAddresses(_AddressBase): def membership_key(member): # Sort first by mailing list, then by address, then by role. - return member.mailing_list, member.address.email, int(member.role) + return member.list_id, member.address.email, int(member.role) class AddressMemberships(MemberCollection): diff --git a/src/mailman/rest/configuration.py b/src/mailman/rest/configuration.py index d6b27cc6c..83d4c74f6 100644 --- a/src/mailman/rest/configuration.py +++ b/src/mailman/rest/configuration.py @@ -17,7 +17,7 @@ """Mailing list configuration via REST API.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -30,6 +30,7 @@ from restish import http, resource from mailman.config import config from mailman.interfaces.action import Action +from mailman.interfaces.archiver import ArchivePolicy from mailman.interfaces.autorespond import ResponseAction from mailman.interfaces.mailinglist import IAcceptableAliasSet, ReplyToMunging from mailman.rest.helpers import PATCH, etag, no_content @@ -170,6 +171,7 @@ ATTRIBUTES = dict( autoresponse_owner_text=GetterSetter(unicode), autoresponse_postings_text=GetterSetter(unicode), autoresponse_request_text=GetterSetter(unicode), + archive_policy=GetterSetter(enum_validator(ArchivePolicy)), bounces_address=GetterSetter(None), collapse_alternatives=GetterSetter(as_boolean), convert_html_to_plaintext=GetterSetter(as_boolean), @@ -181,9 +183,8 @@ ATTRIBUTES = dict( digest_size_threshold=GetterSetter(float), filter_content=GetterSetter(as_boolean), fqdn_listname=GetterSetter(None), - generic_nonmember_action=GetterSetter(int), mail_host=GetterSetter(None), - include_list_post_header=GetterSetter(as_boolean), + allow_list_posts=GetterSetter(as_boolean), include_rfc2369_headers=GetterSetter(as_boolean), join_address=GetterSetter(None), last_post_at=GetterSetter(None), diff --git a/src/mailman/rest/docs/addresses.rst b/src/mailman/rest/docs/addresses.rst index a8f875d12..cb9242d2b 100644 --- a/src/mailman/rest/docs/addresses.rst +++ b/src/mailman/rest/docs/addresses.rst @@ -170,16 +170,16 @@ Elle can get her memberships for each of her email addresses. entry 0: address: elle@example.com delivery_mode: regular - fqdn_listname: ant@example.com http_etag: "..." + list_id: ant.example.com role: member self_link: http://localhost:9001/3.0/members/1 user: http://localhost:9001/3.0/users/2 entry 1: address: elle@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: "..." + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/2 user: http://localhost:9001/3.0/users/2 @@ -207,16 +207,16 @@ does not show up in the list of memberships for his other address. entry 0: address: elle@example.com delivery_mode: regular - fqdn_listname: ant@example.com http_etag: "..." + list_id: ant.example.com role: member self_link: http://localhost:9001/3.0/members/1 user: http://localhost:9001/3.0/users/2 entry 1: address: elle@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: "..." + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/2 user: http://localhost:9001/3.0/users/2 @@ -229,8 +229,8 @@ does not show up in the list of memberships for his other address. entry 0: address: eperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: "..." + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/3 user: http://localhost:9001/3.0/users/2 diff --git a/src/mailman/rest/docs/configuration.rst b/src/mailman/rest/docs/configuration.rst index 676b3426c..4507345c3 100644 --- a/src/mailman/rest/docs/configuration.rst +++ b/src/mailman/rest/docs/configuration.rst @@ -20,7 +20,9 @@ All readable attributes for a list are available on a sub-resource. admin_notify_mchanges: False administrivia: True advertised: True + allow_list_posts: True anonymous_list: False + archive_policy: public autorespond_owner: none autorespond_postings: none autorespond_requests: none @@ -40,9 +42,7 @@ All readable attributes for a list are available on a sub-resource. display_name: Test-one filter_content: False fqdn_listname: test-one@example.com - generic_nonmember_action: 1 http_etag: "..." - include_list_post_header: True include_rfc2369_headers: True join_address: test-one-join@example.com last_post_at: None @@ -81,6 +81,7 @@ all the writable attributes in one request. ... administrivia=False, ... advertised=False, ... anonymous_list=True, + ... archive_policy='never', ... autorespond_owner='respond_and_discard', ... autorespond_postings='respond_and_continue', ... autorespond_requests='respond_and_discard', @@ -91,7 +92,7 @@ all the writable attributes in one request. ... display_name='Fnords', ... description='This is my mailing list', ... include_rfc2369_headers=False, - ... include_list_post_header=False, + ... allow_list_posts=False, ... digest_size_threshold=10.5, ... posting_pipeline='virgin', ... filter_content=True, @@ -102,7 +103,6 @@ all the writable attributes in one request. ... welcome_message_uri='mailman:///welcome.txt', ... default_member_action='hold', ... default_nonmember_action='discard', - ... generic_nonmember_action=2, ... ), ... 'PUT') content-length: 0 @@ -119,7 +119,9 @@ These values are changed permanently. admin_notify_mchanges: True administrivia: False advertised: False + allow_list_posts: False anonymous_list: True + archive_policy: never autorespond_owner: respond_and_discard autorespond_postings: respond_and_continue autorespond_requests: respond_and_discard @@ -139,7 +141,6 @@ These values are changed permanently. display_name: Fnords filter_content: True ... - include_list_post_header: False include_rfc2369_headers: False ... posting_pipeline: virgin @@ -161,6 +162,7 @@ must be included. It is an error to leave one or more out... ... administrivia=False, ... advertised=False, ... anonymous_list=True, + ... archive_policy='public', ... autorespond_owner='respond_and_discard', ... autorespond_postings='respond_and_continue', ... autorespond_requests='respond_and_discard', @@ -171,7 +173,7 @@ must be included. It is an error to leave one or more out... ... display_name='Fnords', ... description='This is my mailing list', ... include_rfc2369_headers=False, - ... include_list_post_header=False, + ... allow_list_posts=False, ... digest_size_threshold=10.5, ... posting_pipeline='virgin', ... filter_content=True, @@ -182,7 +184,6 @@ must be included. It is an error to leave one or more out... ... welcome_message_uri='welcome message', ... default_member_action='accept', ... default_nonmember_action='accept', - ... generic_nonmember_action=2, ... ), ... 'PUT') Traceback (most recent call last): @@ -201,6 +202,7 @@ must be included. It is an error to leave one or more out... ... administrivia=False, ... advertised=False, ... anonymous_list=True, + ... archive_policy='public', ... autorespond_owner='respond_and_discard', ... autorespond_postings='respond_and_continue', ... autorespond_requests='respond_and_discard', @@ -211,7 +213,7 @@ must be included. It is an error to leave one or more out... ... display_name='Fnords', ... description='This is my mailing list', ... include_rfc2369_headers=False, - ... include_list_post_header=False, + ... allow_list_posts=False, ... digest_size_threshold=10.5, ... posting_pipeline='virgin', ... filter_content=True, @@ -244,7 +246,7 @@ It is also an error to spell an attribute value incorrectly... ... display_name='Fnords', ... description='This is my mailing list', ... include_rfc2369_headers=False, - ... include_list_post_header=False, + ... allow_list_posts=False, ... digest_size_threshold=10.5, ... posting_pipeline='virgin', ... filter_content=True, @@ -266,6 +268,7 @@ It is also an error to spell an attribute value incorrectly... ... admin_notify_mchanges=True, ... advertised=False, ... anonymous_list=True, + ... archive_policy='public', ... autorespond_owner='respond_and_discard', ... autorespond_postings='respond_and_continue', ... autorespond_requests='respond_and_discard', @@ -276,7 +279,7 @@ It is also an error to spell an attribute value incorrectly... ... display_name='Fnords', ... description='This is my mailing list', ... include_rfc2369_headers=False, - ... include_list_post_header=False, + ... allow_list_posts=False, ... digest_size_threshold=10.5, ... posting_pipeline='dummy', ... filter_content=True, @@ -308,7 +311,7 @@ It is also an error to spell an attribute value incorrectly... ... display_name='Fnords', ... description='This is my mailing list', ... include_rfc2369_headers=False, - ... include_list_post_header=False, + ... allow_list_posts=False, ... digest_size_threshold=10.5, ... posting_pipeline='virgin', ... filter_content=True, diff --git a/src/mailman/rest/docs/membership.rst b/src/mailman/rest/docs/membership.rst index 860d33c21..6189990cb 100644 --- a/src/mailman/rest/docs/membership.rst +++ b/src/mailman/rest/docs/membership.rst @@ -43,8 +43,8 @@ the REST interface. entry 0: address: bperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/1 user: http://localhost:9001/3.0/users/1 @@ -57,8 +57,8 @@ Bart's specific membership can be accessed directly: >>> dump_json('http://localhost:9001/3.0/members/1') address: bperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/1 user: http://localhost:9001/3.0/users/1 @@ -71,16 +71,16 @@ the REST interface. entry 0: address: bperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/1 user: http://localhost:9001/3.0/users/1 entry 1: address: cperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/2 user: http://localhost:9001/3.0/users/2 @@ -98,24 +98,24 @@ subscribes, she is returned first. entry 0: address: aperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/3 user: http://localhost:9001/3.0/users/3 entry 1: address: bperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/1 user: http://localhost:9001/3.0/users/1 entry 2: address: cperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/2 user: http://localhost:9001/3.0/users/2 @@ -137,40 +137,40 @@ User ids are different than member ids. entry 0: address: aperson@example.com delivery_mode: regular - fqdn_listname: ant@example.com http_etag: ... + list_id: ant.example.com role: member self_link: http://localhost:9001/3.0/members/4 user: http://localhost:9001/3.0/users/3 entry 1: address: cperson@example.com delivery_mode: regular - fqdn_listname: ant@example.com http_etag: ... + list_id: ant.example.com role: member self_link: http://localhost:9001/3.0/members/5 user: http://localhost:9001/3.0/users/2 entry 2: address: aperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/3 user: http://localhost:9001/3.0/users/3 entry 3: address: bperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/1 user: http://localhost:9001/3.0/users/1 entry 4: address: cperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/2 user: http://localhost:9001/3.0/users/2 @@ -185,16 +185,16 @@ We can also get just the members of a single mailing list. entry 0: address: aperson@example.com delivery_mode: regular - fqdn_listname: ant@example.com http_etag: ... + list_id: ant.example.com role: member self_link: http://localhost:9001/3.0/members/4 user: http://localhost:9001/3.0/users/3 entry 1: address: cperson@example.com delivery_mode: regular - fqdn_listname: ant@example.com http_etag: ... + list_id: ant.example.com role: member self_link: http://localhost:9001/3.0/members/5 user: http://localhost:9001/3.0/users/2 @@ -212,7 +212,7 @@ mailing list. :: >>> dump_json('http://localhost:9001/3.0/members', { - ... 'fqdn_listname': 'ant@example.com', + ... 'list_id': 'ant.example.com', ... 'subscriber': 'dperson@example.com', ... 'role': 'moderator', ... }) @@ -223,7 +223,7 @@ mailing list. status: 201 >>> dump_json('http://localhost:9001/3.0/members', { - ... 'fqdn_listname': 'bee@example.com', + ... 'list_id': 'bee.example.com', ... 'subscriber': 'cperson@example.com', ... 'role': 'owner', ... }) @@ -237,56 +237,56 @@ mailing list. entry 0: address: dperson@example.com delivery_mode: regular - fqdn_listname: ant@example.com http_etag: ... + list_id: ant.example.com role: moderator self_link: http://localhost:9001/3.0/members/6 user: http://localhost:9001/3.0/users/4 entry 1: address: aperson@example.com delivery_mode: regular - fqdn_listname: ant@example.com http_etag: ... + list_id: ant.example.com role: member self_link: http://localhost:9001/3.0/members/4 user: http://localhost:9001/3.0/users/3 entry 2: address: cperson@example.com delivery_mode: regular - fqdn_listname: ant@example.com http_etag: ... + list_id: ant.example.com role: member self_link: http://localhost:9001/3.0/members/5 user: http://localhost:9001/3.0/users/2 entry 3: address: cperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: owner self_link: http://localhost:9001/3.0/members/7 user: http://localhost:9001/3.0/users/2 entry 4: address: aperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/3 user: http://localhost:9001/3.0/users/3 entry 5: address: bperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/1 user: http://localhost:9001/3.0/users/1 entry 6: address: cperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/2 user: http://localhost:9001/3.0/users/2 @@ -301,8 +301,8 @@ We can access all the owners of a list. entry 0: address: cperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: owner self_link: http://localhost:9001/3.0/members/7 user: http://localhost:9001/3.0/users/2 @@ -320,8 +320,8 @@ A specific member can always be referenced by their role and address. ... 'bee@example.com/owner/cperson@example.com') address: cperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: owner self_link: http://localhost:9001/3.0/members/7 user: http://localhost:9001/3.0/users/2 @@ -335,16 +335,16 @@ example, we can search for all the memberships of a particular address. entry 0: address: aperson@example.com delivery_mode: regular - fqdn_listname: ant@example.com http_etag: ... + list_id: ant.example.com role: member self_link: http://localhost:9001/3.0/members/4 user: http://localhost:9001/3.0/users/3 entry 1: address: aperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/3 user: http://localhost:9001/3.0/users/3 @@ -355,37 +355,37 @@ example, we can search for all the memberships of a particular address. Or, we can find all the memberships for a particular mailing list. >>> dump_json('http://localhost:9001/3.0/members/find', { - ... 'fqdn_listname': 'bee@example.com', + ... 'list_id': 'bee.example.com', ... }) entry 0: address: aperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/3 user: http://localhost:9001/3.0/users/3 entry 1: address: bperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/1 user: http://localhost:9001/3.0/users/1 entry 2: address: cperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/2 user: http://localhost:9001/3.0/users/2 entry 3: address: cperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: owner self_link: http://localhost:9001/3.0/members/7 user: http://localhost:9001/3.0/users/2 @@ -398,21 +398,21 @@ list. >>> dump_json('http://localhost:9001/3.0/members/find', { ... 'subscriber': 'cperson@example.com', - ... 'fqdn_listname': 'bee@example.com', + ... 'list_id': 'bee.example.com', ... }) entry 0: address: cperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/2 user: http://localhost:9001/3.0/users/2 entry 1: address: cperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: owner self_link: http://localhost:9001/3.0/members/7 user: http://localhost:9001/3.0/users/2 @@ -429,16 +429,16 @@ Or, we can find all the memberships for an address with a specific role. entry 0: address: cperson@example.com delivery_mode: regular - fqdn_listname: ant@example.com http_etag: ... + list_id: ant.example.com role: member self_link: http://localhost:9001/3.0/members/5 user: http://localhost:9001/3.0/users/2 entry 1: address: cperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/2 user: http://localhost:9001/3.0/users/2 @@ -450,14 +450,14 @@ Finally, we can search for a specific member given all three criteria. >>> dump_json('http://localhost:9001/3.0/members/find', { ... 'subscriber': 'cperson@example.com', - ... 'fqdn_listname': 'bee@example.com', + ... 'list_id': 'bee.example.com', ... 'role': 'member', ... }) entry 0: address: cperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: ... + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/2 user: http://localhost:9001/3.0/users/2 @@ -478,7 +478,7 @@ address is not yet known to Mailman, a user is created for her. By default, get gets a regular delivery. >>> dump_json('http://localhost:9001/3.0/members', { - ... 'fqdn_listname': 'ant@example.com', + ... 'list_id': 'ant.example.com', ... 'subscriber': 'eperson@example.com', ... 'display_name': 'Elly Person', ... }) @@ -495,8 +495,8 @@ Elly is now a known user, and a member of the mailing list. >>> elly <User "Elly Person" (...) at ...> - >>> set(member.mailing_list for member in elly.memberships.members) - set([u'ant@example.com']) + >>> set(member.list_id for member in elly.memberships.members) + set([u'ant.example.com']) >>> dump_json('http://localhost:9001/3.0/members') entry 0: @@ -504,8 +504,8 @@ Elly is now a known user, and a member of the mailing list. entry 3: address: eperson@example.com delivery_mode: regular - fqdn_listname: ant@example.com http_etag: ... + list_id: ant.example.com role: member self_link: http://localhost:9001/3.0/members/8 user: http://localhost:9001/3.0/users/5 @@ -530,7 +530,7 @@ list with her preferred address. >>> transaction.commit() >>> dump_json('http://localhost:9001/3.0/members', { - ... 'fqdn_listname': 'ant@example.com', + ... 'list_id': 'ant.example.com', ... 'subscriber': user_id, ... }) content-length: 0 @@ -545,8 +545,8 @@ list with her preferred address. entry 4: address: gwen@example.com delivery_mode: regular - fqdn_listname: ant@example.com http_etag: "..." + list_id: ant.example.com role: member self_link: http://localhost:9001/3.0/members/9 user: http://localhost:9001/3.0/users/6 @@ -568,8 +568,8 @@ the new address. entry 4: address: gwen.person@example.com delivery_mode: regular - fqdn_listname: ant@example.com http_etag: "..." + list_id: ant.example.com role: member self_link: http://localhost:9001/3.0/members/9 user: http://localhost:9001/3.0/users/6 @@ -606,7 +606,7 @@ Fred joins the `ant` mailing list but wants MIME digest delivery. >>> transaction.abort() >>> dump_json('http://localhost:9001/3.0/members', { - ... 'fqdn_listname': 'ant@example.com', + ... 'list_id': 'ant.example.com', ... 'subscriber': 'fperson@example.com', ... 'display_name': 'Fred Person', ... 'delivery_mode': 'mime_digests', @@ -633,8 +633,8 @@ Fred is getting MIME deliveries. >>> dump_json('http://localhost:9001/3.0/members/10') address: fperson@example.com delivery_mode: mime_digests - fqdn_listname: ant@example.com http_etag: "..." + list_id: ant.example.com role: member self_link: http://localhost:9001/3.0/members/10 user: http://localhost:9001/3.0/users/7 @@ -655,8 +655,8 @@ This can be done by PATCH'ing his member with the `delivery_mode` parameter. >>> dump_json('http://localhost:9001/3.0/members/10') address: fperson@example.com delivery_mode: regular - fqdn_listname: ant@example.com http_etag: "..." + list_id: ant.example.com role: member self_link: http://localhost:9001/3.0/members/10 user: http://localhost:9001/3.0/users/7 @@ -673,8 +673,8 @@ If a PATCH request changes no attributes, nothing happens. >>> dump_json('http://localhost:9001/3.0/members/10') address: fperson@example.com delivery_mode: regular - fqdn_listname: ant@example.com http_etag: "..." + list_id: ant.example.com role: member self_link: http://localhost:9001/3.0/members/10 user: http://localhost:9001/3.0/users/7 @@ -715,8 +715,8 @@ addresses. entry 5: address: herb@example.com delivery_mode: regular - fqdn_listname: ant@example.com http_etag: "..." + list_id: ant.example.com role: member self_link: http://localhost:9001/3.0/members/11 user: http://localhost:9001/3.0/users/8 @@ -724,8 +724,8 @@ addresses. entry 10: address: herb@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: "..." + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/12 user: http://localhost:9001/3.0/users/8 @@ -780,16 +780,16 @@ his membership ids have not changed. entry 0: address: hperson@example.com delivery_mode: regular - fqdn_listname: ant@example.com http_etag: "..." + list_id: ant.example.com role: member self_link: http://localhost:9001/3.0/members/11 user: http://localhost:9001/3.0/users/8 entry 1: address: hperson@example.com delivery_mode: regular - fqdn_listname: bee@example.com http_etag: "..." + list_id: bee.example.com role: member self_link: http://localhost:9001/3.0/members/12 user: http://localhost:9001/3.0/users/8 diff --git a/src/mailman/rest/docs/users.rst b/src/mailman/rest/docs/users.rst index a20306e17..cdede10ee 100644 --- a/src/mailman/rest/docs/users.rst +++ b/src/mailman/rest/docs/users.rst @@ -94,7 +94,7 @@ It is also available via the location given in the response. created_on: 2005-08-01T07:49:23 display_name: Bart Person http_etag: "..." - password: {CLEARTEXT}bbb + password: {plaintext}bbb self_link: http://localhost:9001/3.0/users/3 user_id: 3 @@ -105,7 +105,7 @@ them with user ids. Thus, a user can be retrieved via its email address. created_on: 2005-08-01T07:49:23 display_name: Bart Person http_etag: "..." - password: {CLEARTEXT}bbb + password: {plaintext}bbb self_link: http://localhost:9001/3.0/users/3 user_id: 3 @@ -129,7 +129,7 @@ therefore cannot be retrieved. It can be reset though. created_on: 2005-08-01T07:49:23 display_name: Cris Person http_etag: "..." - password: {CLEARTEXT}... + password: {plaintext}... self_link: http://localhost:9001/3.0/users/4 user_id: 4 @@ -227,7 +227,7 @@ In fact, any of these addresses can be used to look up Bart's user record. created_on: 2005-08-01T07:49:23 display_name: Bart Person http_etag: "..." - password: {CLEARTEXT}bbb + password: {plaintext}bbb self_link: http://localhost:9001/3.0/users/3 user_id: 3 @@ -235,7 +235,7 @@ In fact, any of these addresses can be used to look up Bart's user record. created_on: 2005-08-01T07:49:23 display_name: Bart Person http_etag: "..." - password: {CLEARTEXT}bbb + password: {plaintext}bbb self_link: http://localhost:9001/3.0/users/3 user_id: 3 @@ -243,7 +243,7 @@ In fact, any of these addresses can be used to look up Bart's user record. created_on: 2005-08-01T07:49:23 display_name: Bart Person http_etag: "..." - password: {CLEARTEXT}bbb + password: {plaintext}bbb self_link: http://localhost:9001/3.0/users/3 user_id: 3 @@ -251,6 +251,6 @@ In fact, any of these addresses can be used to look up Bart's user record. created_on: 2005-08-01T07:49:23 display_name: Bart Person http_etag: "..." - password: {CLEARTEXT}bbb + password: {plaintext}bbb self_link: http://localhost:9001/3.0/users/3 user_id: 3 diff --git a/src/mailman/rest/lists.py b/src/mailman/rest/lists.py index 3374e8f73..f25133211 100644 --- a/src/mailman/rest/lists.py +++ b/src/mailman/rest/lists.py @@ -137,7 +137,7 @@ class AList(_ListBase): """Delete the named mailing list.""" if self._mlist is None: return http.not_found() - remove_list(self._mlist.fqdn_listname, self._mlist) + remove_list(self._mlist) return no_content() @resource.child(member_matcher) @@ -146,7 +146,7 @@ class AList(_ListBase): if self._mlist is None: return http.not_found() members = getUtility(ISubscriptionService).find_members( - email, self._mlist.fqdn_listname, role) + email, self._mlist.list_id, role) if len(members) == 0: return http.not_found() assert len(members) == 1, 'Too many matches' diff --git a/src/mailman/rest/members.py b/src/mailman/rest/members.py index 761e3147c..c6aaa7e39 100644 --- a/src/mailman/rest/members.py +++ b/src/mailman/rest/members.py @@ -61,7 +61,7 @@ class _MemberBase(resource.Resource, CollectionMixin): user_id = member.user.user_id.int member_id = member.member_id.int return dict( - fqdn_listname=member.mailing_list, + list_id=member.list_id, address=member.address.email, role=role, user=path_to('users/{0}'.format(user_id)), @@ -148,7 +148,7 @@ class AMember(_MemberBase): # an admin or user notification. if self._member is None: return http.not_found() - mlist = getUtility(IListManager).get(self._member.mailing_list) + mlist = getUtility(IListManager).get_by_list_id(self._member.list_id) if self._member.role is MemberRole.member: try: delete_member(mlist, self._member.address.email, False, False) @@ -197,7 +197,7 @@ class AllMembers(_MemberBase): service = getUtility(ISubscriptionService) try: validator = Validator( - fqdn_listname=unicode, + list_id=unicode, subscriber=subscriber_validator, display_name=unicode, delivery_mode=enum_validator(DeliveryMode), @@ -248,10 +248,10 @@ class FindMembers(_MemberBase): """Find a member""" service = getUtility(ISubscriptionService) validator = Validator( - fqdn_listname=unicode, + list_id=unicode, subscriber=unicode, role=enum_validator(MemberRole), - _optional=('fqdn_listname', 'subscriber', 'role')) + _optional=('list_id', 'subscriber', 'role')) members = service.find_members(**validator(request)) # We can't just return the _FoundMembers instance, because # CollectionMixins have only a GET method, which is incompatible with diff --git a/src/mailman/rest/tests/test_addresses.py b/src/mailman/rest/tests/test_addresses.py index f4e7cab62..385b83912 100644 --- a/src/mailman/rest/tests/test_addresses.py +++ b/src/mailman/rest/tests/test_addresses.py @@ -17,10 +17,11 @@ """REST address tests.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ + 'TestAddresses', ] @@ -29,7 +30,7 @@ import unittest from urllib2 import HTTPError from mailman.app.lifecycle import create_list -from mailman.config import config +from mailman.database.transaction import transaction from mailman.testing.helpers import call_api from mailman.testing.layers import RESTLayer @@ -39,8 +40,8 @@ class TestAddresses(unittest.TestCase): layer = RESTLayer def setUp(self): - self._mlist = create_list('test@example.com') - config.db.commit() + with transaction(): + self._mlist = create_list('test@example.com') def test_membership_of_missing_address(self): # Try to get the memberships of a missing address. diff --git a/src/mailman/rest/tests/test_domains.py b/src/mailman/rest/tests/test_domains.py index 89cc34630..a86768481 100644 --- a/src/mailman/rest/tests/test_domains.py +++ b/src/mailman/rest/tests/test_domains.py @@ -17,10 +17,11 @@ """REST domain tests.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ + 'TestDomains', ] @@ -30,7 +31,7 @@ from urllib2 import HTTPError from zope.component import getUtility from mailman.app.lifecycle import create_list -from mailman.config import config +from mailman.database.transaction import transaction from mailman.interfaces.listmanager import IListManager from mailman.testing.helpers import call_api from mailman.testing.layers import RESTLayer @@ -41,8 +42,8 @@ class TestDomains(unittest.TestCase): layer = RESTLayer def setUp(self): - self._mlist = create_list('test@example.com') - config.db.commit() + with transaction(): + self._mlist = create_list('test@example.com') def test_bogus_endpoint_extension(self): # /domains/<domain>/lists/<anything> is not a valid endpoint. @@ -67,8 +68,8 @@ class TestDomains(unittest.TestCase): def test_lists_are_deleted_when_domain_is_deleted(self): # /domains/<domain> DELETE removes all associated mailing lists. - create_list('ant@example.com') - config.db.commit() + with transaction(): + create_list('ant@example.com') content, response = call_api( 'http://localhost:9001/3.0/domains/example.com', method='DELETE') self.assertEqual(response.status, 204) diff --git a/src/mailman/rest/tests/test_lists.py b/src/mailman/rest/tests/test_lists.py index b030a2e8d..cd0ebaf8e 100644 --- a/src/mailman/rest/tests/test_lists.py +++ b/src/mailman/rest/tests/test_lists.py @@ -32,7 +32,7 @@ from urllib2 import HTTPError from zope.component import getUtility from mailman.app.lifecycle import create_list -from mailman.config import config +from mailman.database.transaction import transaction from mailman.interfaces.usermanager import IUserManager from mailman.testing.helpers import call_api from mailman.testing.layers import RESTLayer @@ -96,8 +96,8 @@ class TestLists(unittest.TestCase): layer = RESTLayer def setUp(self): - self._mlist = create_list('test@example.com') - config.db.commit() + with transaction(): + self._mlist = create_list('test@example.com') self._usermanager = getUtility(IUserManager) def test_member_count_with_no_members(self): @@ -109,9 +109,9 @@ class TestLists(unittest.TestCase): def test_member_count_with_one_member(self): # Add a member to a list and check that the resource reflects this. - anne = self._usermanager.create_address('anne@example.com') - self._mlist.subscribe(anne) - config.db.commit() + with transaction(): + anne = self._usermanager.create_address('anne@example.com') + self._mlist.subscribe(anne) resource, response = call_api( 'http://localhost:9001/3.0/lists/test@example.com') self.assertEqual(response.status, 200) @@ -119,11 +119,11 @@ class TestLists(unittest.TestCase): def test_member_count_with_two_members(self): # Add two members to a list and check that the resource reflects this. - anne = self._usermanager.create_address('anne@example.com') - self._mlist.subscribe(anne) - bart = self._usermanager.create_address('bar@example.com') - self._mlist.subscribe(bart) - config.db.commit() + with transaction(): + anne = self._usermanager.create_address('anne@example.com') + self._mlist.subscribe(anne) + bart = self._usermanager.create_address('bar@example.com') + self._mlist.subscribe(bart) resource, response = call_api( 'http://localhost:9001/3.0/lists/test@example.com') self.assertEqual(response.status, 200) diff --git a/src/mailman/rest/tests/test_membership.py b/src/mailman/rest/tests/test_membership.py index 202e5f057..18469e537 100644 --- a/src/mailman/rest/tests/test_membership.py +++ b/src/mailman/rest/tests/test_membership.py @@ -17,10 +17,11 @@ """REST membership tests.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ + 'TestMembership', ] @@ -31,6 +32,7 @@ from zope.component import getUtility from mailman.app.lifecycle import create_list from mailman.config import config +from mailman.database.transaction import transaction from mailman.interfaces.usermanager import IUserManager from mailman.testing.helpers import call_api from mailman.testing.layers import RESTLayer @@ -42,8 +44,8 @@ class TestMembership(unittest.TestCase): layer = RESTLayer def setUp(self): - self._mlist = create_list('test@example.com') - config.db.commit() + with transaction(): + self._mlist = create_list('test@example.com') self._usermanager = getUtility(IUserManager) def test_try_to_join_missing_list(self): @@ -51,7 +53,7 @@ class TestMembership(unittest.TestCase): try: # For Python 2.6. call_api('http://localhost:9001/3.0/members', { - 'fqdn_listname': 'missing@example.com', + 'list_id': 'missing.example.com', 'subscriber': 'nobody@example.com', }) except HTTPError as exc: @@ -85,9 +87,9 @@ class TestMembership(unittest.TestCase): raise AssertionError('Expected HTTPError') def test_try_to_leave_a_list_twice(self): - anne = self._usermanager.create_address('anne@example.com') - self._mlist.subscribe(anne) - config.db.commit() + with transaction(): + anne = self._usermanager.create_address('anne@example.com') + self._mlist.subscribe(anne) url = 'http://localhost:9001/3.0/members/1' content, response = call_api(url, method='DELETE') # For a successful DELETE, the response code is 204 and there is no @@ -104,13 +106,13 @@ class TestMembership(unittest.TestCase): raise AssertionError('Expected HTTPError') def test_try_to_join_a_list_twice(self): - anne = self._usermanager.create_address('anne@example.com') - self._mlist.subscribe(anne) - config.db.commit() + with transaction(): + anne = self._usermanager.create_address('anne@example.com') + self._mlist.subscribe(anne) try: # For Python 2.6. call_api('http://localhost:9001/3.0/members', { - 'fqdn_listname': 'test@example.com', + 'list_id': 'test.example.com', 'subscriber': 'anne@example.com', }) except HTTPError as exc: @@ -122,7 +124,7 @@ class TestMembership(unittest.TestCase): def test_join_with_invalid_delivery_mode(self): try: call_api('http://localhost:9001/3.0/members', { - 'fqdn_listname': 'test@example.com', + 'list_id': 'test.example.com', 'subscriber': 'anne@example.com', 'display_name': 'Anne Person', 'delivery_mode': 'invalid-mode', @@ -136,7 +138,7 @@ class TestMembership(unittest.TestCase): def test_join_email_contains_slash(self): content, response = call_api('http://localhost:9001/3.0/members', { - 'fqdn_listname': 'test@example.com', + 'list_id': 'test.example.com', 'subscriber': 'hugh/person@example.com', 'display_name': 'Hugh Person', }) @@ -151,12 +153,12 @@ class TestMembership(unittest.TestCase): self.assertEqual(members[0].address.email, 'hugh/person@example.com') def test_join_as_user_with_preferred_address(self): - anne = self._usermanager.create_user('anne@example.com') - preferred = list(anne.addresses)[0] - preferred.verified_on = now() - anne.preferred_address = preferred - self._mlist.subscribe(anne) - config.db.commit() + with transaction(): + anne = self._usermanager.create_user('anne@example.com') + preferred = list(anne.addresses)[0] + preferred.verified_on = now() + anne.preferred_address = preferred + self._mlist.subscribe(anne) content, response = call_api('http://localhost:9001/3.0/members') self.assertEqual(response.status, 200) self.assertEqual(int(content['total_size']), 1) @@ -166,15 +168,15 @@ class TestMembership(unittest.TestCase): self.assertEqual(entry_0['role'], 'member') self.assertEqual(entry_0['user'], 'http://localhost:9001/3.0/users/1') self.assertEqual(entry_0['address'], 'anne@example.com') - self.assertEqual(entry_0['fqdn_listname'], 'test@example.com') + self.assertEqual(entry_0['list_id'], 'test.example.com') def test_member_changes_preferred_address(self): - anne = self._usermanager.create_user('anne@example.com') - preferred = list(anne.addresses)[0] - preferred.verified_on = now() - anne.preferred_address = preferred - self._mlist.subscribe(anne) - config.db.commit() + with transaction(): + anne = self._usermanager.create_user('anne@example.com') + preferred = list(anne.addresses)[0] + preferred.verified_on = now() + anne.preferred_address = preferred + self._mlist.subscribe(anne) # Take a look at Anne's current membership. content, response = call_api('http://localhost:9001/3.0/members') self.assertEqual(int(content['total_size']), 1) @@ -182,10 +184,10 @@ class TestMembership(unittest.TestCase): self.assertEqual(entry_0['address'], 'anne@example.com') # Anne registers a new address and makes it her preferred address. # There are no changes to her membership. - new_preferred = anne.register('aperson@example.com') - new_preferred.verified_on = now() - anne.preferred_address = new_preferred - config.db.commit() + with transaction(): + new_preferred = anne.register('aperson@example.com') + new_preferred.verified_on = now() + anne.preferred_address = new_preferred # Take another look at Anne's current membership. content, response = call_api('http://localhost:9001/3.0/members') self.assertEqual(int(content['total_size']), 1) @@ -214,9 +216,9 @@ class TestMembership(unittest.TestCase): def test_patch_member_bogus_attribute(self): # /members/<id> PATCH 'bogus' returns 400 - anne = self._usermanager.create_address('anne@example.com') - self._mlist.subscribe(anne) - config.db.commit() + with transaction(): + anne = self._usermanager.create_address('anne@example.com') + self._mlist.subscribe(anne) try: # For Python 2.6 call_api('http://localhost:9001/3.0/members/1', { diff --git a/src/mailman/rest/tests/test_moderation.py b/src/mailman/rest/tests/test_moderation.py index 79b0c8b80..dfcedef05 100644 --- a/src/mailman/rest/tests/test_moderation.py +++ b/src/mailman/rest/tests/test_moderation.py @@ -31,6 +31,7 @@ from urllib2 import HTTPError from mailman.app.lifecycle import create_list from mailman.app.moderator import hold_message, hold_subscription from mailman.config import config +from mailman.database.transaction import transaction from mailman.interfaces.member import DeliveryMode from mailman.testing.helpers import ( call_api, specialized_message_from_string as mfs) @@ -42,7 +43,8 @@ class TestModeration(unittest.TestCase): layer = RESTLayer def setUp(self): - self._mlist = create_list('ant@example.com') + with transaction(): + self._mlist = create_list('ant@example.com') self._msg = mfs("""\ From: anne@example.com To: ant@example.com @@ -51,7 +53,6 @@ Message-ID: <alpha> Something else. """) - config.db.commit() def test_not_found(self): # When a bogus mailing list is given, 404 should result. diff --git a/src/mailman/rest/tests/test_users.py b/src/mailman/rest/tests/test_users.py index 1630eb96a..301027885 100644 --- a/src/mailman/rest/tests/test_users.py +++ b/src/mailman/rest/tests/test_users.py @@ -21,6 +21,7 @@ from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ + 'TestUsers', ] @@ -29,7 +30,7 @@ import unittest from urllib2 import HTTPError from mailman.app.lifecycle import create_list -from mailman.config import config +from mailman.database.transaction import transaction from mailman.testing.helpers import call_api from mailman.testing.layers import RESTLayer @@ -39,8 +40,8 @@ class TestUsers(unittest.TestCase): layer = RESTLayer def setUp(self): - self._mlist = create_list('test@example.com') - config.db.commit() + with transaction(): + self._mlist = create_list('test@example.com') def test_delete_bogus_user(self): # Try to delete a user that does not exist. diff --git a/src/mailman/rest/users.py b/src/mailman/rest/users.py index 4e1362120..bce541ae5 100644 --- a/src/mailman/rest/users.py +++ b/src/mailman/rest/users.py @@ -26,7 +26,7 @@ __all__ = [ ] -from flufl.password import lookup, make_secret, generate +from passlib.utils import generate_password as generate from restish import http, resource from uuid import UUID from zope.component import getUtility @@ -102,8 +102,7 @@ class AllUsers(_UserBase): if password is None: # This will have to be reset since it cannot be retrieved. password = generate(int(config.passwords.password_length)) - scheme = lookup(config.passwords.password_scheme.upper()) - user.password = make_secret(password, scheme) + user.password = config.password_context.encrypt(password) location = path_to('users/{0}'.format(user.user_id.int)) return http.created(location, [], None) diff --git a/src/mailman/rest/wsgiapp.py b/src/mailman/rest/wsgiapp.py index 36e8ae5ac..a735d2012 100644 --- a/src/mailman/rest/wsgiapp.py +++ b/src/mailman/rest/wsgiapp.py @@ -33,6 +33,7 @@ from wsgiref.simple_server import WSGIRequestHandler from wsgiref.simple_server import make_server as wsgi_server from mailman.config import config +from mailman.database.transaction import transactional from mailman.rest.root import Root @@ -51,17 +52,11 @@ class AdminWebServiceWSGIRequestHandler(WSGIRequestHandler): class AdminWebServiceApplication(RestishApp): """Connect the restish WSGI application to Mailman's database.""" + @transactional def __call__(self, environ, start_response): """See `RestishApp`.""" - try: - response = super(AdminWebServiceApplication, self).__call__( - environ, start_response) - except: - config.db.abort() - raise - else: - config.db.commit() - return response + return super(AdminWebServiceApplication, self).__call__( + environ, start_response) diff --git a/src/mailman/rules/administrivia.py b/src/mailman/rules/administrivia.py index 41c6edf30..4c49e4ff2 100644 --- a/src/mailman/rules/administrivia.py +++ b/src/mailman/rules/administrivia.py @@ -17,7 +17,7 @@ """The administrivia rule.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -26,7 +26,7 @@ __all__ = [ from email.iterators import typed_subpart_iterator -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.core.i18n import _ @@ -53,9 +53,9 @@ EMAIL_COMMANDS = { +@implementer(IRule) class Administrivia: """The administrivia rule.""" - implements(IRule) name = 'administrivia' description = _('Catch mis-addressed email commands.') diff --git a/src/mailman/rules/any.py b/src/mailman/rules/any.py index b0d147bec..33c20394d 100644 --- a/src/mailman/rules/any.py +++ b/src/mailman/rules/any.py @@ -17,7 +17,7 @@ """Check if any previous rules have matched.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -25,16 +25,16 @@ __all__ = [ ] -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.interfaces.rules import IRule +@implementer(IRule) class Any: """Look for any previous rule match.""" - implements(IRule) name = 'any' description = _('Look for any previous rule hit.') diff --git a/src/mailman/rules/approved.py b/src/mailman/rules/approved.py index 3e2b7bc83..3ff7d21ec 100644 --- a/src/mailman/rules/approved.py +++ b/src/mailman/rules/approved.py @@ -28,9 +28,9 @@ __all__ = [ import re from email.iterators import typed_subpart_iterator -from flufl.password import verify -from zope.interface import implements +from zope.interface import implementer +from mailman.config import config from mailman.core.i18n import _ from mailman.interfaces.rules import IRule @@ -45,9 +45,9 @@ HEADERS = [ +@implementer(IRule) class Approved: """Look for moderator pre-approval.""" - implements(IRule) name = 'approved' description = _('The message has a matching Approve or Approved header.') @@ -119,8 +119,14 @@ class Approved: else: for header in HEADERS: del msg[header] - return (password is not missing and - verify(mlist.moderator_password, password)) + if password is missing: + return False + is_valid, new_hash = config.password_context.verify( + mlist.moderator_password, password) + if is_valid and new_hash: + # Hash algorithm migration. + mlist.moderator_password = new_hash + return is_valid diff --git a/src/mailman/rules/docs/approved.rst b/src/mailman/rules/docs/approved.rst index 9c61a7419..3f3d54455 100644 --- a/src/mailman/rules/docs/approved.rst +++ b/src/mailman/rules/docs/approved.rst @@ -20,9 +20,8 @@ which is shared among all the administrators. This password will not be stored in clear text, so it must be hashed using the configured hash protocol. - >>> from flufl.password import lookup, make_secret - >>> scheme = lookup(config.passwords.password_scheme.upper()) - >>> mlist.moderator_password = make_secret('super secret', scheme) + >>> mlist.moderator_password = config.password_context.encrypt( + ... 'super secret') The ``approved`` rule determines whether the message contains the proper approval or not. diff --git a/src/mailman/rules/docs/header-matching.rst b/src/mailman/rules/docs/header-matching.rst index 021974e69..20e55fadd 100644 --- a/src/mailman/rules/docs/header-matching.rst +++ b/src/mailman/rules/docs/header-matching.rst @@ -74,17 +74,19 @@ The header may exist and match the pattern. By default, when the header matches, it gets held for moderator approval. :: + >>> from mailman.interfaces.chain import ChainEvent >>> from mailman.testing.helpers import event_subscribers >>> def handler(event): - ... print event.__class__.__name__, \ - ... event.chain.name, event.msg['message-id'] + ... if isinstance(event, ChainEvent): + ... print event.__class__.__name__, \ + ... event.chain.name, event.msg['message-id'] >>> del msg['x-spam-score'] >>> msg['X-Spam-Score'] = '*****' >>> msgdata = {} >>> with event_subscribers(handler): ... process(mlist, msg, msgdata, 'header-match') - HoldNotification hold <ant> + HoldEvent hold <ant> >>> hits_and_misses(msgdata) Rule hits: @@ -100,7 +102,7 @@ discard such messages. >>> with event_subscribers(handler): ... with configuration('antispam', jump_chain='discard'): ... process(mlist, msg, msgdata, 'header-match') - DiscardNotification discard <ant> + DiscardEvent discard <ant> These programmatically added headers can be removed by flushing the chain. Now, nothing with match this message. diff --git a/src/mailman/rules/docs/news-moderation.rst b/src/mailman/rules/docs/news-moderation.rst index c695740fa..0400c8d9f 100644 --- a/src/mailman/rules/docs/news-moderation.rst +++ b/src/mailman/rules/docs/news-moderation.rst @@ -16,8 +16,8 @@ directly to the mailing list. Set the list configuration variable to enable newsgroup moderation. - >>> from mailman.interfaces.nntp import NewsModeration - >>> mlist.news_moderation = NewsModeration.moderated + >>> from mailman.interfaces.nntp import NewsgroupModeration + >>> mlist.newsgroup_moderation = NewsgroupModeration.moderated And now all messages will match the rule. @@ -32,6 +32,6 @@ And now all messages will match the rule. When moderation is turned off, the rule does not match. - >>> mlist.news_moderation = NewsModeration.none + >>> mlist.newsgroup_moderation = NewsgroupModeration.none >>> rule.check(mlist, msg, {}) False diff --git a/src/mailman/rules/emergency.py b/src/mailman/rules/emergency.py index 7b2100875..7c11b0aa3 100644 --- a/src/mailman/rules/emergency.py +++ b/src/mailman/rules/emergency.py @@ -17,7 +17,7 @@ """The emergency hold rule.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -25,16 +25,16 @@ __all__ = [ ] -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.interfaces.rules import IRule +@implementer(IRule) class Emergency: """The emergency hold rule.""" - implements(IRule) name = 'emergency' diff --git a/src/mailman/rules/implicit_dest.py b/src/mailman/rules/implicit_dest.py index 321a1775d..41d0403bc 100644 --- a/src/mailman/rules/implicit_dest.py +++ b/src/mailman/rules/implicit_dest.py @@ -17,7 +17,7 @@ """The implicit destination rule.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -27,7 +27,7 @@ __all__ = [ import re from email.utils import getaddresses -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.interfaces.mailinglist import IAcceptableAliasSet @@ -35,9 +35,9 @@ from mailman.interfaces.rules import IRule +@implementer(IRule) class ImplicitDestination: """The implicit destination rule.""" - implements(IRule) name = 'implicit-dest' description = _('Catch messages with implicit destination.') diff --git a/src/mailman/rules/loop.py b/src/mailman/rules/loop.py index 9e4aa0061..9282b1705 100644 --- a/src/mailman/rules/loop.py +++ b/src/mailman/rules/loop.py @@ -17,7 +17,7 @@ """Look for a posting loop.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -25,16 +25,16 @@ __all__ = [ ] -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.interfaces.rules import IRule +@implementer(IRule) class Loop: """Look for a posting loop.""" - implements(IRule) name = 'loop' description = _('Look for a posting loop.') diff --git a/src/mailman/rules/max_recipients.py b/src/mailman/rules/max_recipients.py index 79396c72f..fd09114e2 100644 --- a/src/mailman/rules/max_recipients.py +++ b/src/mailman/rules/max_recipients.py @@ -17,7 +17,7 @@ """The maximum number of recipients rule.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -26,16 +26,16 @@ __all__ = [ from email.utils import getaddresses -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.interfaces.rules import IRule +@implementer(IRule) class MaximumRecipients: """The maximum number of recipients rule.""" - implements(IRule) name = 'max-recipients' description = _('Catch messages with too many explicit recipients.') diff --git a/src/mailman/rules/max_size.py b/src/mailman/rules/max_size.py index 420d63571..a67d87771 100644 --- a/src/mailman/rules/max_size.py +++ b/src/mailman/rules/max_size.py @@ -17,7 +17,7 @@ """The maximum message size rule.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -25,16 +25,16 @@ __all__ = [ ] -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.interfaces.rules import IRule +@implementer(IRule) class MaximumSize: """The implicit destination rule.""" - implements(IRule) name = 'max-size' description = _('Catch messages that are bigger than a specified maximum.') diff --git a/src/mailman/rules/moderation.py b/src/mailman/rules/moderation.py index cb27d89d8..b2c777dc7 100644 --- a/src/mailman/rules/moderation.py +++ b/src/mailman/rules/moderation.py @@ -17,7 +17,7 @@ """Membership related rules.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -27,7 +27,7 @@ __all__ = [ from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.interfaces.action import Action @@ -37,9 +37,9 @@ from mailman.interfaces.usermanager import IUserManager +@implementer(IRule) class MemberModeration: """The member moderation rule.""" - implements(IRule) name = 'member-moderation' description = _('Match messages sent by moderated members.') @@ -65,9 +65,9 @@ class MemberModeration: +@implementer(IRule) class NonmemberModeration: """The nonmember moderation rule.""" - implements(IRule) name = 'nonmember-moderation' description = _('Match messages sent by nonmembers.') diff --git a/src/mailman/rules/news_moderation.py b/src/mailman/rules/news_moderation.py index 4ca9a0d8a..be0d56cb4 100644 --- a/src/mailman/rules/news_moderation.py +++ b/src/mailman/rules/news_moderation.py @@ -17,7 +17,7 @@ """The news moderation rule.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -25,17 +25,17 @@ __all__ = [ ] -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ -from mailman.interfaces.nntp import NewsModeration +from mailman.interfaces.nntp import NewsgroupModeration from mailman.interfaces.rules import IRule +@implementer(IRule) class ModeratedNewsgroup: """The news moderation rule.""" - implements(IRule) name = 'news-moderation' description = _( @@ -46,4 +46,4 @@ class ModeratedNewsgroup: def check(self, mlist, msg, msgdata): """See `IRule`.""" - return mlist.news_moderation == NewsModeration.moderated + return mlist.newsgroup_moderation == NewsgroupModeration.moderated diff --git a/src/mailman/rules/no_subject.py b/src/mailman/rules/no_subject.py index 1a2bab1d5..1fd0dfb8b 100644 --- a/src/mailman/rules/no_subject.py +++ b/src/mailman/rules/no_subject.py @@ -17,7 +17,7 @@ """The no-Subject header rule.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -25,16 +25,16 @@ __all__ = [ ] -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.interfaces.rules import IRule +@implementer(IRule) class NoSubject: """The no-Subject rule.""" - implements(IRule) name = 'no-subject' description = _('Catch messages with no, or empty, Subject headers.') diff --git a/src/mailman/rules/suspicious.py b/src/mailman/rules/suspicious.py index ad1ab42cd..75fe0afab 100644 --- a/src/mailman/rules/suspicious.py +++ b/src/mailman/rules/suspicious.py @@ -17,7 +17,7 @@ """The historical 'suspicious header' rule.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -28,7 +28,7 @@ __all__ = [ import re import logging -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.interfaces.rules import IRule @@ -37,9 +37,9 @@ log = logging.getLogger('mailman.error') +@implementer(IRule) class SuspiciousHeader: """The historical 'suspicious header' rule.""" - implements(IRule) name = 'suspicious-header' description = _('Catch messages with suspicious headers.') diff --git a/src/mailman/rules/tests/test_approved.py b/src/mailman/rules/tests/test_approved.py index d078556ba..a1b8f99ac 100644 --- a/src/mailman/rules/tests/test_approved.py +++ b/src/mailman/rules/tests/test_approved.py @@ -25,17 +25,18 @@ __all__ = [ 'TestApprovedNonASCII', 'TestApprovedPseudoHeader', 'TestApprovedPseudoHeaderMIME', + 'TestPasswordHashMigration', ] +import os import unittest -from flufl.password import lookup, make_secret - from mailman.app.lifecycle import create_list from mailman.config import config from mailman.rules import approved from mailman.testing.helpers import ( + configuration, specialized_message_from_string as mfs) from mailman.testing.layers import ConfigLayer @@ -48,8 +49,8 @@ class TestApproved(unittest.TestCase): def setUp(self): self._mlist = create_list('test@example.com') - scheme = lookup(config.passwords.password_scheme.upper()) - self._mlist.moderator_password = make_secret('super secret', scheme) + self._mlist.moderator_password = config.password_context.encrypt( + 'super secret') self._rule = approved.Approved() self._msg = mfs("""\ From: anne@example.com @@ -150,8 +151,8 @@ class TestApprovedPseudoHeader(unittest.TestCase): def setUp(self): self._mlist = create_list('test@example.com') - scheme = lookup(config.passwords.password_scheme.upper()) - self._mlist.moderator_password = make_secret('super secret', scheme) + self._mlist.moderator_password = config.password_context.encrypt( + 'super secret') self._rule = approved.Approved() self._msg = mfs("""\ From: anne@example.com @@ -283,8 +284,8 @@ class TestApprovedPseudoHeaderMIME(unittest.TestCase): def setUp(self): self._mlist = create_list('test@example.com') - scheme = lookup(config.passwords.password_scheme.upper()) - self._mlist.moderator_password = make_secret('super secret', scheme) + self._mlist.moderator_password = config.password_context.encrypt( + 'super secret') self._rule = approved.Approved() self._msg_text_template = """\ From: anne@example.com @@ -415,3 +416,78 @@ This is a message body with a non-ascii character =E4 # unicode errors. LP: #949924. result = self._rule.check(self._mlist, self._msg, {}) self.assertFalse(result) + + +class TestPasswordHashMigration(unittest.TestCase): + """Test that password hashing migrations work.""" + # http://packages.python.org/passlib/lib/passlib.context-tutorial.html#integrating-hash-migration + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + # The default testing hash algorithm is "roundup_plaintext" which + # yields hashed passwords of the form: {plaintext}abc + # + # Migration is automatically supported when a more modern password + # hash is chosen after the original password is set. As long as the + # old password still validates, the migration happens automatically. + self._mlist.moderator_password = config.password_context.encrypt( + b'super secret') + self._rule = approved.Approved() + self._msg = mfs("""\ +From: anne@example.com +To: test@example.com +Subject: A Message with non-ascii body +Message-ID: <ant> +MIME-Version: 1.0 + +A message body. +""") + + def test_valid_password_migrates(self): + # Now that the moderator password is set, change the default password + # hashing algorithm. When the old password is validated, it will be + # automatically migrated to the new hash. + self.assertEqual(self._mlist.moderator_password, + b'{plaintext}super secret') + config_file = os.path.join(config.VAR_DIR, 'passlib.config') + # XXX passlib seems to choose the default hashing scheme even if it is + # deprecated. The default scheme is either specified explicitly, or + # is the first in this list. This seems like a bug. + with open(config_file, 'w') as fp: + print("""\ +[passlib] +schemes = roundup_plaintext, plaintext +default = plaintext +deprecated = roundup_plaintext +""", file=fp) + with configuration('passwords', path=config_file): + self._msg['Approved'] = 'super secret' + result = self._rule.check(self._mlist, self._msg, {}) + self.assertTrue(result) + self.assertEqual(self._mlist.moderator_password, b'super secret') + + def test_invalid_password_does_not_migrate(self): + # Now that the moderator password is set, change the default password + # hashing algorithm. When the old password is invalid, it will not be + # automatically migrated to the new hash. + self.assertEqual(self._mlist.moderator_password, + b'{plaintext}super secret') + config_file = os.path.join(config.VAR_DIR, 'passlib.config') + # XXX passlib seems to choose the default hashing scheme even if it is + # deprecated. The default scheme is either specified explicitly, or + # is the first in this list. This seems like a bug. + with open(config_file, 'w') as fp: + print("""\ +[passlib] +schemes = roundup_plaintext, plaintext +default = plaintext +deprecated = roundup_plaintext +""", file=fp) + with configuration('passwords', path=config_file): + self._msg['Approved'] = 'not the password' + result = self._rule.check(self._mlist, self._msg, {}) + self.assertFalse(result) + self.assertEqual(self._mlist.moderator_password, + b'{plaintext}super secret') diff --git a/src/mailman/rules/truth.py b/src/mailman/rules/truth.py index 20c152e4d..752618ced 100644 --- a/src/mailman/rules/truth.py +++ b/src/mailman/rules/truth.py @@ -17,7 +17,7 @@ """A rule which always matches.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -25,16 +25,16 @@ __all__ = [ ] -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.interfaces.rules import IRule +@implementer(IRule) class Truth: """Look for any previous rule match.""" - implements(IRule) name = 'truth' description = _('A rule which always matches.') diff --git a/src/mailman/runners/command.py b/src/mailman/runners/command.py index ac611ed3a..6501474a3 100644 --- a/src/mailman/runners/command.py +++ b/src/mailman/runners/command.py @@ -38,7 +38,7 @@ from email.errors import HeaderParseError from email.header import decode_header, make_header from email.iterators import typed_subpart_iterator from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer from mailman.config import config from mailman.core.i18n import _ @@ -117,7 +117,7 @@ class CommandFinder: continue # Ensure that all the parts are unicodes. Since we only accept # ASCII commands and arguments, ignore anything else. - parts = [(part + parts = [(part if isinstance(part, unicode) else part.decode('ascii', 'ignore')) for part in parts] @@ -125,11 +125,10 @@ class CommandFinder: +@implementer(IEmailResults) class Results: """The email command results.""" - implements(IEmailResults) - def __init__(self, charset='us-ascii'): self._output = StringIO() self.charset = charset diff --git a/src/mailman/runners/digest.py b/src/mailman/runners/digest.py index c347667f8..99710dff5 100644 --- a/src/mailman/runners/digest.py +++ b/src/mailman/runners/digest.py @@ -201,7 +201,7 @@ class MIMEDigester(Digester): except URLError: log.exception( 'Digest footer decorator URI not found ({0}): {1}'.format( - self._mlist.fqdn_listname, + self._mlist.fqdn_listname, self._mlist.digest_footer_uri)) footer_text = '' footer = MIMEText(footer_text.encode(self._charset), @@ -281,9 +281,9 @@ class RFC1153Digester(Digester): except URLError: log.exception( 'Digest footer decorator URI not found ({0}): {1}'.format( - self._mlist.fqdn_listname, + self._mlist.fqdn_listname, self._mlist.digest_footer_uri)) - footer_text = '' + footer_text = '' # MAS: There is no real place for the digest_footer in an RFC 1153 # compliant digest, so add it as an additional message with # Subject: Digest Footer diff --git a/src/mailman/runners/docs/incoming.rst b/src/mailman/runners/docs/incoming.rst index 4a9db778e..8830031e6 100644 --- a/src/mailman/runners/docs/incoming.rst +++ b/src/mailman/runners/docs/incoming.rst @@ -143,9 +143,9 @@ chain will now hold all posted messages, so nothing will show up in the pipeline queue. :: - >>> from mailman.chains.base import ChainNotification + >>> from mailman.interfaces.chain import ChainEvent >>> def on_chain(event): - ... if isinstance(event, ChainNotification): + ... if isinstance(event, ChainEvent): ... print event ... print event.chain ... print 'From: {0}\nTo: {1}\nMessage-ID: {2}'.format( @@ -158,7 +158,7 @@ pipeline queue. >>> with event_subscribers(on_chain): ... inject_message(mlist, msg) ... incoming.run() - <mailman.chains.hold.HoldNotification ...> + <mailman.interfaces.chain.HoldEvent ...> <mailman.chains.hold.HoldChain ...> From: aperson@example.com To: test@example.com @@ -193,7 +193,7 @@ new chain and set it as the mailing list's start chain. >>> with event_subscribers(on_chain): ... inject_message(mlist, msg) ... incoming.run() - <mailman.chains.discard.DiscardNotification ...> + <mailman.interfaces.chain.DiscardEvent ...> <mailman.chains.discard.DiscardChain ...> From: aperson@example.com To: test@example.com @@ -222,7 +222,7 @@ just create a new chain that does. >>> with event_subscribers(on_chain): ... inject_message(mlist, msg) ... incoming.run() - <mailman.chains.reject.RejectNotification ...> + <mailman.interfaces.chain.RejectEvent ...> <mailman.chains.reject.RejectChain ...> From: aperson@example.com To: test@example.com diff --git a/src/mailman/runners/incoming.py b/src/mailman/runners/incoming.py index d8db926c7..1e4ceaa65 100644 --- a/src/mailman/runners/incoming.py +++ b/src/mailman/runners/incoming.py @@ -26,7 +26,7 @@ prepared for delivery. Rejections, discards, and holds are processed immediately. """ -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -36,9 +36,9 @@ __all__ = [ from zope.component import getUtility -from mailman.config import config from mailman.core.chains import process from mailman.core.runner import Runner +from mailman.database.transaction import transaction from mailman.interfaces.address import ExistingAddressError from mailman.interfaces.usermanager import IUserManager @@ -54,12 +54,12 @@ class IncomingRunner(Runner): # Ensure that the email addresses of the message's senders are known # to Mailman. This will be used in nonmember posting dispositions. user_manager = getUtility(IUserManager) - for sender in msg.senders: - try: - user_manager.create_address(sender) - except ExistingAddressError: - pass - config.db.commit() + with transaction(): + for sender in msg.senders: + try: + user_manager.create_address(sender) + except ExistingAddressError: + pass # Process the message through the mailing list's start chain. start_chain = (mlist.owner_chain if msgdata.get('to_owner', False) diff --git a/src/mailman/runners/lmtp.py b/src/mailman/runners/lmtp.py index 45fa5a783..61db6b848 100644 --- a/src/mailman/runners/lmtp.py +++ b/src/mailman/runners/lmtp.py @@ -15,6 +15,9 @@ # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see <http://www.gnu.org/licenses/>. +# XXX This module needs to be refactored to avoid direct access to the +# config.db global. + """Mailman LMTP runner (server). Most mail servers can be configured to deliver local messages via 'LMTP'[1]. @@ -31,6 +34,14 @@ so that the peer mail server can provide better diagnostics. http://www.faqs.org/rfcs/rfc2033.html """ +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'LMTPRunner', + ] + + import email import smtpd import logging @@ -41,7 +52,7 @@ from zope.component import getUtility from mailman.config import config from mailman.core.runner import Runner -from mailman.database.transaction import txn +from mailman.database.transaction import transactional from mailman.email.message import Message from mailman.interfaces.listmanager import IListManager from mailman.utilities.datetime import now @@ -80,15 +91,15 @@ SUBADDRESS_QUEUES = dict( ) DASH = '-' -CRLF = '\r\n' -ERR_451 = '451 Requested action aborted: error in processing' -ERR_501 = '501 Message has defects' -ERR_502 = '502 Error: command HELO not implemented' -ERR_550 = '550 Requested action not taken: mailbox unavailable' -ERR_550_MID = '550 No Message-ID header provided' +CRLF = b'\r\n' +ERR_451 = b'451 Requested action aborted: error in processing' +ERR_501 = b'501 Message has defects' +ERR_502 = b'502 Error: command HELO not implemented' +ERR_550 = b'550 Requested action not taken: mailbox unavailable' +ERR_550_MID = b'550 No Message-ID header provided' # XXX Blech -smtpd.__version__ = 'Python LMTP runner 1.0' +smtpd.__version__ = b'Python LMTP runner 1.0' @@ -154,7 +165,7 @@ class LMTPRunner(Runner, smtpd.SMTPServer): Channel(self, conn, addr) slog.debug('LMTP accept from %s', addr) - @txn + @transactional def process_message(self, peer, mailfrom, rcpttos, data): try: # Refresh the list of list names every time we process a message @@ -228,7 +239,7 @@ class LMTPRunner(Runner, smtpd.SMTPServer): config.switchboards[queue].enqueue(msg, msgdata) slog.debug('%s subaddress: %s, queue: %s', message_id, canonical_subaddress, queue) - status.append('250 Ok') + status.append(b'250 Ok') except Exception: slog.exception('Queue detection: %s', msg['message-id']) config.db.abort() diff --git a/src/mailman/runners/nntp.py b/src/mailman/runners/nntp.py index 8339c735e..4b6cd414f 100644 --- a/src/mailman/runners/nntp.py +++ b/src/mailman/runners/nntp.py @@ -35,7 +35,7 @@ from cStringIO import StringIO from mailman.config import config from mailman.core.runner import Runner -from mailman.interfaces.nntp import NewsModeration +from mailman.interfaces.nntp import NewsgroupModeration COMMA = ',' COMMASPACE = ', ' @@ -106,8 +106,8 @@ def prepare_message(mlist, msg, msgdata): # software to accept the posting, and not forward it on to the n.g.'s # moderation address. The posting would not have gotten here if it hadn't # already been approved. 1 == open list, mod n.g., 2 == moderated - if mlist.news_moderation in (NewsModeration.open_moderated, - NewsModeration.moderated): + if mlist.newsgroup_moderation in (NewsgroupModeration.open_moderated, + NewsgroupModeration.moderated): del msg['approved'] msg['Approved'] = mlist.posting_address # Should we restore the original, non-prefixed subject for gatewayed @@ -116,9 +116,7 @@ def prepare_message(mlist, msg, msgdata): # came from mailing list user. stripped_subject = msgdata.get('stripped_subject', msgdata.get('original_subject')) - # XXX 2012-03-31 BAW: rename news_prefix_subject_too to nntp_. This - # requires a schema change. - if not mlist.news_prefix_subject_too and stripped_subject is not None: + if not mlist.nntp_prefix_subject_too and stripped_subject is not None: del msg['subject'] msg['subject'] = stripped_subject # Add the appropriate Newsgroups header. Multiple Newsgroups headers are diff --git a/src/mailman/runners/tests/test_archiver.py b/src/mailman/runners/tests/test_archiver.py index 6f5804cae..30cb9e461 100644 --- a/src/mailman/runners/tests/test_archiver.py +++ b/src/mailman/runners/tests/test_archiver.py @@ -29,7 +29,7 @@ import os import unittest from email import message_from_file -from zope.interface import implements +from zope.interface import implementer from mailman.app.lifecycle import create_list from mailman.config import config @@ -44,8 +44,8 @@ from mailman.utilities.datetime import RFC822_DATE_FMT, factory, now +@implementer(IArchiver) class DummyArchiver: - implements(IArchiver) name = 'dummy' @staticmethod @@ -193,7 +193,7 @@ First post! self.assertEqual(archived['x-original-date'], 'Mon, 01 Aug 2005 07:49:23 +0000') - @configuration('archiver.dummy', + @configuration('archiver.dummy', enable='yes', clobber_date='maybe', clobber_skew='1d') def test_clobber_date_maybe_when_insane(self): # The date is clobbered if it's farther off from now than its skew diff --git a/src/mailman/runners/tests/test_bounce.py b/src/mailman/runners/tests/test_bounce.py index 76eb65c1f..5c21000bf 100644 --- a/src/mailman/runners/tests/test_bounce.py +++ b/src/mailman/runners/tests/test_bounce.py @@ -17,17 +17,20 @@ """Test the bounce runner.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ + 'TestBounceRunner', + 'TestBounceRunnerBug876774', + 'TestStyle', ] import unittest from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer from mailman.app.bounces import send_probe from mailman.app.lifecycle import create_list @@ -235,8 +238,9 @@ Message-Id: <third> # attributes. In particular, this will not set the bogus `bounce_processing` # attribute which the default style set (before LP: #876774 was fixed). +@implementer(IStyle) class TestStyle: - implements(IStyle) + """See `IStyle`.""" name = 'test' priority = 10 @@ -249,6 +253,7 @@ class TestStyle: styles.append(self) + class TestBounceRunnerBug876774(unittest.TestCase): """Test LP: #876774. diff --git a/src/mailman/runners/tests/test_confirm.py b/src/mailman/runners/tests/test_confirm.py index d2b24a2d1..62171979c 100644 --- a/src/mailman/runners/tests/test_confirm.py +++ b/src/mailman/runners/tests/test_confirm.py @@ -32,6 +32,7 @@ from zope.component import getUtility from mailman.app.lifecycle import create_list from mailman.config import config +from mailman.database.transaction import transaction from mailman.interfaces.registrar import IRegistrar from mailman.interfaces.usermanager import IUserManager from mailman.runners.command import CommandRunner @@ -50,14 +51,14 @@ class TestConfirm(unittest.TestCase): layer = ConfigLayer def setUp(self): - # Register a subscription requiring confirmation. registrar = getUtility(IRegistrar) - self._mlist = create_list('test@example.com') - self._mlist.send_welcome_message = False - self._token = registrar.register(self._mlist, 'anne@example.org') self._commandq = config.switchboards['command'] self._runner = make_testable_runner(CommandRunner, 'command') - config.db.commit() + with transaction(): + # Register a subscription requiring confirmation. + self._mlist = create_list('test@example.com') + self._mlist.send_welcome_message = False + self._token = registrar.register(self._mlist, 'anne@example.org') def test_confirm_with_re_prefix(self): subject = 'Re: confirm {0}'.format(self._token) diff --git a/src/mailman/runners/tests/test_lmtp.py b/src/mailman/runners/tests/test_lmtp.py index 87b69c7e4..46d4ed986 100644 --- a/src/mailman/runners/tests/test_lmtp.py +++ b/src/mailman/runners/tests/test_lmtp.py @@ -31,7 +31,7 @@ import unittest from datetime import datetime from mailman.app.lifecycle import create_list -from mailman.config import config +from mailman.database.transaction import transaction from mailman.testing.helpers import get_lmtp_client, get_queue_messages from mailman.testing.layers import LMTPLayer @@ -43,8 +43,8 @@ class TestLMTP(unittest.TestCase): layer = LMTPLayer def setUp(self): - self._mlist = create_list('test@example.com') - config.db.commit() + with transaction(): + self._mlist = create_list('test@example.com') self._lmtp = get_lmtp_client(quiet=True) self._lmtp.lhlo('remote.example.org') diff --git a/src/mailman/runners/tests/test_nntp.py b/src/mailman/runners/tests/test_nntp.py index 426e829d8..477bccfa3 100644 --- a/src/mailman/runners/tests/test_nntp.py +++ b/src/mailman/runners/tests/test_nntp.py @@ -33,7 +33,7 @@ import unittest from mailman.app.lifecycle import create_list from mailman.config import config -from mailman.interfaces.nntp import NewsModeration +from mailman.interfaces.nntp import NewsgroupModeration from mailman.runners import nntp from mailman.testing.helpers import ( LogFileMark, @@ -67,7 +67,7 @@ Testing # Approved header, which NNTP software uses to forward to the # newsgroup. The message would not have gotten to the mailing list if # it wasn't already approved. - self._mlist.news_moderation = NewsModeration.moderated + self._mlist.newsgroup_moderation = NewsgroupModeration.moderated nntp.prepare_message(self._mlist, self._msg, {}) self.assertEqual(self._msg['approved'], 'test@example.com') @@ -76,14 +76,14 @@ Testing # message will get an Approved header, which NNTP software uses to # forward to the newsgroup. The message would not have gotten to the # mailing list if it wasn't already approved. - self._mlist.news_moderation = NewsModeration.open_moderated + self._mlist.newsgroup_moderation = NewsgroupModeration.open_moderated nntp.prepare_message(self._mlist, self._msg, {}) self.assertEqual(self._msg['approved'], 'test@example.com') def test_moderation_removes_previous_approved_header(self): # Any existing Approved header is removed from moderated messages. self._msg['Approved'] = 'a bogus approval' - self._mlist.news_moderation = NewsModeration.moderated + self._mlist.newsgroup_moderation = NewsgroupModeration.moderated nntp.prepare_message(self._mlist, self._msg, {}) headers = self._msg.get_all('approved') self.assertEqual(len(headers), 1) @@ -92,7 +92,7 @@ Testing def test_open_moderation_removes_previous_approved_header(self): # Any existing Approved header is removed from moderated messages. self._msg['Approved'] = 'a bogus approval' - self._mlist.news_moderation = NewsModeration.open_moderated + self._mlist.newsgroup_moderation = NewsgroupModeration.open_moderated nntp.prepare_message(self._mlist, self._msg, {}) headers = self._msg.get_all('approved') self.assertEqual(len(headers), 1) @@ -102,7 +102,7 @@ Testing # The cook-headers handler adds the original and/or stripped (of the # prefix) subject to the metadata. Assume that handler's been run; # check the Subject header. - self._mlist.news_prefix_subject_too = False + self._mlist.nntp_prefix_subject_too = False del self._msg['subject'] self._msg['subject'] = 'Re: Your test' msgdata = dict(stripped_subject='Your test') @@ -115,7 +115,7 @@ Testing # The cook-headers handler adds the original and/or stripped (of the # prefix) subject to the metadata. Assume that handler's been run; # check the Subject header. - self._mlist.news_prefix_subject_too = False + self._mlist.nntp_prefix_subject_too = False del self._msg['subject'] self._msg['subject'] = 'Re: Your test' msgdata = dict(original_subject='Your test') @@ -128,7 +128,7 @@ Testing # The cook-headers handler adds the original and/or stripped (of the # prefix) subject to the metadata. Assume that handler's been run; # check the Subject header. - self._mlist.news_prefix_subject_too = True + self._mlist.nntp_prefix_subject_too = True del self._msg['subject'] self._msg['subject'] = 'Re: Your test' msgdata = dict(stripped_subject='Your test') @@ -141,7 +141,7 @@ Testing # The cook-headers handler adds the original and/or stripped (of the # prefix) subject to the metadata. Assume that handler's been run; # check the Subject header. - self._mlist.news_prefix_subject_too = True + self._mlist.nntp_prefix_subject_too = True del self._msg['subject'] self._msg['subject'] = 'Re: Your test' msgdata = dict(original_subject='Your test') diff --git a/src/mailman/runners/tests/test_owner.py b/src/mailman/runners/tests/test_owner.py index 622bb2255..4ea5771be 100644 --- a/src/mailman/runners/tests/test_owner.py +++ b/src/mailman/runners/tests/test_owner.py @@ -37,6 +37,7 @@ from zope.component import getUtility from mailman.app.lifecycle import create_list from mailman.config import config +from mailman.database.transaction import transaction from mailman.interfaces.member import MemberRole from mailman.interfaces.usermanager import IUserManager from mailman.testing.helpers import ( @@ -59,17 +60,17 @@ class TestEmailToOwner(unittest.TestCase): self._mlist = create_list('test@example.com') # Add some owners, moderators, and members manager = getUtility(IUserManager) - anne = manager.create_address('anne@example.com') - bart = manager.create_address('bart@example.com') - cris = manager.create_address('cris@example.com') - dave = manager.create_address('dave@example.com') - self._mlist.subscribe(anne, MemberRole.member) - self._mlist.subscribe(anne, MemberRole.owner) - self._mlist.subscribe(bart, MemberRole.moderator) - self._mlist.subscribe(bart, MemberRole.owner) - self._mlist.subscribe(cris, MemberRole.moderator) - self._mlist.subscribe(dave, MemberRole.member) - config.db.commit() + with transaction(): + anne = manager.create_address('anne@example.com') + bart = manager.create_address('bart@example.com') + cris = manager.create_address('cris@example.com') + dave = manager.create_address('dave@example.com') + self._mlist.subscribe(anne, MemberRole.member) + self._mlist.subscribe(anne, MemberRole.owner) + self._mlist.subscribe(bart, MemberRole.moderator) + self._mlist.subscribe(bart, MemberRole.owner) + self._mlist.subscribe(cris, MemberRole.moderator) + self._mlist.subscribe(dave, MemberRole.member) self._inq = make_testable_runner(IncomingRunner, 'in') self._pipelineq = make_testable_runner(PipelineRunner, 'pipeline') self._outq = make_testable_runner(OutgoingRunner, 'out') diff --git a/src/mailman/runners/tests/test_pipeline.py b/src/mailman/runners/tests/test_pipeline.py index 8776bf844..3f8f62ab0 100644 --- a/src/mailman/runners/tests/test_pipeline.py +++ b/src/mailman/runners/tests/test_pipeline.py @@ -27,7 +27,7 @@ __all__ = [ import unittest -from zope.interface import implements +from zope.interface import implementer from mailman.app.lifecycle import create_list from mailman.config import config @@ -41,8 +41,10 @@ from mailman.testing.layers import ConfigLayer +@implementer(IHandler) class MyTestHandler: - implements(IHandler) + """See `IHandler`.""" + name = 'test handler' description = 'A test handler' @@ -54,8 +56,9 @@ class MyTestHandler: self._test.mark(self._marker) + +@implementer(IPipeline) class MyTestPipeline: - implements(IPipeline) name = 'test' description = 'a test pipeline' diff --git a/src/mailman/styles/default.py b/src/mailman/styles/default.py index b6900dca6..cb4da396d 100644 --- a/src/mailman/styles/default.py +++ b/src/mailman/styles/default.py @@ -17,7 +17,7 @@ """Application of list styles to new and existing lists.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -28,24 +28,24 @@ __all__ = [ # XXX Styles need to be reconciled with lazr.config. from datetime import timedelta -from zope.interface import implements +from zope.interface import implementer from mailman.core.i18n import _ from mailman.interfaces.action import Action, FilterAction +from mailman.interfaces.archiver import ArchivePolicy from mailman.interfaces.bounce import UnrecognizedBounceDisposition from mailman.interfaces.digests import DigestFrequency from mailman.interfaces.autorespond import ResponseAction from mailman.interfaces.mailinglist import Personalization, ReplyToMunging -from mailman.interfaces.nntp import NewsModeration +from mailman.interfaces.nntp import NewsgroupModeration from mailman.interfaces.styles import IStyle +@implementer(IStyle) class DefaultStyle: """The default (i.e. legacy) style.""" - implements(IStyle) - name = 'default' priority = 0 # the lowest priority style @@ -55,9 +55,8 @@ class DefaultStyle: mlist = mailing_list # List identity. mlist.display_name = mlist.list_name.capitalize() - mlist.list_id = '{0.list_name}.{0.mail_host}'.format(mlist) mlist.include_rfc2369_headers = True - mlist.include_list_post_header = True + mlist.allow_list_posts = True # Most of these were ripped from the old MailList.InitVars() method. mlist.volume = 1 mlist.post_id = 1 @@ -117,9 +116,7 @@ from: .*@uplinkpro.com mlist.default_member_action = Action.defer mlist.default_nonmember_action = Action.hold # Archiver - mlist.archive = True - mlist.archive_private = 0 - mlist.archive_volume_frequency = 1 + mlist.archive_policy = ArchivePolicy.public mlist.emergency = False mlist.member_moderation_notice = '' mlist.accept_these_nonmembers = [] @@ -127,7 +124,6 @@ from: .*@uplinkpro.com mlist.reject_these_nonmembers = [] mlist.discard_these_nonmembers = [] mlist.forward_auto_discards = True - mlist.generic_nonmember_action = 1 mlist.nonmember_rejection_notice = '' # Max autoresponses per day. A mapping between addresses and a # 2-tuple of the date of the last autoresponse and the number of @@ -175,10 +171,10 @@ from: .*@uplinkpro.com mlist.linked_newsgroup = '' mlist.gateway_to_news = False mlist.gateway_to_mail = False - mlist.news_prefix_subject_too = True + mlist.nntp_prefix_subject_too = True # In patch #401270, this was called newsgroup_is_moderated, but the # semantics weren't quite the same. - mlist.news_moderation = NewsModeration.none + mlist.newsgroup_moderation = NewsgroupModeration.none # Topics # # `topics' is a list of 4-tuples of the following form: diff --git a/src/mailman/styles/docs/styles.rst b/src/mailman/styles/docs/styles.rst index 90a02227b..8f589f10b 100644 --- a/src/mailman/styles/docs/styles.rst +++ b/src/mailman/styles/docs/styles.rst @@ -62,10 +62,10 @@ Registering styles New styles must implement the ``IStyle`` interface. - >>> from zope.interface import implements + >>> from zope.interface import implementer >>> from mailman.interfaces.styles import IStyle - >>> class TestStyle: - ... implements(IStyle) + >>> @implementer(IStyle) + ... class TestStyle: ... name = 'test' ... priority = 10 ... def apply(self, mailing_list): diff --git a/src/mailman/styles/manager.py b/src/mailman/styles/manager.py index f1d3f1dfb..c2729abfb 100644 --- a/src/mailman/styles/manager.py +++ b/src/mailman/styles/manager.py @@ -17,29 +17,31 @@ """Style manager.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ 'StyleManager', + 'handle_ConfigurationUpdatedEvent', ] from operator import attrgetter -from zope.interface import implements +from zope.component import getUtility +from zope.interface import implementer from zope.interface.verify import verifyObject +from mailman.interfaces.configuration import ConfigurationUpdatedEvent from mailman.interfaces.styles import ( DuplicateStyleError, IStyle, IStyleManager) from mailman.utilities.modules import call_name +@implementer(IStyleManager) class StyleManager: """The built-in style manager.""" - implements(IStyleManager) - def __init__(self): """Install all styles from the configuration files.""" self._styles = {} @@ -89,3 +91,9 @@ class StyleManager: """See `IStyleManager`.""" # Let KeyErrors percolate up. del self._styles[style.name] + + + +def handle_ConfigurationUpdatedEvent(event): + if isinstance(event, ConfigurationUpdatedEvent): + getUtility(IStyleManager).populate() diff --git a/src/mailman/styles/tests/test_styles.py b/src/mailman/styles/tests/test_styles.py index ce8b5064d..990ce541f 100644 --- a/src/mailman/styles/tests/test_styles.py +++ b/src/mailman/styles/tests/test_styles.py @@ -28,7 +28,7 @@ __all__ = [ import unittest from zope.component import getUtility -from zope.interface import implements +from zope.interface import implementer from zope.interface.exceptions import DoesNotImplement from mailman.interfaces.styles import ( @@ -37,8 +37,8 @@ from mailman.testing.layers import ConfigLayer +@implementer(IStyle) class DummyStyle: - implements(IStyle) name = 'dummy' priority = 1 diff --git a/src/mailman/testing/helpers.py b/src/mailman/testing/helpers.py index 3648a6710..054dd4ff7 100644 --- a/src/mailman/testing/helpers.py +++ b/src/mailman/testing/helpers.py @@ -17,7 +17,7 @@ """Various test helpers.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -25,6 +25,7 @@ __all__ = [ 'TestableMaster', 'body_line_iterator', 'call_api', + 'chdir', 'configuration', 'digest_mbox', 'event_subscribers', @@ -35,6 +36,7 @@ __all__ = [ 'reset_the_world', 'specialized_message_from_string', 'subscribe', + 'temporary_db', 'wait_for_webservice', ] @@ -64,6 +66,7 @@ from zope.component import getUtility from mailman.bin.master import Loop as Master from mailman.config import config +from mailman.database.transaction import transaction from mailman.email.message import Message from mailman.interfaces.member import MemberRole from mailman.interfaces.messages import IMessageStore @@ -237,13 +240,14 @@ def get_lmtp_client(quiet=False): # It's possible the process has started but is not yet accepting # connections. Wait a little while. lmtp = LMTP() + #lmtp.debuglevel = 1 until = datetime.datetime.now() + as_timedelta(config.devmode.wait) while datetime.datetime.now() < until: try: response = lmtp.connect( config.mta.lmtp_host, int(config.mta.lmtp_port)) if not quiet: - print response + print(response) return lmtp except socket.error as error: if error[0] == errno.ECONNREFUSED: @@ -341,14 +345,14 @@ def call_api(url, data=None, method=None, username=None, password=None): @contextmanager def event_subscribers(*subscribers): - """Temporarily set the Zope event subscribers list. + """Temporarily extend the Zope event subscribers list. :param subscribers: A sequence of event subscribers. :type subscribers: sequence of callables, each receiving one argument, the event. """ old_subscribers = event.subscribers[:] - event.subscribers = list(subscribers) + event.subscribers.extend(subscribers) try: yield finally: @@ -361,8 +365,14 @@ class configuration: def __init__(self, section, **kws): self._section = section + # Most tests don't care about the name given to the temporary + # configuration. Usually we'll just craft a random one, but some + # tests do care, so give them a hook to set it. + if '_configname' in kws: + self._uuid = kws.pop('_configname') + else: + self._uuid = uuid.uuid4().hex self._values = kws.copy() - self._uuid = uuid.uuid4().hex def _apply(self): lines = ['[{0}]'.format(self._section)] @@ -392,24 +402,52 @@ class configuration: +@contextmanager +def temporary_db(db): + real_db = config.db + config.db = db + try: + yield + finally: + config.db = real_db + + + +class chdir: + """A context manager for temporary directory changing.""" + def __init__(self, directory): + self._curdir = None + self._directory = directory + + def __enter__(self): + self._curdir = os.getcwd() + os.chdir(self._directory) + + def __exit__(self, exc_type, exc_val, exc_tb): + os.chdir(self._curdir) + # Don't suppress exceptions. + return False + + + def subscribe(mlist, first_name, role=MemberRole.member): """Helper for subscribing a sample person to a mailing list.""" user_manager = getUtility(IUserManager) email = '{0}person@example.com'.format(first_name[0].lower()) full_name = '{0} Person'.format(first_name) - person = user_manager.get_user(email) - if person is None: - address = user_manager.get_address(email) - if address is None: - person = user_manager.create_user(email, full_name) + with transaction(): + person = user_manager.get_user(email) + if person is None: + address = user_manager.get_address(email) + if address is None: + person = user_manager.create_user(email, full_name) + preferred_address = list(person.addresses)[0] + mlist.subscribe(preferred_address, role) + else: + mlist.subscribe(address, role) + else: preferred_address = list(person.addresses)[0] mlist.subscribe(preferred_address, role) - else: - mlist.subscribe(address, role) - else: - preferred_address = list(person.addresses)[0] - mlist.subscribe(preferred_address, role) - config.db.commit() @@ -437,9 +475,9 @@ def reset_the_world(): os.remove(os.path.join(dirpath, filename)) # Clear out messages in the message store. message_store = getUtility(IMessageStore) - for message in message_store.messages: - message_store.delete_message(message['message-id']) - config.db.commit() + with transaction(): + for message in message_store.messages: + message_store.delete_message(message['message-id']) # Reset the global style manager. getUtility(IStyleManager).populate() # Remove all dynamic header-match rules. diff --git a/src/mailman/testing/layers.py b/src/mailman/testing/layers.py index 41ef86935..3a3e1f684 100644 --- a/src/mailman/testing/layers.py +++ b/src/mailman/testing/layers.py @@ -25,7 +25,7 @@ # eventually get rid of the zope.test* dependencies and use something like # testresources or some such. -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -56,6 +56,7 @@ from mailman.config import config from mailman.core import initialize from mailman.core.initialize import INHIBIT_CONFIG_FILE from mailman.core.logging import get_handler +from mailman.database.transaction import transaction from mailman.interfaces.domain import IDomainManager from mailman.testing.helpers import ( TestableMaster, get_lmtp_client, reset_the_world) @@ -116,8 +117,6 @@ class ConfigLayer(MockAndMonkeyLayer): test_config = dedent(""" [mailman] layout: testing - [passwords] - password_scheme: cleartext [paths.testing] var_dir: %s [devmode] @@ -128,7 +127,7 @@ class ConfigLayer(MockAndMonkeyLayer): config.create_paths = True config.push('test config', test_config) # Initialize everything else. - initialize.initialize_2() + initialize.initialize_2(testing=True) initialize.initialize_3() # When stderr debugging is enabled, subprocess root loggers should # also be more verbose. @@ -176,7 +175,7 @@ class ConfigLayer(MockAndMonkeyLayer): config_file = os.path.join(cls.var_dir, 'test.cfg') with open(config_file, 'w') as fp: fp.write(test_config) - print >> fp + print(file=fp) config.filename = config_file @classmethod @@ -189,10 +188,10 @@ class ConfigLayer(MockAndMonkeyLayer): @classmethod def testSetUp(cls): # Add an example domain. - getUtility(IDomainManager).add( - 'example.com', 'An example domain.', - 'http://lists.example.com', 'postmaster@example.com') - config.db.commit() + with transaction(): + getUtility(IDomainManager).add( + 'example.com', 'An example domain.', + 'http://lists.example.com', 'postmaster@example.com') @classmethod def testTearDown(cls): diff --git a/src/mailman/testing/mta.py b/src/mailman/testing/mta.py index 4699cb882..bba450352 100644 --- a/src/mailman/testing/mta.py +++ b/src/mailman/testing/mta.py @@ -17,7 +17,7 @@ """Fake MTA for testing purposes.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -31,7 +31,7 @@ from Queue import Empty, Queue from lazr.smtptest.controller import QueueController from lazr.smtptest.server import Channel, QueueServer -from zope.interface import implements +from zope.interface import implementer from mailman.interfaces.mta import IMailTransportAgentLifecycle @@ -40,11 +40,10 @@ log = logging.getLogger('lazr.smtptest') +@implementer(IMailTransportAgentLifecycle) class FakeMTA: """Fake MTA for testing purposes.""" - implements(IMailTransportAgentLifecycle) - def create(self, mlist): pass diff --git a/src/mailman/testing/passlib.cfg b/src/mailman/testing/passlib.cfg new file mode 100644 index 000000000..225ecd49b --- /dev/null +++ b/src/mailman/testing/passlib.cfg @@ -0,0 +1,4 @@ +[passlib] +# Use a predictable hashing algorithm with plain text and no salt. This is +# *only* useful for debugging and unit testing. +schemes = roundup_plaintext diff --git a/src/mailman/testing/testing.cfg b/src/mailman/testing/testing.cfg index 91613cc8d..141d74a8f 100644 --- a/src/mailman/testing/testing.cfg +++ b/src/mailman/testing/testing.cfg @@ -18,9 +18,9 @@ # A testing configuration. # For testing against PostgreSQL. -#[database] -#class: mailman.database.postgresql.PostgreSQLDatabase -#url: postgres://barry:barry@localhost/mailman +# [database] +# class: mailman.database.postgresql.PostgreSQLDatabase +# url: postgres://barry:barry@localhost/mailman [mailman] site_owner: noreply@example.com @@ -30,6 +30,9 @@ smtp_port: 9025 lmtp_port: 9024 incoming: mailman.testing.mta.FakeMTA +[passwords] +path: python:mailman.testing.passlib + [webservice] port: 9001 diff --git a/src/mailman/tests/test_documentation.py b/src/mailman/tests/test_documentation.py index a2c1ab592..b769f07d6 100644 --- a/src/mailman/tests/test_documentation.py +++ b/src/mailman/tests/test_documentation.py @@ -34,11 +34,14 @@ import sys import doctest import unittest +from inspect import isfunction, ismethod + import mailman from mailman.app.lifecycle import create_list from mailman.config import config -from mailman.testing.helpers import call_api, specialized_message_from_string +from mailman.testing.helpers import ( + call_api, chdir, specialized_message_from_string) from mailman.testing.layers import SMTPLayer @@ -46,23 +49,6 @@ DOT = '.' -class chdir: - """A context manager for temporary directory changing.""" - def __init__(self, directory): - self._curdir = None - self._directory = directory - - def __enter__(self): - self._curdir = os.getcwd() - os.chdir(self._directory) - - def __exit__(self, exc_type, exc_val, exc_tb): - os.chdir(self._curdir) - # Don't suppress exceptions. - return False - - - def stop(): """Call into pdb.set_trace()""" # Do the import here so that you get the wacky special hacked pdb instead @@ -185,7 +171,10 @@ def setup(testobj): def teardown(testobj): for cleanup in testobj.globs['cleanups']: - cleanup() + if isfunction(cleanup) or ismethod(cleanup): + cleanup() + else: + cleanup[0](*cleanup[1:]) diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py index f77d86e9a..6cdba0de3 100644 --- a/src/mailman/utilities/importer.py +++ b/src/mailman/utilities/importer.py @@ -17,7 +17,7 @@ """Importer routines.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -31,7 +31,7 @@ import datetime from mailman.interfaces.autorespond import ResponseAction from mailman.interfaces.digests import DigestFrequency from mailman.interfaces.mailinglist import Personalization, ReplyToMunging -from mailman.interfaces.nntp import NewsModeration +from mailman.interfaces.nntp import NewsgroupModeration @@ -47,7 +47,7 @@ TYPES = dict( bounce_info_stale_after=seconds_to_delta, bounce_you_are_disabled_warnings_interval=seconds_to_delta, digest_volume_frequency=DigestFrequency, - news_moderation=NewsModeration, + newsgroup_moderation=NewsgroupModeration, personalize=Personalization, reply_goes_to_list=ReplyToMunging, ) @@ -56,6 +56,7 @@ TYPES = dict( # Attribute names in Mailman 2 which are renamed in Mailman 3. NAME_MAPPINGS = dict( host_name='mail_host', + include_list_post_header='allow_list_posts', real_name='display_name', ) @@ -85,5 +86,5 @@ def import_config_pck(mlist, config_dict): try: setattr(mlist, key, value) except TypeError: - print >> sys.stderr, 'Type conversion error:', key + print('Type conversion error:', key, file=sys.stderr) raise diff --git a/src/mailman/utilities/passwords.py b/src/mailman/utilities/passwords.py new file mode 100644 index 000000000..95c85c47a --- /dev/null +++ b/src/mailman/utilities/passwords.py @@ -0,0 +1,62 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""A wrapper around passlib.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'handle_ConfigurationUpdatedEvent', + ] + + + +from passlib.context import CryptContext +from pkg_resources import resource_string + +from mailman.interfaces.configuration import ConfigurationUpdatedEvent + + + +class PasswordContext: + def __init__(self, config): + # Is the context coming from a file system or Python path? + if config.passwords.path.startswith('python:'): + resource_path = config.passwords.path[7:] + package, dot, resource = resource_path.rpartition('.') + config_string = resource_string(package, resource + '.cfg') + else: + with open(config.passwords.path, 'rb') as fp: + config_string = fp.read() + self._context = CryptContext.from_string(config_string) + + def encrypt(self, secret): + return self._context.encrypt(secret) + + def verify(self, hashed, password): + # Support hash algorithm migration. Yes, the order of arguments is + # reversed, for backward compatibility with flufl.password. XXX fix + # this eventually. + return self._context.verify_and_update(password, hashed) + + + +def handle_ConfigurationUpdatedEvent(event): + if isinstance(event, ConfigurationUpdatedEvent): + # Just reset the password context. + event.config.password_context = PasswordContext(event.config) diff --git a/src/mailman/utilities/tests/test_import.py b/src/mailman/utilities/tests/test_import.py index 58a51e61b..396da4aa8 100644 --- a/src/mailman/utilities/tests/test_import.py +++ b/src/mailman/utilities/tests/test_import.py @@ -17,10 +17,11 @@ """Tests for config.pck imports.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ + 'TestBasicImport', ] @@ -44,7 +45,7 @@ class TestBasicImport(unittest.TestCase): self._pckdict = cPickle.load(fp) def tearDown(self): - remove_list(self._mlist.fqdn_listname, self._mlist) + remove_list(self._mlist) def _import(self): import_config_pck(self._mlist, self._pckdict) @@ -62,8 +63,8 @@ class TestBasicImport(unittest.TestCase): self.assertEqual(self._mlist.mail_host, 'heresy.example.org') def test_rfc2369_headers(self): - self._mlist.include_list_post_header = False + self._mlist.allow_list_posts = False self._mlist.include_rfc2369_headers = False self._import() - self.assertTrue(self._mlist.include_list_post_header) + self.assertTrue(self._mlist.allow_list_posts) self.assertTrue(self._mlist.include_rfc2369_headers) diff --git a/src/mailman/utilities/tests/test_passwords.py b/src/mailman/utilities/tests/test_passwords.py new file mode 100644 index 000000000..7b2931855 --- /dev/null +++ b/src/mailman/utilities/tests/test_passwords.py @@ -0,0 +1,60 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Testing the password utility.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestPasswords', + ] + + +import os +import unittest + +from mailman.config import config +from mailman.testing.helpers import configuration +from mailman.testing.layers import ConfigLayer + + + +class TestPasswords(unittest.TestCase): + layer = ConfigLayer + + def test_default_passlib(self): + # By default, testing uses the roundup_plaintext hash algorithm, which + # is just plaintext with a prefix. + self.assertEqual(config.password_context.encrypt('my password'), + '{plaintext}my password') + + def test_passlib_from_file_path(self): + # Set up this test to use a passlib configuration file specified with + # a file system path. We prove we're using the new configuration + # because a non-prefixed, i.e. non-roundup, plaintext hash algorithm + # will be used. When a file system path is used, the file can end in + # any suffix. + config_file = os.path.join(config.VAR_DIR, 'passlib.config') + with open(config_file, 'w') as fp: + print("""\ +[passlib] +schemes = plaintext +""", file=fp) + with configuration('passwords', path=config_file): + self.assertEqual(config.password_context.encrypt('my password'), + 'my password') |