diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/mailman/config/configure.zcml | 23 | ||||
| -rw-r--r-- | src/mailman/database/mailman.sql | 11 | ||||
| -rw-r--r-- | src/mailman/database/types.py | 5 | ||||
| -rw-r--r-- | src/mailman/interfaces/bounce.py | 63 | ||||
| -rw-r--r-- | src/mailman/model/bounce.py | 66 | ||||
| -rw-r--r-- | src/mailman/model/docs/bounce.rst | 84 | ||||
| -rw-r--r-- | src/mailman/model/mailinglist.py | 2 | ||||
| -rw-r--r-- | src/mailman/model/tests/test_bounce.py | 46 | ||||
| -rw-r--r-- | src/mailman/queue/outgoing.py | 11 | ||||
| -rw-r--r-- | src/mailman/queue/tests/__init__.py | 0 | ||||
| -rw-r--r-- | src/mailman/queue/tests/test_outgoing.py | 245 | ||||
| -rw-r--r-- | src/mailman/testing/helpers.py | 11 |
12 files changed, 548 insertions, 19 deletions
diff --git a/src/mailman/config/configure.zcml b/src/mailman/config/configure.zcml index 3b4497ab8..299a0ce67 100644 --- a/src/mailman/config/configure.zcml +++ b/src/mailman/config/configure.zcml @@ -22,6 +22,11 @@ /> <utility + factory="mailman.model.bounce.BounceProcessor" + provides="mailman.interfaces.bounce.IBounceProcessor" + /> + + <utility factory="mailman.model.domain.DomainManager" provides="mailman.interfaces.domain.IDomainManager" /> @@ -37,11 +42,6 @@ /> <utility - factory="mailman.model.usermanager.UserManager" - provides="mailman.interfaces.usermanager.IUserManager" - /> - - <utility factory="mailman.model.messagestore.MessageStore" provides="mailman.interfaces.messages.IMessageStore" /> @@ -52,13 +52,13 @@ /> <utility - factory="mailman.model.requests.Requests" - provides="mailman.interfaces.requests.IRequests" + factory="mailman.app.registrar.Registrar" + provides="mailman.interfaces.registrar.IRegistrar" /> <utility - factory="mailman.app.registrar.Registrar" - provides="mailman.interfaces.registrar.IRegistrar" + factory="mailman.model.requests.Requests" + provides="mailman.interfaces.requests.IRequests" /> <utility @@ -67,6 +67,11 @@ /> <utility + factory="mailman.model.usermanager.UserManager" + provides="mailman.interfaces.usermanager.IUserManager" + /> + + <utility factory="mailman.email.validate.Validator" provides="mailman.interfaces.address.IEmailValidator" /> diff --git a/src/mailman/database/mailman.sql b/src/mailman/database/mailman.sql index 5d07c607a..6312066c0 100644 --- a/src/mailman/database/mailman.sql +++ b/src/mailman/database/mailman.sql @@ -54,6 +54,17 @@ CREATE INDEX ix_autoresponserecord_address_id CREATE INDEX ix_autoresponserecord_mailing_list_id ON autoresponserecord (mailing_list_id); +CREATE TABLE bounceevent ( + id INTEGER NOT NULL, + list_name TEXT, + email TEXT, + 'timestamp' TIMESTAMP, + message_id TEXT, + context TEXT, + processed BOOLEAN, + PRIMARY KEY (id) + ); + CREATE TABLE contentfilter ( id INTEGER NOT NULL, mailing_list_id INTEGER, diff --git a/src/mailman/database/types.py b/src/mailman/database/types.py index 1195802ff..f126cc05a 100644 --- a/src/mailman/database/types.py +++ b/src/mailman/database/types.py @@ -34,7 +34,10 @@ from mailman.utilities.modules import find_name class _EnumVariable(Variable): - """Storm variable.""" + """Storm variable. + + To use this, make the database column a TEXT. + """ def parse_set(self, value, from_db): if value is None: diff --git a/src/mailman/interfaces/bounce.py b/src/mailman/interfaces/bounce.py index 22e2467b8..168ca7ee0 100644 --- a/src/mailman/interfaces/bounce.py +++ b/src/mailman/interfaces/bounce.py @@ -21,13 +21,16 @@ from __future__ import absolute_import, unicode_literals __metaclass__ = type __all__ = [ + 'BounceContext', 'IBounceDetector', + 'IBounceEvent', + 'IBounceProcessor', 'Stop', ] from flufl.enum import Enum -from zope.interface import Interface +from zope.interface import Attribute, Interface @@ -39,6 +42,19 @@ Stop = object() +class BounceContext(Enum): + """The context in which the bounce was detected.""" + + # This is a normal bounce detection. IOW, Mailman received a bounce in + # response to a mailing list post. + normal = 1 + + # A probe email bounced. This can be considered a bit more serious, since + # it occurred in response to a specific message to a specific user. + probe = 2 + + + class IBounceDetector(Interface): """Detect a bounce in an email message.""" @@ -52,3 +68,48 @@ class IBounceDetector(Interface): returned to halt any bounce processing pipeline. :rtype: A set strings, or `Stop` """ + + + +class IBounceEvent(Interface): + """Registration record for a single bounce event.""" + + list_name = Attribute( + """The name of the mailing list that received this bounce.""") + + email = Attribute( + """The email address that bounced.""") + + timestamp = Attribute( + """The timestamp for when the bounce was received.""") + + message_id = Attribute( + """The Message-ID of the bounce message.""") + + context = Attribute( + """Where was the bounce detected?""") + + processed = Attribute( + """Has this bounce event been processed?""") + + + +class IBounceProcessor(Interface): + """Manager/processor of bounce events.""" + + def register(mlist, email, msg, context=None): + """Register a bounce event. + + :param mlist: The mailing list that the bounce occurred on. + :type mlist: IMailingList + :param email: The email address that is bouncing. + :type email: str + :param msg: The bounce message. + :type msg: email.message.Message + :param context: In what context was the bounce detected? The default + is 'normal' context (i.e. we received a normal bounce for the + address). + :type context: BounceContext + :return: The registered bounce event. + :rtype: IBounceEvent + """ diff --git a/src/mailman/model/bounce.py b/src/mailman/model/bounce.py new file mode 100644 index 000000000..8abe3d149 --- /dev/null +++ b/src/mailman/model/bounce.py @@ -0,0 +1,66 @@ +# Copyright (C) 2011 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Bounce support.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'BounceEvent', + 'BounceProcessor', + ] + + +from storm.locals import Bool, Int, DateTime, Unicode +from zope.interface import implements + +from mailman.interfaces.bounce import ( + BounceContext, IBounceEvent, IBounceProcessor) +from mailman.database.model import Model +from mailman.database.types import Enum +from mailman.utilities.datetime import now + + + +class BounceEvent(Model): + implements(IBounceEvent) + + id = Int(primary=True) + list_name = Unicode() + email = Unicode() + timestamp = DateTime() + message_id = Unicode() + context = Enum() + processed = Bool() + + def __init__(self, list_name, email, msg, context=None): + self.list_name = list_name + self.email = email + self.timestamp = now() + self.message_id = msg['message-id'] + self.context = (BounceContext.normal if context is None else context) + self.processed = False + + + +class BounceProcessor: + implements(IBounceProcessor) + + def register(self, mlist, email, msg, where=None): + """See `IBounceProcessor`.""" + return BounceEvent(mlist.fqdn_listname, email, msg, where) diff --git a/src/mailman/model/docs/bounce.rst b/src/mailman/model/docs/bounce.rst new file mode 100644 index 000000000..41784cd9c --- /dev/null +++ b/src/mailman/model/docs/bounce.rst @@ -0,0 +1,84 @@ +======= +Bounces +======= + +When a message to an email address bounces, Mailman's bounce runner will +register a bounce event. This registration is done through a utility. + + >>> from zope.component import getUtility + >>> from zope.interface.verify import verifyObject + >>> from mailman.interfaces.bounce import IBounceProcessor + >>> processor = getUtility(IBounceProcessor) + >>> verifyObject(IBounceProcessor, processor) + True + + +Registration +============ + +When a bounce occurs, it's always within the context of a specific mailing +list. + + >>> mlist = create_list('test@example.com') + +The bouncing email contains useful information that will be registered as +well. In particular, the Message-ID is a key piece of data that needs to be +recorded. + + >>> msg = message_from_string("""\ + ... From: mail-daemon@example.org + ... To: test-bounces@example.com + ... Message-ID: <first> + ... + ... """) + +There is a suite of bounce detectors that are used to heuristically extract +the bouncing email addresses. Various techniques are employed including VERP, +DSN, and magic. It is the bounce queue's responsibility to extract the set of +bouncing email addrsses. These are passed one-by-one to the registration +interface. + + >>> event = processor.register(mlist, 'anne@example.com', msg) + >>> print event.list_name + test@example.com + >>> print event.email + anne@example.com + >>> print event.message_id + <first> + +Bounce events have a timestamp. + + >>> print event.timestamp + 2005-08-01 07:49:23 + +Bounce events have a flag indicating whether they've been processed or not. + + >>> event.processed + False + +When a bounce is registered, you can indicate the bounce context. + + >>> msg = message_from_string("""\ + ... From: mail-daemon@example.org + ... To: test-bounces@example.com + ... Message-ID: <second> + ... + ... """) + +If no context is given, then a default one is used. + + >>> event = processor.register(mlist, 'bart@example.com', msg) + >>> print event.message_id + <second> + >>> print event.context + BounceContext.normal + +A probe bounce carries more weight than just a normal bounce. + + >>> from mailman.interfaces.bounce import BounceContext + >>> event = processor.register( + ... mlist, 'bart@example.com', msg, BounceContext.probe) + >>> print event.message_id + <second> + >>> print event.context + BounceContext.probe diff --git a/src/mailman/model/mailinglist.py b/src/mailman/model/mailinglist.py index 9294fe7cc..6952abcf0 100644 --- a/src/mailman/model/mailinglist.py +++ b/src/mailman/model/mailinglist.py @@ -195,7 +195,7 @@ class MailingList(Model): # For the pending database self.next_request_id = 1 self._restore() - self.personalization = Personalization.none + self.personalize = Personalization.none self.real_name = string.capwords( SPACE.join(listname.split(UNDERSCORE))) makedirs(self.data_path) diff --git a/src/mailman/model/tests/test_bounce.py b/src/mailman/model/tests/test_bounce.py new file mode 100644 index 000000000..fb0bf0875 --- /dev/null +++ b/src/mailman/model/tests/test_bounce.py @@ -0,0 +1,46 @@ +# Copyright (C) 2011 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test bounce model objects.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'test_suite', + ] + + +import unittest + +from mailman.model.bounce import BounceEvent +from mailman.testing.layers import ConfigLayer + + + +class TestBounceEvents(unittest.TestCase): + layer = ConfigLayer + + + + + + +def test_suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(TestBounceEvents)) + return suite diff --git a/src/mailman/queue/outgoing.py b/src/mailman/queue/outgoing.py index 7ff194219..dfa60a02d 100644 --- a/src/mailman/queue/outgoing.py +++ b/src/mailman/queue/outgoing.py @@ -28,6 +28,7 @@ from mailman.interfaces.mailinglist import Personalization from mailman.interfaces.mta import SomeRecipientsFailed from mailman.queue import Runner from mailman.queue.bounce import BounceMixin +from mailman.utilities.datetime import now from mailman.utilities.modules import find_name @@ -56,7 +57,7 @@ class OutgoingRunner(Runner, BounceMixin): def _dispose(self, mlist, msg, msgdata): # See if we should retry delivery of this message again. deliver_after = msgdata.get('deliver_after', datetime.fromtimestamp(0)) - if datetime.now() < deliver_after: + if now() < deliver_after: return True # Calculate whether we should VERP this message or not. The results of # this set the 'verp' key in the message metadata. @@ -69,7 +70,7 @@ class OutgoingRunner(Runner, BounceMixin): # Also, if personalization is /not/ enabled, but # verp_delivery_interval is set (and we've hit this interval), then # again, this message should be VERP'd. Otherwise, no. - elif mlist.personalize <> Personalization.none: + elif mlist.personalize != Personalization.none: if as_boolean(config.mta.verp_personalized_deliveries): msgdata['verp'] = True elif interval == 0: @@ -118,15 +119,15 @@ class OutgoingRunner(Runner, BounceMixin): # occasionally move them back here for another shot at # delivery. if error.temporary_failures: - now = datetime.now() + current_time = now() recips = error.temporary_failures last_recip_count = msgdata.get('last_recip_count', 0) - deliver_until = msgdata.get('deliver_until', now) + deliver_until = msgdata.get('deliver_until', current_time) if len(recips) == last_recip_count: # We didn't make any progress, so don't attempt # delivery any longer. BAW: is this the best # disposition? - if now > deliver_until: + if current_time > deliver_until: return False else: # Keep trying to delivery this message for a while diff --git a/src/mailman/queue/tests/__init__.py b/src/mailman/queue/tests/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/mailman/queue/tests/__init__.py diff --git a/src/mailman/queue/tests/test_outgoing.py b/src/mailman/queue/tests/test_outgoing.py new file mode 100644 index 000000000..da090f31e --- /dev/null +++ b/src/mailman/queue/tests/test_outgoing.py @@ -0,0 +1,245 @@ +# Copyright (C) 2011 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test the outgoing queue runner.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'test_suite', + ] + + +import unittest + +from contextlib import contextmanager +from datetime import timedelta + +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.interfaces.mailinglist import Personalization +from mailman.queue.outgoing import OutgoingRunner +from mailman.testing.helpers import ( + get_queue_messages, + make_testable_runner, + specialized_message_from_string as message_from_string) +from mailman.testing.layers import ConfigLayer, SMTPLayer +from mailman.utilities.datetime import now + + + +def run_once(qrunner): + """Predicate for make_testable_runner(). + + Ensures that the queue runner only runs once. + """ + return True + + +@contextmanager +def temporary_config(name, settings): + """Temporarily set a configuration (use in a with-statement).""" + config.push(name, settings) + try: + yield + finally: + config.pop(name) + + + +class TestOnce(unittest.TestCase): + """Test outgoing runner message disposition.""" + + layer = SMTPLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._outq = config.switchboards['out'] + self._runner = make_testable_runner(OutgoingRunner, 'out', run_once) + self._msg = message_from_string("""\ +From: anne@example.com +To: test@example.com +Message-Id: <first> + +""") + self._msgdata = {} + + def test_deliver_after(self): + # When the metadata has a deliver_after key in the future, the queue + # runner will re-enqueue the message rather than delivering it. + deliver_after = now() + timedelta(days=10) + self._msgdata['deliver_after'] = deliver_after + self._outq.enqueue(self._msg, self._msgdata, + tolist=True, listname='test@example.com') + self._runner.run() + items = get_queue_messages('out') + self.assertEqual(len(items), 1) + self.assertEqual(items[0].msgdata['deliver_after'], deliver_after) + self.assertEqual(items[0].msg['message-id'], '<first>') + + + +captured_mlist = None +captured_msg = None +captured_msgdata = None + +def capture(mlist, msg, msgdata): + global captured_mlist, captured_msg, captured_msgdata + captured_mlist = mlist + captured_msg = msg + captured_msgdata = msgdata + + +class TestVERPSettings(unittest.TestCase): + """Test the selection of VERP based on various criteria.""" + + layer = ConfigLayer + + def setUp(self): + global captured_mlist, captured_msg, captured_msgdata + # Push a config where actual delivery is handled by a dummy function. + # We generally don't care what this does, since we're just testing the + # setting of the 'verp' key in the metadata. + config.push('fake outgoing', """ + [mta] + outgoing: mailman.queue.tests.test_outgoing.capture + """) + # Reset the captured data. + captured_mlist = None + captured_msg = None + captured_msgdata = None + self._mlist = create_list('test@example.com') + self._outq = config.switchboards['out'] + self._runner = make_testable_runner(OutgoingRunner, 'out') + self._msg = message_from_string("""\ +From: anne@example.com +To: test@example.com +Message-Id: <first> + +""") + + def tearDown(self): + config.pop('fake outgoing') + + def test_delivery_callback(self): + # Test that the configuration variable calls the appropriate callback. + self._outq.enqueue(self._msg, {}, listname='test@example.com') + self._runner.run() + self.assertEqual(captured_mlist, self._mlist) + self.assertEqual(captured_msg.as_string(), self._msg.as_string()) + # Of course, the message metadata will contain a bunch of keys added + # by the processing. We don't really care about the details, so this + # test is a good enough stand-in. + self.assertEqual(captured_msgdata['listname'], 'test@example.com') + + def test_verp_in_metadata(self): + # Test that if the metadata has a 'verp' key, it is unchanged. + marker = 'yepper' + msgdata = dict(verp=marker) + self._outq.enqueue(self._msg, msgdata, listname='test@example.com') + self._runner.run() + self.assertEqual(captured_msgdata['verp'], marker) + + def test_personalized_individual_deliveries_verp(self): + # When deliveries are personalized, and the configuration setting + # indicates, messages will be VERP'd. + msgdata = {} + self._mlist.personalize = Personalization.individual + self._outq.enqueue(self._msg, msgdata, listname='test@example.com') + with temporary_config('personalize', """ + [mta] + verp_personalized_deliveries: yes + """): + self._runner.run() + self.assertTrue(captured_msgdata['verp']) + + def test_personalized_full_deliveries_verp(self): + # When deliveries are personalized, and the configuration setting + # indicates, messages will be VERP'd. + msgdata = {} + self._mlist.personalize = Personalization.full + self._outq.enqueue(self._msg, msgdata, listname='test@example.com') + with temporary_config('personalize', """ + [mta] + verp_personalized_deliveries: yes + """): + self._runner.run() + self.assertTrue(captured_msgdata['verp']) + + def test_personalized_deliveries_no_verp(self): + # When deliveries are personalized, but the configuration setting + # does not indicate, messages will not be VERP'd. + msgdata = {} + self._mlist.personalize = Personalization.full + self._outq.enqueue(self._msg, msgdata, listname='test@example.com') + self._runner.run() + self.assertFalse('verp' in captured_msgdata) + + def test_verp_never(self): + # Never VERP when the interval is zero. + msgdata = {} + self._outq.enqueue(self._msg, msgdata, listname='test@example.com') + with temporary_config('personalize', """ + [mta] + verp_delivery_interval: 0 + """): + self._runner.run() + self.assertEqual(captured_msgdata['verp'], False) + + def test_verp_always(self): + # Always VERP when the interval is one. + msgdata = {} + self._outq.enqueue(self._msg, msgdata, listname='test@example.com') + with temporary_config('personalize', """ + [mta] + verp_delivery_interval: 1 + """): + self._runner.run() + self.assertEqual(captured_msgdata['verp'], True) + + def test_verp_on_interval_match(self): + # VERP every so often, when the post_id matches. + self._mlist.post_id = 5 + msgdata = {} + self._outq.enqueue(self._msg, msgdata, listname='test@example.com') + with temporary_config('personalize', """ + [mta] + verp_delivery_interval: 5 + """): + self._runner.run() + self.assertEqual(captured_msgdata['verp'], True) + + def test_no_verp_on_interval_miss(self): + # VERP every so often, when the post_id matches. + self._mlist.post_id = 4 + msgdata = {} + self._outq.enqueue(self._msg, msgdata, listname='test@example.com') + with temporary_config('personalize', """ + [mta] + verp_delivery_interval: 5 + """): + self._runner.run() + self.assertEqual(captured_msgdata['verp'], False) + + + +def test_suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(TestOnce)) + suite.addTest(unittest.makeSuite(TestVERPSettings)) + return suite diff --git a/src/mailman/testing/helpers.py b/src/mailman/testing/helpers.py index 71cddd0f4..9f9ea6181 100644 --- a/src/mailman/testing/helpers.py +++ b/src/mailman/testing/helpers.py @@ -63,7 +63,7 @@ from mailman.utilities.mailbox import Mailbox -def make_testable_runner(runner_class, name=None): +def make_testable_runner(runner_class, name=None, predicate=None): """Create a queue runner that runs until its queue is empty. :param runner_class: The queue runner's class. @@ -71,6 +71,10 @@ def make_testable_runner(runner_class, name=None): :param name: Optional queue name; if not given, it is calculated from the class name. :type name: string or None + :param predicate: Optional alternative predicate for deciding when to stop + the queue runner. When None (the default) it stops when the queue is + empty. + :type predicate: callable that gets one argument, the queue runner. :return: A runner instance. """ @@ -90,7 +94,10 @@ def make_testable_runner(runner_class, name=None): def _do_periodic(self): """Stop when the queue is empty.""" - self._stop = (len(self.switchboard.files) == 0) + if predicate is None: + self._stop = (len(self.switchboard.files) == 0) + else: + self._stop = predicate(self) return EmptyingRunner(name) |