From 1317dadb98e422a05342009d85ab1adfb79f86f2 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Thu, 22 Mar 2012 22:38:49 -0400 Subject: Add an owner chain and pipeline to the schema, model, and interface. Plumb this through the incoming runner. --- src/mailman/database/schema/postgres.sql | 2 + src/mailman/database/schema/sqlite.sql | 2 + src/mailman/interfaces/mailinglist.py | 20 ++++++- src/mailman/model/mailinglist.py | 2 + src/mailman/runners/incoming.py | 5 +- src/mailman/runners/tests/test_incoming.py | 94 ++++++++++++++++++++++++++++++ 6 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 src/mailman/runners/tests/test_incoming.py diff --git a/src/mailman/database/schema/postgres.sql b/src/mailman/database/schema/postgres.sql index bd7ef3f6b..2e9ba249f 100644 --- a/src/mailman/database/schema/postgres.sql +++ b/src/mailman/database/schema/postgres.sql @@ -84,6 +84,8 @@ CREATE TABLE mailinglist ( nondigestable BOOLEAN, nonmember_rejection_notice TEXT, obscure_addresses BOOLEAN, + owner_chain TEXT, + owner_pipeline TEXT, personalize INTEGER, post_id INTEGER, posting_chain TEXT, diff --git a/src/mailman/database/schema/sqlite.sql b/src/mailman/database/schema/sqlite.sql index 37b6ed8f2..e6211bf53 100644 --- a/src/mailman/database/schema/sqlite.sql +++ b/src/mailman/database/schema/sqlite.sql @@ -180,6 +180,8 @@ CREATE TABLE mailinglist ( nondigestable BOOLEAN, nonmember_rejection_notice TEXT, obscure_addresses BOOLEAN, + owner_chain TEXT, + owner_pipeline TEXT, personalize INTEGER, post_id INTEGER, posting_chain TEXT, diff --git a/src/mailman/interfaces/mailinglist.py b/src/mailman/interfaces/mailinglist.py index d92bae464..bced070d3 100644 --- a/src/mailman/interfaces/mailinglist.py +++ b/src/mailman/interfaces/mailinglist.py @@ -382,7 +382,7 @@ class IMailingList(Interface): # Processing. posting_chain = Attribute( - """This mailing list's moderation chain. + """This mailing list's posting moderation chain. When messages are posted to a mailing list, it first goes through a moderation chain to determine whether the message will be accepted. @@ -397,6 +397,24 @@ class IMailingList(Interface): This attribute names a pipeline for postings, which must exist. """) + owner_chain = Attribute( + """This mailing list's owner moderation chain. + + When messages are posted to the owners of a mailing list, it first + goes through a moderation chain to determine whether the message will + be accepted. This attribute names a chain for postings, which must + exist. + """) + + owner_pipeline = Attribute( + """This mailing list's owner posting pipeline. + + Every mailing list has a processing pipeline that messages flow + through once they've been accepted for posting to the owners of a + mailing list. This attribute names a pipeline for postings, which + must exist. + """) + data_path = Attribute( """The file system path to list-specific data. diff --git a/src/mailman/model/mailinglist.py b/src/mailman/model/mailinglist.py index 76f88caa7..e397d59d6 100644 --- a/src/mailman/model/mailinglist.py +++ b/src/mailman/model/mailinglist.py @@ -168,6 +168,8 @@ class MailingList(Model): nondigestable = Bool() nonmember_rejection_notice = Unicode() obscure_addresses = Bool() + owner_chain = Unicode() + owner_pipeline = Unicode() personalize = Enum(Personalization) post_id = Int() posting_chain = Unicode() diff --git a/src/mailman/runners/incoming.py b/src/mailman/runners/incoming.py index 7072f9bcc..d8db926c7 100644 --- a/src/mailman/runners/incoming.py +++ b/src/mailman/runners/incoming.py @@ -61,6 +61,9 @@ class IncomingRunner(Runner): pass config.db.commit() # Process the message through the mailing list's start chain. - process(mlist, msg, msgdata, mlist.posting_chain) + start_chain = (mlist.owner_chain + if msgdata.get('to_owner', False) + else mlist.posting_chain) + process(mlist, msg, msgdata, start_chain) # Do not keep this message queued. return False diff --git a/src/mailman/runners/tests/test_incoming.py b/src/mailman/runners/tests/test_incoming.py new file mode 100644 index 000000000..5a0d82765 --- /dev/null +++ b/src/mailman/runners/tests/test_incoming.py @@ -0,0 +1,94 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Test the incoming queue runner.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestIncoming', + ] + + +import unittest + +from mailman.app.lifecycle import create_list +from mailman.chains.base import TerminalChainBase +from mailman.config import config +from mailman.runners.incoming import IncomingRunner +from mailman.testing.helpers import ( + get_queue_messages, + make_testable_runner, + specialized_message_from_string as mfs) +from mailman.testing.layers import ConfigLayer + + + +class Chain(TerminalChainBase): + name = 'test' + description = 'a test chain' + + def __init__(self, marker): + self._marker = marker + + def _process(self, mlist, msg, msgdata): + msgdata['marker'] = self._marker + config.switchboards['out'].enqueue(msg, msgdata) + + + +class TestIncoming(unittest.TestCase): + """Test the incoming queue runner.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._mlist.posting_chain = 'test posting' + self._mlist.owner_chain = 'test owner' + config.chains['test posting'] = Chain('posting') + config.chains['test owner'] = Chain('owner') + self._in = make_testable_runner(IncomingRunner, 'in') + self._msg = mfs("""\ +From: anne@example.com +To: test@example.com + +""") + + def tearDown(self): + del config.chains['test posting'] + del config.chains['test owner'] + + def test_posting(self): + # A message posted to the list goes through the posting chain. + msgdata = dict(listname='test@example.com') + config.switchboards['in'].enqueue(self._msg, msgdata) + self._in.run() + messages = get_queue_messages('out') + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0].msgdata.get('marker'), 'posting') + + def test_owner(self): + # A message posted to the list goes through the posting chain. + msgdata = dict(listname='test@example.com', + to_owner=True) + config.switchboards['in'].enqueue(self._msg, msgdata) + self._in.run() + messages = get_queue_messages('out') + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0].msgdata.get('marker'), 'owner') -- cgit v1.3.1 From 3e8f20cd4d543bdbd575b9c61748e4a815f5e0ea Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Thu, 22 Mar 2012 23:15:15 -0400 Subject: Add a simple owner processing chain. --- src/mailman/chains/accept.py | 3 +- src/mailman/chains/owner.py | 56 +++++++++++++++++++++++++ src/mailman/chains/tests/test_owner.py | 75 ++++++++++++++++++++++++++++++++++ src/mailman/docs/NEWS.rst | 2 + 4 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 src/mailman/chains/owner.py create mode 100644 src/mailman/chains/tests/test_owner.py diff --git a/src/mailman/chains/accept.py b/src/mailman/chains/accept.py index b1e4b1cf0..4b326142b 100644 --- a/src/mailman/chains/accept.py +++ b/src/mailman/chains/accept.py @@ -62,7 +62,6 @@ class AcceptChain(TerminalChainBase): rule_misses = msgdata.get('rule_misses') if rule_misses: msg['X-Mailman-Rule-Misses'] = SEMISPACE.join(rule_misses) - accept_queue = config.switchboards['pipeline'] - accept_queue.enqueue(msg, msgdata) + config.switchboards['pipeline'].enqueue(msg, msgdata) log.info('ACCEPT: %s', msg.get('message-id', 'n/a')) notify(AcceptNotification(mlist, msg, msgdata, self)) diff --git a/src/mailman/chains/owner.py b/src/mailman/chains/owner.py new file mode 100644 index 000000000..ad0a04cea --- /dev/null +++ b/src/mailman/chains/owner.py @@ -0,0 +1,56 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""The standard -owner posting chain.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'BuiltInOwnerChain', + ] + + +import logging + +from zope.event import notify + +from mailman.chains.base import ChainNotification, TerminalChainBase +from mailman.config import config +from mailman.core.i18n import _ + + +log = logging.getLogger('mailman.vette') + + + +class OwnerNotification(ChainNotification): + """An event signaling that a message is accepted to the -owner address.""" + + + +class BuiltInOwnerChain(TerminalChainBase): + """Default built-in -owner address chain.""" + + name = 'default-owner-chain' + description = _('The built-in -owner posting chain.') + + def _process(self, mlist, msg, msgdata): + # At least for now, everything posted to -owners goes through. + config.switchboards['pipeline'].enqueue(msg, msgdata) + log.info('OWNER: %s', msg.get('message-id', 'n/a')) + notify(OwnerNotification(mlist, msg, msgdata, self)) diff --git a/src/mailman/chains/tests/test_owner.py b/src/mailman/chains/tests/test_owner.py new file mode 100644 index 000000000..db85d4967 --- /dev/null +++ b/src/mailman/chains/tests/test_owner.py @@ -0,0 +1,75 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Test the owner chain.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestOwnerChain', + ] + + +import unittest + +from mailman.app.lifecycle import create_list +from mailman.chains.owner import BuiltInOwnerChain, OwnerNotification +from mailman.core.chains import process +from mailman.testing.helpers import ( + event_subscribers, + get_queue_messages, + specialized_message_from_string as mfs) +from mailman.testing.layers import ConfigLayer + + + +class TestOwnerChain(unittest.TestCase): + """Test the owner chain.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._msg = mfs("""\ +From: anne@example.com +To: test@example.com +Message-ID: + +""") + + def test_owner_pipeline(self): + # Messages processed through the default owners chain end up in the + # pipeline queue, and an event gets sent. + # + # This event subscriber records the event that occurs when the message + # is processed by the owner chain. + events = [] + def catch_event(event): + events.append(event) + with event_subscribers(catch_event): + process(self._mlist, self._msg, {}, 'default-owner-chain') + self.assertEqual(len(events), 1) + event = events[0] + self.assertTrue(isinstance(event, OwnerNotification)) + self.assertEqual(event.mlist, self._mlist) + self.assertEqual(event.msg['message-id'], '') + self.assertTrue(isinstance(event.chain, BuiltInOwnerChain)) + messages = get_queue_messages('pipeline') + self.assertEqual(len(messages), 1) + message = messages[0].msg + self.assertEqual(message['message-id'], '') diff --git a/src/mailman/docs/NEWS.rst b/src/mailman/docs/NEWS.rst index b92a3b618..fa5d1cb90 100644 --- a/src/mailman/docs/NEWS.rst +++ b/src/mailman/docs/NEWS.rst @@ -75,6 +75,8 @@ Database - real_name -> display_name (mailinglist, user, address) * Schema additions: - mailinglist.filter_action + - mailinglist.owner_chain + - mailinglist.owner_pipeline REST ---- -- cgit v1.3.1 From 5e0bfbfec4cddcce6f170944b2a6d33866612add Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Fri, 23 Mar 2012 00:46:57 -0400 Subject: Extend the pipeline runner to handle messages to the -owner address. --- src/mailman/runners/pipeline.py | 5 +- src/mailman/runners/tests/test_pipeline.py | 115 +++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 src/mailman/runners/tests/test_pipeline.py diff --git a/src/mailman/runners/pipeline.py b/src/mailman/runners/pipeline.py index 8bee2c4cb..b031662b3 100644 --- a/src/mailman/runners/pipeline.py +++ b/src/mailman/runners/pipeline.py @@ -30,6 +30,9 @@ from mailman.core.runner import Runner class PipelineRunner(Runner): def _dispose(self, mlist, msg, msgdata): # Process the message through the mailing list's pipeline. - process(mlist, msg, msgdata, mlist.posting_pipeline) + pipeline = (mlist.owner_pipeline + if msgdata.get('to_owner', False) + else mlist.posting_pipeline) + process(mlist, msg, msgdata, pipeline) # Do not keep this message queued. return False diff --git a/src/mailman/runners/tests/test_pipeline.py b/src/mailman/runners/tests/test_pipeline.py new file mode 100644 index 000000000..8776bf844 --- /dev/null +++ b/src/mailman/runners/tests/test_pipeline.py @@ -0,0 +1,115 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Test the pipeline runner.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestPipelineRunner', + ] + + +import unittest + +from zope.interface import implements + +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.interfaces.handler import IHandler +from mailman.interfaces.pipeline import IPipeline +from mailman.runners.pipeline import PipelineRunner +from mailman.testing.helpers import ( + make_testable_runner, + specialized_message_from_string as mfs) +from mailman.testing.layers import ConfigLayer + + + +class MyTestHandler: + implements(IHandler) + name = 'test handler' + description = 'A test handler' + + def __init__(self, marker, test): + self._marker = marker + self._test = test + + def process(self, mlist, msg, msgdata): + self._test.mark(self._marker) + + +class MyTestPipeline: + implements(IPipeline) + name = 'test' + description = 'a test pipeline' + + def __init__(self, marker, test): + self._marker = marker + self._test = test + + def __iter__(self): + yield MyTestHandler(self._marker, self._test) + + + +class TestPipelineRunner(unittest.TestCase): + """Test the pipeline runner.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._mlist.posting_pipeline = 'test posting' + self._mlist.owner_pipeline = 'test owner' + config.pipelines['test posting'] = MyTestPipeline('posting', self) + config.pipelines['test owner'] = MyTestPipeline('owner', self) + self._pipeline = make_testable_runner(PipelineRunner, 'pipeline') + self._markers = [] + self._msg = mfs("""\ +From: anne@example.com +To: test@example.com + +""") + + def tearDown(self): + del config.pipelines['test posting'] + del config.pipelines['test owner'] + + def mark(self, marker): + # Record a marker seen by a handler. + self._markers.append(marker) + + def test_posting(self): + # A message accepted for posting gets processed through the posting + # pipeline. + msgdata = dict(listname='test@example.com') + config.switchboards['pipeline'].enqueue(self._msg, msgdata) + self._pipeline.run() + self.assertEqual(len(self._markers), 1) + self.assertEqual(self._markers[0], 'posting') + + def test_owner(self): + # A message accepted for posting to a list's owners gets processed + # through the owner pipeline. + msgdata = dict(listname='test@example.com', + to_owner=True) + config.switchboards['pipeline'].enqueue(self._msg, msgdata) + self._pipeline.run() + self.assertEqual(len(self._markers), 1) + self.assertEqual(self._markers[0], 'owner') -- cgit v1.3.1 From 56ebf923ff311b4c1d195d46c04cadca4940b638 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Fri, 23 Mar 2012 13:36:58 -0400 Subject: Rename the src/mailman/pipeline directory to src/mailman/handlers since that's how I think about them anyway. --- src/mailman/core/pipelines.py | 2 +- src/mailman/handlers/__init__.py | 0 src/mailman/handlers/acknowledge.py | 89 +++++++ src/mailman/handlers/after_delivery.py | 48 ++++ src/mailman/handlers/avoid_duplicates.py | 116 +++++++++ src/mailman/handlers/calculate_recipients.py | 149 +++++++++++ src/mailman/handlers/cleanse.py | 77 ++++++ src/mailman/handlers/cleanse_dkim.py | 58 +++++ src/mailman/handlers/cook_headers.py | 292 +++++++++++++++++++++ src/mailman/handlers/decorate.py | 245 +++++++++++++++++ src/mailman/handlers/docs/ack-headers.rst | 43 +++ src/mailman/handlers/docs/acknowledge.rst | 174 +++++++++++++ src/mailman/handlers/docs/after-delivery.rst | 30 +++ src/mailman/handlers/docs/archives.rst | 133 ++++++++++ src/mailman/handlers/docs/avoid-duplicates.rst | 175 +++++++++++++ src/mailman/handlers/docs/calc-recips.rst | 114 ++++++++ src/mailman/handlers/docs/cleanse.rst | 100 +++++++ src/mailman/handlers/docs/cook-headers.rst | 129 +++++++++ src/mailman/handlers/docs/decorate.rst | 348 +++++++++++++++++++++++++ src/mailman/handlers/docs/digests.rst | 113 ++++++++ src/mailman/handlers/docs/file-recips.rst | 111 ++++++++ src/mailman/handlers/docs/filtering.rst | 347 ++++++++++++++++++++++++ src/mailman/handlers/docs/nntp.rst | 68 +++++ src/mailman/handlers/docs/reply-to.rst | 131 ++++++++++ src/mailman/handlers/docs/replybot.rst | 343 ++++++++++++++++++++++++ src/mailman/handlers/docs/rfc-2369.rst | 206 +++++++++++++++ src/mailman/handlers/docs/subject-munging.rst | 249 ++++++++++++++++++ src/mailman/handlers/docs/tagger.rst | 238 +++++++++++++++++ src/mailman/handlers/docs/to-outgoing.rst | 42 +++ src/mailman/handlers/file_recipients.py | 64 +++++ src/mailman/handlers/mime_delete.py | 302 +++++++++++++++++++++ src/mailman/handlers/owner_recipients.py | 35 +++ src/mailman/handlers/replybot.py | 123 +++++++++ src/mailman/handlers/rfc_2369.py | 113 ++++++++ src/mailman/handlers/tagger.py | 191 ++++++++++++++ src/mailman/handlers/tests/__init__.py | 0 src/mailman/handlers/tests/test_mimedel.py | 213 +++++++++++++++ src/mailman/handlers/to_archive.py | 55 ++++ src/mailman/handlers/to_digest.py | 123 +++++++++ src/mailman/handlers/to_outgoing.py | 52 ++++ src/mailman/handlers/to_usenet.py | 69 +++++ src/mailman/pipeline/__init__.py | 0 src/mailman/pipeline/acknowledge.py | 89 ------- src/mailman/pipeline/after_delivery.py | 48 ---- src/mailman/pipeline/avoid_duplicates.py | 116 --------- src/mailman/pipeline/calculate_recipients.py | 149 ----------- src/mailman/pipeline/cleanse.py | 77 ------ src/mailman/pipeline/cleanse_dkim.py | 58 ----- src/mailman/pipeline/cook_headers.py | 292 --------------------- src/mailman/pipeline/decorate.py | 245 ----------------- src/mailman/pipeline/docs/ack-headers.rst | 43 --- src/mailman/pipeline/docs/acknowledge.rst | 174 ------------- src/mailman/pipeline/docs/after-delivery.rst | 30 --- src/mailman/pipeline/docs/archives.rst | 133 ---------- src/mailman/pipeline/docs/avoid-duplicates.rst | 175 ------------- src/mailman/pipeline/docs/calc-recips.rst | 114 -------- src/mailman/pipeline/docs/cleanse.rst | 100 ------- src/mailman/pipeline/docs/cook-headers.rst | 129 --------- src/mailman/pipeline/docs/decorate.rst | 348 ------------------------- src/mailman/pipeline/docs/digests.rst | 113 -------- src/mailman/pipeline/docs/file-recips.rst | 111 -------- src/mailman/pipeline/docs/filtering.rst | 347 ------------------------ src/mailman/pipeline/docs/nntp.rst | 68 ----- src/mailman/pipeline/docs/reply-to.rst | 131 ---------- src/mailman/pipeline/docs/replybot.rst | 343 ------------------------ src/mailman/pipeline/docs/rfc-2369.rst | 206 --------------- src/mailman/pipeline/docs/subject-munging.rst | 249 ------------------ src/mailman/pipeline/docs/tagger.rst | 238 ----------------- src/mailman/pipeline/docs/to-outgoing.rst | 42 --- src/mailman/pipeline/file_recipients.py | 64 ----- src/mailman/pipeline/mime_delete.py | 302 --------------------- src/mailman/pipeline/owner_recipients.py | 35 --- src/mailman/pipeline/replybot.py | 123 --------- src/mailman/pipeline/rfc_2369.py | 112 -------- src/mailman/pipeline/tagger.py | 191 -------------- src/mailman/pipeline/tests/__init__.py | 0 src/mailman/pipeline/tests/test_mimedel.py | 213 --------------- src/mailman/pipeline/to_archive.py | 55 ---- src/mailman/pipeline/to_digest.py | 123 --------- src/mailman/pipeline/to_outgoing.py | 52 ---- src/mailman/pipeline/to_usenet.py | 69 ----- src/mailman/runners/digest.py | 3 +- 82 files changed, 5510 insertions(+), 5510 deletions(-) create mode 100644 src/mailman/handlers/__init__.py create mode 100644 src/mailman/handlers/acknowledge.py create mode 100644 src/mailman/handlers/after_delivery.py create mode 100644 src/mailman/handlers/avoid_duplicates.py create mode 100644 src/mailman/handlers/calculate_recipients.py create mode 100644 src/mailman/handlers/cleanse.py create mode 100644 src/mailman/handlers/cleanse_dkim.py create mode 100644 src/mailman/handlers/cook_headers.py create mode 100644 src/mailman/handlers/decorate.py create mode 100644 src/mailman/handlers/docs/ack-headers.rst create mode 100644 src/mailman/handlers/docs/acknowledge.rst create mode 100644 src/mailman/handlers/docs/after-delivery.rst create mode 100644 src/mailman/handlers/docs/archives.rst create mode 100644 src/mailman/handlers/docs/avoid-duplicates.rst create mode 100644 src/mailman/handlers/docs/calc-recips.rst create mode 100644 src/mailman/handlers/docs/cleanse.rst create mode 100644 src/mailman/handlers/docs/cook-headers.rst create mode 100644 src/mailman/handlers/docs/decorate.rst create mode 100644 src/mailman/handlers/docs/digests.rst create mode 100644 src/mailman/handlers/docs/file-recips.rst create mode 100644 src/mailman/handlers/docs/filtering.rst create mode 100644 src/mailman/handlers/docs/nntp.rst create mode 100644 src/mailman/handlers/docs/reply-to.rst create mode 100644 src/mailman/handlers/docs/replybot.rst create mode 100644 src/mailman/handlers/docs/rfc-2369.rst create mode 100644 src/mailman/handlers/docs/subject-munging.rst create mode 100644 src/mailman/handlers/docs/tagger.rst create mode 100644 src/mailman/handlers/docs/to-outgoing.rst create mode 100644 src/mailman/handlers/file_recipients.py create mode 100644 src/mailman/handlers/mime_delete.py create mode 100644 src/mailman/handlers/owner_recipients.py create mode 100644 src/mailman/handlers/replybot.py create mode 100644 src/mailman/handlers/rfc_2369.py create mode 100644 src/mailman/handlers/tagger.py create mode 100644 src/mailman/handlers/tests/__init__.py create mode 100644 src/mailman/handlers/tests/test_mimedel.py create mode 100644 src/mailman/handlers/to_archive.py create mode 100644 src/mailman/handlers/to_digest.py create mode 100644 src/mailman/handlers/to_outgoing.py create mode 100644 src/mailman/handlers/to_usenet.py delete mode 100644 src/mailman/pipeline/__init__.py delete mode 100644 src/mailman/pipeline/acknowledge.py delete mode 100644 src/mailman/pipeline/after_delivery.py delete mode 100644 src/mailman/pipeline/avoid_duplicates.py delete mode 100644 src/mailman/pipeline/calculate_recipients.py delete mode 100644 src/mailman/pipeline/cleanse.py delete mode 100644 src/mailman/pipeline/cleanse_dkim.py delete mode 100644 src/mailman/pipeline/cook_headers.py delete mode 100644 src/mailman/pipeline/decorate.py delete mode 100644 src/mailman/pipeline/docs/ack-headers.rst delete mode 100644 src/mailman/pipeline/docs/acknowledge.rst delete mode 100644 src/mailman/pipeline/docs/after-delivery.rst delete mode 100644 src/mailman/pipeline/docs/archives.rst delete mode 100644 src/mailman/pipeline/docs/avoid-duplicates.rst delete mode 100644 src/mailman/pipeline/docs/calc-recips.rst delete mode 100644 src/mailman/pipeline/docs/cleanse.rst delete mode 100644 src/mailman/pipeline/docs/cook-headers.rst delete mode 100644 src/mailman/pipeline/docs/decorate.rst delete mode 100644 src/mailman/pipeline/docs/digests.rst delete mode 100644 src/mailman/pipeline/docs/file-recips.rst delete mode 100644 src/mailman/pipeline/docs/filtering.rst delete mode 100644 src/mailman/pipeline/docs/nntp.rst delete mode 100644 src/mailman/pipeline/docs/reply-to.rst delete mode 100644 src/mailman/pipeline/docs/replybot.rst delete mode 100644 src/mailman/pipeline/docs/rfc-2369.rst delete mode 100644 src/mailman/pipeline/docs/subject-munging.rst delete mode 100644 src/mailman/pipeline/docs/tagger.rst delete mode 100644 src/mailman/pipeline/docs/to-outgoing.rst delete mode 100644 src/mailman/pipeline/file_recipients.py delete mode 100644 src/mailman/pipeline/mime_delete.py delete mode 100644 src/mailman/pipeline/owner_recipients.py delete mode 100644 src/mailman/pipeline/replybot.py delete mode 100644 src/mailman/pipeline/rfc_2369.py delete mode 100644 src/mailman/pipeline/tagger.py delete mode 100644 src/mailman/pipeline/tests/__init__.py delete mode 100644 src/mailman/pipeline/tests/test_mimedel.py delete mode 100644 src/mailman/pipeline/to_archive.py delete mode 100644 src/mailman/pipeline/to_digest.py delete mode 100644 src/mailman/pipeline/to_outgoing.py delete mode 100644 src/mailman/pipeline/to_usenet.py diff --git a/src/mailman/core/pipelines.py b/src/mailman/core/pipelines.py index d5cee588b..9d24cc61f 100644 --- a/src/mailman/core/pipelines.py +++ b/src/mailman/core/pipelines.py @@ -131,7 +131,7 @@ class VirginPipeline(BasePipeline): def initialize(): """Initialize the pipelines.""" # Find all handlers in the registered plugins. - for handler_class in find_components('mailman.pipeline', IHandler): + for handler_class in find_components('mailman.handlers', IHandler): handler = handler_class() verifyObject(IHandler, handler) assert handler.name not in config.handlers, ( diff --git a/src/mailman/handlers/__init__.py b/src/mailman/handlers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/mailman/handlers/acknowledge.py b/src/mailman/handlers/acknowledge.py new file mode 100644 index 000000000..0e0916337 --- /dev/null +++ b/src/mailman/handlers/acknowledge.py @@ -0,0 +1,89 @@ +# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Send an acknowledgment of the successful post to the sender. + +This only happens if the sender has set their AcknowledgePosts attribute. +""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'Acknowledge', + ] + + +from zope.component import getUtility +from zope.interface import implements + +from mailman.core.i18n import _ +from mailman.email.message import UserNotification +from mailman.interfaces.handler import IHandler +from mailman.interfaces.languages import ILanguageManager +from mailman.utilities.i18n import make +from mailman.utilities.string import oneline + + + +class Acknowledge: + """Send an acknowledgment.""" + implements(IHandler) + + name = 'acknowledge' + description = _("""Send an acknowledgment of a posting.""") + + def process(self, mlist, msg, msgdata): + """See `IHandler`.""" + # Extract the sender's address and find them in the user database + sender = msgdata.get('original_sender', msg.sender) + member = mlist.members.get_member(sender) + if member is None or not member.acknowledge_posts: + # Either the sender is not a member, in which case we can't know + # whether they want an acknowlegment or not, or they are a member + # who definitely does not want an acknowlegment. + return + # Okay, they are a member that wants an acknowledgment of their post. + # Give them their original subject. BAW: do we want to use the + # decoded header? + original_subject = msgdata.get( + 'origsubj', msg.get('subject', _('(no subject)'))) + # Get the user's preferred language. + language_manager = getUtility(ILanguageManager) + language = (language_manager[msgdata['lang']] + if 'lang' in msgdata + else member.preferred_language) + charset = language_manager[language.code].charset + # Now get the acknowledgement template. + display_name = mlist.display_name + text = make('postack.txt', + mailing_list=mlist, + language=language.code, + wrap=False, + subject=oneline(original_subject, charset), + list_name=mlist.list_name, + display_name=display_name, + listinfo_url=mlist.script_url('listinfo'), + optionsurl=member.options_url, + ) + # Craft the outgoing message, with all headers and attributes + # necessary for general delivery. Then enqueue it to the outgoing + # queue. + subject = _('$display_name post acknowledgment') + usermsg = UserNotification(sender, mlist.bounces_address, + subject, text, language) + usermsg.send(mlist) diff --git a/src/mailman/handlers/after_delivery.py b/src/mailman/handlers/after_delivery.py new file mode 100644 index 000000000..46007092b --- /dev/null +++ b/src/mailman/handlers/after_delivery.py @@ -0,0 +1,48 @@ +# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Perform some bookkeeping after a successful post.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'AfterDelivery', + ] + + +import datetime + +from zope.interface import implements + +from mailman.core.i18n import _ +from mailman.interfaces.handler import IHandler + + + +class AfterDelivery: + """Perform some bookkeeping after a successful post.""" + + implements(IHandler) + + name = 'after-delivery' + description = _('Perform some bookkeeping after a successful post.') + + def process(self, mlist, msg, msgdata): + """See `IHander`.""" + mlist.last_post_time = datetime.datetime.now() + mlist.post_id += 1 diff --git a/src/mailman/handlers/avoid_duplicates.py b/src/mailman/handlers/avoid_duplicates.py new file mode 100644 index 000000000..ffbc80c85 --- /dev/null +++ b/src/mailman/handlers/avoid_duplicates.py @@ -0,0 +1,116 @@ +# Copyright (C) 2002-2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""If the user wishes it, do not send duplicates of the same message. + +This module keeps an in-memory dictionary of Message-ID: and recipient pairs. +If a message with an identical Message-ID: is about to be sent to someone who +has already received a copy, we either drop the message, add a duplicate +warning header, or pass it through, depending on the user's preferences. +""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'AvoidDuplicates', + ] + + +from email.utils import getaddresses, formataddr +from zope.interface import implements + +from mailman.core.i18n import _ +from mailman.interfaces.handler import IHandler + + +COMMASPACE = ', ' + + + +class AvoidDuplicates: + """If the user wishes it, do not send duplicates of the same message.""" + + implements(IHandler) + + name = 'avoid-duplicates' + description = _('Suppress some duplicates of the same message.') + + def process(self, mlist, msg, msgdata): + """See `IHandler`.""" + recips = msgdata.get('recipients') + # Short circuit + if not recips: + return + # Seed this set with addresses we don't care about dup avoiding. + listaddrs = set((mlist.posting_address, + mlist.bounces_address, + mlist.owner_address, + mlist.request_address)) + explicit_recips = listaddrs.copy() + # Figure out the set of explicit recipients. + cc_addresses = {} + for header in ('to', 'cc', 'resent-to', 'resent-cc'): + addrs = getaddresses(msg.get_all(header, [])) + header_addresses = dict((addr, formataddr((name, addr))) + for name, addr in addrs + if addr) + if header == 'cc': + # Yes, it's possible that an address is mentioned in multiple + # CC headers using different names. In that case, the last + # real name will win, but that doesn't seem like such a big + # deal. Besides, how else would you chose? + cc_addresses.update(header_addresses) + # Ignore the list addresses for purposes of dup avoidance. + explicit_recips |= set(header_addresses) + # Now strip out the list addresses. + explicit_recips -= listaddrs + if not explicit_recips: + # No one was explicitly addressed, so we can't do any dup + # collapsing + return + newrecips = set() + for r in recips: + # If this recipient is explicitly addressed... + if r in explicit_recips: + send_duplicate = True + # If the member wants to receive duplicates, or if the + # recipient is not a member at all, they will get a copy. + # header. + member = mlist.members.get_member(r) + if member and not member.receive_list_copy: + send_duplicate = False + # We'll send a duplicate unless the user doesn't wish it. If + # personalization is enabled, the add-dupe-header flag will + # add a X-Mailman-Duplicate: yes header for this user's + # message. + if send_duplicate: + msgdata.setdefault('add-dup-header', set()).add(r) + newrecips.add(r) + elif r in cc_addresses: + del cc_addresses[r] + else: + # Otherwise, this is the first time they've been in the recips + # list. Add them to the newrecips list and flag them as + # having received this message. + newrecips.add(r) + # Set the new list of recipients. XXX recips should always be a set. + msgdata['recipients'] = list(newrecips) + # RFC 2822 specifies zero or one CC header + if cc_addresses: + del msg['cc'] + msg['CC'] = COMMASPACE.join(cc_addresses.values()) diff --git a/src/mailman/handlers/calculate_recipients.py b/src/mailman/handlers/calculate_recipients.py new file mode 100644 index 000000000..15be07ecd --- /dev/null +++ b/src/mailman/handlers/calculate_recipients.py @@ -0,0 +1,149 @@ +# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Calculate the regular (i.e. non-digest) recipients of the message. + +This module calculates the non-digest recipients for the message based on the +list's membership and configuration options. It places the list of recipients +on the `recipients' attribute of the message. This attribute is used by the +SendmailDeliver and BulkDeliver modules. +""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'CalculateRecipients', + ] + +from zope.interface import implements + +from mailman.config import config +from mailman.core import errors +from mailman.core.i18n import _ +from mailman.interfaces.handler import IHandler +from mailman.interfaces.member import DeliveryStatus +from mailman.utilities.string import wrap + + + +class CalculateRecipients: + """Calculate the regular (i.e. non-digest) recipients of the message.""" + + implements(IHandler) + + name = 'calculate-recipients' + description = _('Calculate the regular recipients of the message.') + + def process(self, mlist, msg, msgdata): + """See `IHandler`.""" + # Short circuit if we've already calculated the recipients list, + # regardless of whether the list is empty or not. + if 'recipients' in msgdata: + return + # Should the original sender should be included in the recipients list? + include_sender = True + member = mlist.members.get_member(msg.sender) + if member and not member.receive_own_postings: + include_sender = False + # Support for urgent messages, which bypasses digests and disabled + # delivery and forces an immediate delivery to all members Right Now. + # We are specifically /not/ allowing the site admins password to work + # here because we want to discourage the practice of sending the site + # admin password through email in the clear. (see also Approve.py) + # + # XXX This is broken. + missing = object() + password = msg.get('urgent', missing) + if password is not missing: + if mlist.Authenticate((config.AuthListModerator, + config.AuthListAdmin), + password): + recipients = mlist.getMemberCPAddresses( + mlist.getRegularMemberKeys() + + mlist.getDigestMemberKeys()) + msgdata['recipients'] = recipients + return + else: + # Bad Urgent: password, so reject it instead of passing it on. + # I think it's better that the sender know they screwed up + # than to deliver it normally. + listname = mlist.display_name + text = _("""\ +Your urgent message to the $listname mailing list was not authorized for +delivery. The original message as received by Mailman is attached. +""") + raise errors.RejectMessage(wrap(text)) + # Calculate the regular recipients of the message + recipients = set(member.address.email + for member in mlist.regular_members.members + if member.delivery_status == DeliveryStatus.enabled) + # Remove the sender if they don't want to receive their own posts + if not include_sender and member.address.email in recipients: + recipients.remove(member.address.email) + # Handle topic classifications + do_topic_filters(mlist, msg, msgdata, recipients) + # Bookkeeping + msgdata['recipients'] = recipients + + + +def do_topic_filters(mlist, msg, msgdata, recipients): + """Filter out recipients based on topics.""" + if not mlist.topics_enabled: + # MAS: if topics are currently disabled for the list, send to all + # regardless of ReceiveNonmatchingTopics + return + hits = msgdata.get('topichits') + zap_recipients = [] + if hits: + # The message hit some topics, so only deliver this message to those + # who are interested in one of the hit topics. + for user in recipients: + utopics = mlist.getMemberTopics(user) + if not utopics: + # This user is not interested in any topics, so they get all + # postings. + continue + # BAW: Slow, first-match, set intersection! + for topic in utopics: + if topic in hits: + # The user wants this message + break + else: + # The user was interested in topics, but not any of the ones + # this message matched, so zap him. + zap_recipients.append(user) + else: + # The semantics for a message that did not hit any of the pre-canned + # topics is to troll through the membership list, looking for users + # who selected at least one topic of interest, but turned on + # ReceiveNonmatchingTopics. + for user in recipients: + if not mlist.getMemberTopics(user): + # The user did not select any topics of interest, so he gets + # this message by default. + continue + if not mlist.getMemberOption( + user, config.ReceiveNonmatchingTopics): + # The user has interest in some topics, but elects not to + # receive message that match no topics, so zap him. + zap_recipients.append(user) + # Otherwise, the user wants non-matching messages. + # Prune out the non-receiving users + for user in zap_recipients: + recipients.remove(user) diff --git a/src/mailman/handlers/cleanse.py b/src/mailman/handlers/cleanse.py new file mode 100644 index 000000000..605b843d0 --- /dev/null +++ b/src/mailman/handlers/cleanse.py @@ -0,0 +1,77 @@ +# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Cleanse certain headers from all messages.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'Cleanse', + ] + + +import logging + +from email.utils import formataddr +from zope.interface import implements + +from mailman.core.i18n import _ +from mailman.handlers.cook_headers import uheader +from mailman.interfaces.handler import IHandler + + +log = logging.getLogger('mailman.post') + + + +class Cleanse: + """Cleanse certain headers from all messages.""" + + implements(IHandler) + + name = 'cleanse' + description = _('Cleanse certain headers from all messages.') + + def process(self, mlist, msg, msgdata): + """See `IHandler`.""" + # Remove headers that could contain passwords. + del msg['approved'] + del msg['approve'] + del msg['x-approved'] + del msg['x-approve'] + del msg['urgent'] + # We remove other headers from anonymous lists. + if mlist.anonymous_list: + log.info('post to %s from %s anonymized', + mlist.fqdn_listname, msg.get('from')) + del msg['from'] + del msg['reply-to'] + del msg['sender'] + # Hotmail sets this one + del msg['x-originating-email'] + i18ndesc = str(uheader(mlist, mlist.description, 'From')) + msg['From'] = formataddr((i18ndesc, mlist.posting_address)) + msg['Reply-To'] = mlist.posting_address + # Some headers can be used to fish for membership. + del msg['return-receipt-to'] + del msg['disposition-notification-to'] + del msg['x-confirm-reading-to'] + # Pegasus mail uses this one... sigh. + del msg['x-pmrqc'] + # Don't let this header be spoofed. See RFC 5064. + del msg['archived-at'] diff --git a/src/mailman/handlers/cleanse_dkim.py b/src/mailman/handlers/cleanse_dkim.py new file mode 100644 index 000000000..d2cd32636 --- /dev/null +++ b/src/mailman/handlers/cleanse_dkim.py @@ -0,0 +1,58 @@ +# Copyright (C) 2006-2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Remove any 'DomainKeys' (or similar) headers. + +The values contained in these header lines are intended to be used by the +recipient to detect forgery or tampering in transit, and the modifications +made by Mailman to the headers and body of the message will cause these keys +to appear invalid. Removing them will at least avoid this misleading result, +and it will also give the MTA the opportunity to regenerate valid keys +originating at the Mailman server for the outgoing message. +""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'CleanseDKIM', + ] + + +from lazr.config import as_boolean +from zope.interface import implements + +from mailman.config import config +from mailman.core.i18n import _ +from mailman.interfaces.handler import IHandler + + + +class CleanseDKIM: + """Remove DomainKeys headers.""" + + implements(IHandler) + + name = 'cleanse-dkim' + description = _('Remove DomainKeys headers.') + + def process(self, mlist, msg, msgdata): + """See `IHandler`.""" + if as_boolean(config.mta.remove_dkim_headers): + del msg['domainkey-signature'] + del msg['dkim-signature'] + del msg['authentication-results'] diff --git a/src/mailman/handlers/cook_headers.py b/src/mailman/handlers/cook_headers.py new file mode 100644 index 000000000..2d117429c --- /dev/null +++ b/src/mailman/handlers/cook_headers.py @@ -0,0 +1,292 @@ +# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Cook a message's headers.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'CookHeaders', + ] + + +import re + +from email.errors import HeaderParseError +from email.header import Header, decode_header, make_header +from email.utils import parseaddr, formataddr, getaddresses +from zope.interface import implements + +from mailman.core.i18n import _ +from mailman.interfaces.handler import IHandler +from mailman.interfaces.mailinglist import Personalization, ReplyToMunging +from mailman.version import VERSION + + +COMMASPACE = ', ' +MAXLINELEN = 78 + +nonascii = re.compile('[^\s!-~]') + + + +def uheader(mlist, s, header_name=None, continuation_ws='\t', maxlinelen=None): + """Get the charset to encode the string in. + + Then search if there is any non-ascii character is in the string. If + there is and the charset is us-ascii then we use iso-8859-1 instead. If + the string is ascii only we use 'us-ascii' if another charset is + specified. + """ + charset = mlist.preferred_language.charset + if nonascii.search(s): + # use list charset but ... + if charset == 'us-ascii': + charset = 'iso-8859-1' + else: + # there is no nonascii so ... + charset = 'us-ascii' + return Header(s, charset, maxlinelen, header_name, continuation_ws) + + + +def process(mlist, msg, msgdata): + """Process the headers of the message.""" + # Set the "X-Ack: no" header if noack flag is set. + if msgdata.get('noack'): + del msg['x-ack'] + msg['X-Ack'] = 'no' + # Because we're going to modify various important headers in the email + # message, we want to save some of the information in the msgdata + # dictionary for later. Specifically, the sender header will get waxed, + # but we need it for the Acknowledge module later. + msgdata['original_sender'] = msg.sender + # VirginRunner sets _fasttrack for internally crafted messages. + fasttrack = msgdata.get('_fasttrack') + if not msgdata.get('isdigest') and not fasttrack: + try: + prefix_subject(mlist, msg, msgdata) + except (UnicodeError, ValueError): + # TK: Sometimes subject header is not MIME encoded for 8bit + # simply abort prefixing. + pass + # Add Precedence: and other useful headers. None of these are standard + # and finding information on some of them are fairly difficult. Some are + # just common practice, and we'll add more here as they become necessary. + # Good places to look are: + # + # http://www.dsv.su.se/~jpalme/ietf/jp-ietf-home.html + # http://www.faqs.org/rfcs/rfc2076.html + # + # None of these headers are added if they already exist. BAW: some + # consider the advertising of this a security breach. I.e. if there are + # known exploits in a particular version of Mailman and we know a site is + # using such an old version, they may be vulnerable. It's too easy to + # edit the code to add a configuration variable to handle this. + if 'x-mailman-version' not in msg: + msg['X-Mailman-Version'] = VERSION + # We set "Precedence: list" because this is the recommendation from the + # sendmail docs, the most authoritative source of this header's semantics. + if 'precedence' not in msg: + msg['Precedence'] = 'list' + # Reply-To: munging. Do not do this if the message is "fast tracked", + # meaning it is internally crafted and delivered to a specific user. BAW: + # Yuck, I really hate this feature but I've caved under the sheer pressure + # of the (very vocal) folks want it. OTOH, RFC 2822 allows Reply-To: to + # be a list of addresses, so instead of replacing the original, simply + # augment it. RFC 2822 allows max one Reply-To: header so collapse them + # if we're adding a value, otherwise don't touch it. (Should we collapse + # in all cases?) + if not fasttrack: + # A convenience function, requires nested scopes. pair is (name, addr) + new = [] + d = {} + def add(pair): + lcaddr = pair[1].lower() + if lcaddr in d: + return + d[lcaddr] = pair + new.append(pair) + # List admin wants an explicit Reply-To: added + if mlist.reply_goes_to_list == ReplyToMunging.explicit_header: + add(parseaddr(mlist.reply_to_address)) + # If we're not first stripping existing Reply-To: then we need to add + # the original Reply-To:'s to the list we're building up. In both + # cases we'll zap the existing field because RFC 2822 says max one is + # allowed. + if not mlist.first_strip_reply_to: + orig = msg.get_all('reply-to', []) + for pair in getaddresses(orig): + add(pair) + # Set Reply-To: header to point back to this list. Add this last + # because some folks think that some MUAs make it easier to delete + # addresses from the right than from the left. + if mlist.reply_goes_to_list == ReplyToMunging.point_to_list: + i18ndesc = uheader(mlist, mlist.description, 'Reply-To') + add((str(i18ndesc), mlist.posting_address)) + del msg['reply-to'] + # Don't put Reply-To: back if there's nothing to add! + if new: + # Preserve order + msg['Reply-To'] = COMMASPACE.join( + [formataddr(pair) for pair in new]) + # The To field normally contains the list posting address. However + # when messages are fully personalized, that header will get + # overwritten with the address of the recipient. We need to get the + # posting address in one of the recipient headers or they won't be + # able to reply back to the list. It's possible the posting address + # was munged into the Reply-To header, but if not, we'll add it to a + # Cc header. BAW: should we force it into a Reply-To header in the + # above code? + # Also skip Cc if this is an anonymous list as list posting address + # is already in From and Reply-To in this case. + if (mlist.personalize == Personalization.full and + mlist.reply_goes_to_list != ReplyToMunging.point_to_list and + not mlist.anonymous_list): + # Watch out for existing Cc headers, merge, and remove dups. Note + # that RFC 2822 says only zero or one Cc header is allowed. + new = [] + d = {} + for pair in getaddresses(msg.get_all('cc', [])): + add(pair) + i18ndesc = uheader(mlist, mlist.description, 'Cc') + add((str(i18ndesc), mlist.posting_address)) + del msg['Cc'] + msg['Cc'] = COMMASPACE.join([formataddr(pair) for pair in new]) + + + +def prefix_subject(mlist, msg, msgdata): + """Maybe add a subject prefix. + + Add the subject prefix unless the message is a digest or is being fast + tracked (e.g. internally crafted, delivered to a single user such as the + list admin). + """ + if not mlist.subject_prefix.strip(): + return + prefix = mlist.subject_prefix + subject = msg.get('subject', '') + # Try to figure out what the continuation_ws is for the header + if isinstance(subject, Header): + lines = str(subject).splitlines() + else: + lines = subject.splitlines() + ws = '\t' + if len(lines) > 1 and lines[1] and lines[1][0] in ' \t': + ws = lines[1][0] + msgdata['origsubj'] = subject + # The subject may be multilingual but we take the first charset as major + # one and try to decode. If it is decodable, returned subject is in one + # line and cset is properly set. If fail, subject is mime-encoded and + # cset is set as us-ascii. See detail for ch_oneline() (CookHeaders one + # line function). + subject, cset = ch_oneline(subject) + # TK: Python interpreter has evolved to be strict on ascii charset code + # range. It is safe to use unicode string when manupilating header + # contents with re module. It would be best to return unicode in + # ch_oneline() but here is temporary solution. + subject = unicode(subject, cset) + # If the subject_prefix contains '%d', it is replaced with the + # mailing list sequential number. Sequential number format allows + # '%d' or '%05d' like pattern. + prefix_pattern = re.escape(prefix) + # unescape '%' :-< + prefix_pattern = '%'.join(prefix_pattern.split(r'\%')) + p = re.compile('%\d*d') + if p.search(prefix, 1): + # prefix have number, so we should search prefix w/number in subject. + # Also, force new style. + prefix_pattern = p.sub(r'\s*\d+\s*', prefix_pattern) + subject = re.sub(prefix_pattern, '', subject) + rematch = re.match('((RE|AW|SV|VS)(\[\d+\])?:\s*)+', subject, re.I) + if rematch: + subject = subject[rematch.end():] + recolon = 'Re:' + else: + recolon = '' + # At this point, subject may become null if someone post mail with + # subject: [subject prefix] + if subject.strip() == '': + subject = _('(no subject)') + cset = mlist.preferred_language.charset + # and substitute %d in prefix with post_id + try: + prefix = prefix % mlist.post_id + except TypeError: + pass + # Get the header as a Header instance, with proper unicode conversion + if not recolon: + h = uheader(mlist, prefix, 'Subject', continuation_ws=ws) + else: + h = uheader(mlist, prefix, 'Subject', continuation_ws=ws) + h.append(recolon) + # TK: Subject is concatenated and unicode string. + subject = subject.encode(cset, 'replace') + h.append(subject, cset) + del msg['subject'] + msg['Subject'] = h + ss = uheader(mlist, recolon, 'Subject', continuation_ws=ws) + ss.append(subject, cset) + msgdata['stripped_subject'] = ss + + + +def ch_oneline(headerstr): + # Decode header string in one line and convert into single charset. + # Return (string, cset) tuple as check for failure. + try: + d = decode_header(headerstr) + # At this point, we should rstrip() every string because some + # MUA deliberately add trailing spaces when composing return + # message. + d = [(s.rstrip(), c) for (s, c) in d] + # Find all charsets in the original header. We use 'utf-8' rather + # than using the first charset (in mailman 2.1.x) if multiple + # charsets are used. + csets = [] + for (s, c) in d: + if c and c not in csets: + csets.append(c) + if len(csets) == 0: + cset = 'us-ascii' + elif len(csets) == 1: + cset = csets[0] + else: + cset = 'utf-8' + h = make_header(d) + ustr = unicode(h) + oneline = ''.join(ustr.splitlines()) + return oneline.encode(cset, 'replace'), cset + except (LookupError, UnicodeError, ValueError, HeaderParseError): + # possibly charset problem. return with undecoded string in one line. + return ''.join(headerstr.splitlines()), 'us-ascii' + + + +class CookHeaders: + """Modify message headers.""" + + implements(IHandler) + + name = 'cook-headers' + description = _('Modify message headers.') + + def process(self, mlist, msg, msgdata): + """See `IHandler`.""" + process(mlist, msg, msgdata) diff --git a/src/mailman/handlers/decorate.py b/src/mailman/handlers/decorate.py new file mode 100644 index 000000000..d6d156048 --- /dev/null +++ b/src/mailman/handlers/decorate.py @@ -0,0 +1,245 @@ +# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Decorate a message by sticking the header and footer around it.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'Decorate', + ] + + +import re +import logging + +from email.mime.text import MIMEText +from urllib2 import URLError +from zope.component import getUtility +from zope.interface import implements + +from mailman.core.i18n import _ +from mailman.email.message import Message +from mailman.interfaces.handler import IHandler +from mailman.interfaces.templates import ITemplateLoader +from mailman.utilities.string import expand + + +log = logging.getLogger('mailman.error') + + + +def process(mlist, msg, msgdata): + """Decorate the message with headers and footers.""" + # Digests and Mailman-craft messages should not get additional headers. + if msgdata.get('isdigest') or msgdata.get('nodecorate'): + return + d = {} + member = msgdata.get('member') + if member is not None: + # Calculate the extra personalization dictionary. + recipient = msgdata.get('recipient', member.address.original_email) + d['user_address'] = recipient + d['user_delivered_to'] = member.address.original_email + d['user_language'] = member.preferred_language.description + d['user_name'] = (member.user.display_name + if member.user.display_name + else member.address.original_email) + d['user_optionsurl'] = member.options_url + # These strings are descriptive for the log file and shouldn't be i18n'd + d.update(msgdata.get('decoration-data', {})) + try: + header = decorate(mlist, mlist.header_uri, d) + except URLError: + log.exception('Header decorator URI not found ({0}): {1}'.format( + mlist.fqdn_listname, mlist.header_uri)) + try: + footer = decorate(mlist, mlist.footer_uri, d) + except URLError: + log.exception('Footer decorator URI not found ({0}): {1}'.format( + mlist.fqdn_listname, mlist.footer_uri)) + # Escape hatch if both the footer and header are empty + if not header and not footer: + return + # Be MIME smart here. We only attach the header and footer by + # concatenation when the message is a non-multipart of type text/plain. + # Otherwise, if it is not a multipart, we make it a multipart, and then we + # add the header and footer as text/plain parts. + # + # BJG: In addition, only add the footer if the message's character set + # matches the charset of the list's preferred language. This is a + # suboptimal solution, and should be solved by allowing a list to have + # multiple headers/footers, for each language the list supports. + # + # Also, if the list's preferred charset is us-ascii, we can always + # safely add the header/footer to a plain text message since all + # charsets Mailman supports are strict supersets of us-ascii -- + # no, UTF-16 emails are not supported yet. + # + # TK: Message with 'charset=' cause trouble. So, instead of + # mgs.get_content_charset('us-ascii') ... + mcset = msg.get_content_charset() or 'us-ascii' + lcset = mlist.preferred_language.charset + msgtype = msg.get_content_type() + # BAW: If the charsets don't match, should we add the header and footer by + # MIME multipart chroming the message? + wrap = True + if not msg.is_multipart() and msgtype == 'text/plain': + # Save the RFC-3676 format parameters. + format_param = msg.get_param('format') + delsp = msg.get_param('delsp') + # Save 'Content-Transfer-Encoding' header in case decoration fails. + cte = msg.get('content-transfer-encoding') + # header/footer is now in unicode (2.2) + try: + oldpayload = unicode(msg.get_payload(decode=True), mcset) + del msg['content-transfer-encoding'] + frontsep = endsep = '' + if header and not header.endswith('\n'): + frontsep = '\n' + if footer and not oldpayload.endswith('\n'): + endsep = '\n' + payload = header + frontsep + oldpayload + endsep + footer + # When setting the payload for the message, try various charset + # encodings until one does not produce a UnicodeError. We'll try + # charsets in this order: the list's charset, the message's + # charset, then utf-8. It's okay if some of these are duplicates. + for cset in (lcset, mcset, 'utf-8'): + try: + msg.set_payload(payload.encode(cset), cset) + except UnicodeError: + pass + else: + if format_param: + msg.set_param('format', format_param) + if delsp: + msg.set_param('delsp', delsp) + wrap = False + break + except (LookupError, UnicodeError): + if cte: + # Restore the original c-t-e. + del msg['content-transfer-encoding'] + msg['Content-Transfer-Encoding'] = cte + elif msg.get_content_type() == 'multipart/mixed': + # The next easiest thing to do is just prepend the header and append + # the footer as additional subparts + payload = msg.get_payload() + if not isinstance(payload, list): + payload = [payload] + if footer: + mimeftr = MIMEText(footer.encode(lcset), 'plain', lcset) + mimeftr['Content-Disposition'] = 'inline' + payload.append(mimeftr) + if header: + mimehdr = MIMEText(header.encode(lcset), 'plain', lcset) + mimehdr['Content-Disposition'] = 'inline' + payload.insert(0, mimehdr) + msg.set_payload(payload) + wrap = False + # If we couldn't add the header or footer in a less intrusive way, we can + # at least do it by MIME encapsulation. We want to keep as much of the + # outer chrome as possible. + if not wrap: + return + # Because of the way Message objects are passed around to process(), we + # need to play tricks with the outer message -- i.e. the outer one must + # remain the same instance. So we're going to create a clone of the outer + # message, with all the header chrome intact, then copy the payload to it. + # This will give us a clone of the original message, and it will form the + # basis of the interior, wrapped Message. + inner = Message() + # Which headers to copy? Let's just do the Content-* headers + for h, v in msg.items(): + if h.lower().startswith('content-'): + inner[h] = v + inner.set_payload(msg.get_payload()) + # For completeness + inner.set_unixfrom(msg.get_unixfrom()) + inner.preamble = msg.preamble + inner.epilogue = msg.epilogue + # Don't copy get_charset, as this might be None, even if + # get_content_charset isn't. However, do make sure there is a default + # content-type, even if the original message was not MIME. + inner.set_default_type(msg.get_default_type()) + # BAW: HACK ALERT. + if hasattr(msg, '__version__'): + inner.__version__ = msg.__version__ + # Now, play games with the outer message to make it contain three + # subparts: the header (if any), the wrapped message, and the footer (if + # any). + payload = [inner] + if header: + mimehdr = MIMEText(header.encode(lcset), 'plain', lcset) + mimehdr['Content-Disposition'] = 'inline' + payload.insert(0, mimehdr) + if footer: + mimeftr = MIMEText(footer.encode(lcset), 'plain', lcset) + mimeftr['Content-Disposition'] = 'inline' + payload.append(mimeftr) + msg.set_payload(payload) + del msg['content-type'] + del msg['content-transfer-encoding'] + del msg['content-disposition'] + msg['Content-Type'] = 'multipart/mixed' + + + +def decorate(mlist, uri, extradict=None): + """Expand the decoration template.""" + if uri is None: + return '' + # Get the decorator template. + loader = getUtility(ITemplateLoader) + template_uri = expand(uri, dict( + listname=mlist.fqdn_listname, + language=mlist.preferred_language.code, + )) + template = loader.get(template_uri) + # Create a dictionary which includes the default set of interpolation + # variables allowed in headers and footers. These will be augmented by + # any key/value pairs in the extradict. + substitutions = dict( + fqdn_listname = mlist.fqdn_listname, + list_name = mlist.list_name, + host_name = mlist.mail_host, + display_name = mlist.display_name, + listinfo_uri = mlist.script_url('listinfo'), + list_requests = mlist.request_address, + description = mlist.description, + info = mlist.info, + ) + if extradict is not None: + substitutions.update(extradict) + text = expand(template, substitutions) + # Turn any \r\n line endings into just \n + return re.sub(r' *\r?\n', r'\n', text) + + + +class Decorate: + """Decorate a message with headers and footers.""" + + implements(IHandler) + + name = 'decorate' + description = _('Decorate a message with headers and footers.') + + def process(self, mlist, msg, msgdata): + "See `IHandler`.""" + process(mlist, msg, msgdata) diff --git a/src/mailman/handlers/docs/ack-headers.rst b/src/mailman/handlers/docs/ack-headers.rst new file mode 100644 index 000000000..e700e2fd1 --- /dev/null +++ b/src/mailman/handlers/docs/ack-headers.rst @@ -0,0 +1,43 @@ +====================== +Acknowledgment headers +====================== + +Messages that flow through the global pipeline get their headers `cooked`, +which basically means that their headers go through several mostly unrelated +transformations. Some headers get added, others get changed. Some of these +changes depend on mailing list settings and others depend on how the message +is getting sent through the system. We'll take things one-by-one. + + >>> mlist = create_list('_xtest@example.com') + >>> mlist.subject_prefix = '' + +When the message's metadata has a `noack` key set, an ``X-Ack: no`` header is +added. +:: + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... + ... A message of great import. + ... """) + + >>> from mailman.handlers.cook_headers import process + >>> process(mlist, msg, dict(noack=True)) + >>> print msg.as_string() + From: aperson@example.com + X-Ack: no + ... + +Any existing ``X-Ack`` header in the original message is removed. + + >>> msg = message_from_string("""\ + ... X-Ack: yes + ... From: aperson@example.com + ... + ... A message of great import. + ... """) + >>> process(mlist, msg, dict(noack=True)) + >>> print msg.as_string() + From: aperson@example.com + X-Ack: no + ... diff --git a/src/mailman/handlers/docs/acknowledge.rst b/src/mailman/handlers/docs/acknowledge.rst new file mode 100644 index 000000000..479aa4ea6 --- /dev/null +++ b/src/mailman/handlers/docs/acknowledge.rst @@ -0,0 +1,174 @@ +====================== +Message acknowledgment +====================== + +When a user posts a message to a mailing list, and that user has chosen to +receive acknowledgments of their postings, Mailman will sent them such an +acknowledgment. +:: + + >>> mlist = create_list('test@example.com') + >>> mlist.display_name = 'Test' + >>> mlist.preferred_language = 'en' + >>> # XXX This will almost certainly change once we've worked out the web + >>> # space layout for mailing lists now. + + >>> # Ensure that the virgin queue is empty, since we'll be checking this + >>> # for new auto-response messages. + >>> from mailman.testing.helpers import get_queue_messages + >>> get_queue_messages('virgin') + [] + +Subscribe a user to the mailing list. +:: + + >>> from mailman.interfaces.usermanager import IUserManager + >>> from zope.component import getUtility + >>> user_manager = getUtility(IUserManager) + + >>> from mailman.interfaces.member import MemberRole + >>> user_1 = user_manager.create_user('aperson@example.com') + >>> address_1 = list(user_1.addresses)[0] + >>> mlist.subscribe(address_1, MemberRole.member) + + + +Non-member posts +================ + +Non-members can't get acknowledgments of their posts to the mailing list. +:: + + >>> msg = message_from_string("""\ + ... From: bperson@example.com + ... + ... """) + + >>> handler = config.handlers['acknowledge'] + >>> handler.process(mlist, msg, {}) + >>> get_queue_messages('virgin') + [] + +We can also specify the original sender in the message's metadata. If that +person is also not a member, no acknowledgment will be sent either. + + >>> msg = message_from_string("""\ + ... From: bperson@example.com + ... + ... """) + >>> handler.process(mlist, msg, + ... dict(original_sender='cperson@example.com')) + >>> get_queue_messages('virgin') + [] + + +No acknowledgment requested +=========================== + +Unless the user has requested acknowledgments, they will not get one. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... + ... """) + >>> handler.process(mlist, msg, {}) + >>> get_queue_messages('virgin') + [] + +Similarly if the original sender is specified in the message metadata, and +that sender is a member but not one who has requested acknowledgments, none +will be sent. +:: + + >>> user_2 = user_manager.create_user('dperson@example.com') + >>> address_2 = list(user_2.addresses)[0] + >>> mlist.subscribe(address_2, MemberRole.member) + + + >>> handler.process(mlist, msg, + ... dict(original_sender='dperson@example.com')) + >>> get_queue_messages('virgin') + [] + + +Requested acknowledgments +========================= + +If the member requests acknowledgments, Mailman will send them one when they +post to the mailing list. + + >>> user_1.preferences.acknowledge_posts = True + +The receipt will include the original message's subject in the response body, + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... Subject: Something witty and insightful + ... + ... """) + >>> handler.process(mlist, msg, {}) + >>> messages = get_queue_messages('virgin') + >>> len(messages) + 1 + >>> dump_msgdata(messages[0].msgdata) + _parsemsg : False + listname : test@example.com + nodecorate : True + recipients : set([u'aperson@example.com']) + reduced_list_headers: True + ... + >>> print messages[0].msg.as_string() + ... + MIME-Version: 1.0 + ... + Subject: Test post acknowledgment + From: test-bounces@example.com + To: aperson@example.com + ... + Precedence: bulk + + Your message entitled + + Something witty and insightful + + was successfully received by the Test mailing list. + + List info page: http://lists.example.com/listinfo/test@example.com + Your preferences: http://example.com/aperson@example.com + + +If there is no subject, then the receipt will use a generic message. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... + ... """) + >>> handler.process(mlist, msg, {}) + >>> messages = get_queue_messages('virgin') + >>> len(messages) + 1 + >>> dump_msgdata(messages[0].msgdata) + _parsemsg : False + listname : test@example.com + nodecorate : True + recipients : set([u'aperson@example.com']) + reduced_list_headers: True + ... + >>> print messages[0].msg.as_string() + MIME-Version: 1.0 + ... + Subject: Test post acknowledgment + From: test-bounces@example.com + To: aperson@example.com + ... + Precedence: bulk + + Your message entitled + + (no subject) + + was successfully received by the Test mailing list. + + List info page: http://lists.example.com/listinfo/test@example.com + Your preferences: http://example.com/aperson@example.com + diff --git a/src/mailman/handlers/docs/after-delivery.rst b/src/mailman/handlers/docs/after-delivery.rst new file mode 100644 index 000000000..c3e393cf2 --- /dev/null +++ b/src/mailman/handlers/docs/after-delivery.rst @@ -0,0 +1,30 @@ +============== +After delivery +============== + +After a message is delivered, or more correctly, after it has been processed +by the rest of the handlers in the incoming queue pipeline, a couple of +bookkeeping pieces of information are updated. + + >>> import datetime + >>> mlist = create_list('_xtest@example.com') + >>> post_time = datetime.datetime.now() - datetime.timedelta(minutes=10) + >>> mlist.last_post_time = post_time + >>> mlist.post_id = 10 + +Processing a message with this handler updates the last_post_time and post_id +attributes. +:: + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... + ... Something interesting. + ... """) + + >>> handler = config.handlers['after-delivery'] + >>> handler.process(mlist, msg, {}) + >>> mlist.last_post_time > post_time + True + >>> mlist.post_id + 11 diff --git a/src/mailman/handlers/docs/archives.rst b/src/mailman/handlers/docs/archives.rst new file mode 100644 index 000000000..323d121e8 --- /dev/null +++ b/src/mailman/handlers/docs/archives.rst @@ -0,0 +1,133 @@ +======== +Archives +======== + +Updating the archives with posted messages is handled by a separate queue, +which allows for better memory management and prevents blocking the main +delivery processes while messages are archived. This also allows external +archivers to work in a separate process from the main Mailman delivery +processes. + + >>> handler = config.handlers['to-archive'] + >>> mlist = create_list('_xtest@example.com') + >>> switchboard = config.switchboards['archive'] + +A helper function. + + >>> def clear(): + ... for filebase in switchboard.files: + ... msg, msgdata = switchboard.dequeue(filebase) + ... switchboard.finish(filebase) + +The purpose of this handler is to make a simple decision as to whether the +message should get archived and if so, to drop the message in the archiving +queue. Really the most important things are to determine when a message +should *not* get archived. + +For example, no digests should ever get archived. + + >>> mlist.archive = True + >>> msg = message_from_string("""\ + ... Subject: A sample message + ... + ... A message of great import. + ... """) + >>> handler.process(mlist, msg, dict(isdigest=True)) + >>> switchboard.files + [] + +If the mailing list is not configured to archive, then even regular deliveries +won't be archived. + + >>> mlist.archive = False + >>> handler.process(mlist, msg, {}) + >>> switchboard.files + [] + +There are two de-facto standards for a message to indicate that it does not +want to be archived. We've seen both in the wild so both are supported. The +``X-No-Archive:`` header can be used to indicate that the message should not +be archived. Confusingly, this header's value is actually ignored. + + >>> mlist.archive = True + >>> msg = message_from_string("""\ + ... Subject: A sample message + ... X-No-Archive: YES + ... + ... A message of great import. + ... """) + >>> handler.process(mlist, msg, dict(isdigest=True)) + >>> switchboard.files + [] + +Even a ``no`` value will stop the archiving of the message. + + >>> msg = message_from_string("""\ + ... Subject: A sample message + ... X-No-Archive: No + ... + ... A message of great import. + ... """) + >>> handler.process(mlist, msg, dict(isdigest=True)) + >>> switchboard.files + [] + +Another header that's been observed is the ``X-Archive:`` header. Here, the +header's case folded value must be ``no`` in order to prevent archiving. + + >>> msg = message_from_string("""\ + ... Subject: A sample message + ... X-Archive: No + ... + ... A message of great import. + ... """) + >>> handler.process(mlist, msg, dict(isdigest=True)) + >>> switchboard.files + [] + +But if the value is ``yes``, then the message will be archived. + + >>> msg = message_from_string("""\ + ... Subject: A sample message + ... X-Archive: Yes + ... + ... A message of great import. + ... """) + >>> handler.process(mlist, msg, {}) + >>> len(switchboard.files) + 1 + >>> filebase = switchboard.files[0] + >>> qmsg, qdata = switchboard.dequeue(filebase) + >>> switchboard.finish(filebase) + >>> print qmsg.as_string() + Subject: A sample message + X-Archive: Yes + + A message of great import. + + >>> dump_msgdata(qdata) + _parsemsg: False + version : 3 + +Without either archiving header, and all other things being the same, the +message will get archived. + + >>> msg = message_from_string("""\ + ... Subject: A sample message + ... + ... A message of great import. + ... """) + >>> handler.process(mlist, msg, {}) + >>> len(switchboard.files) + 1 + >>> filebase = switchboard.files[0] + >>> qmsg, qdata = switchboard.dequeue(filebase) + >>> switchboard.finish(filebase) + >>> print qmsg.as_string() + Subject: A sample message + + A message of great import. + + >>> dump_msgdata(qdata) + _parsemsg: False + version : 3 diff --git a/src/mailman/handlers/docs/avoid-duplicates.rst b/src/mailman/handlers/docs/avoid-duplicates.rst new file mode 100644 index 000000000..1e46793c2 --- /dev/null +++ b/src/mailman/handlers/docs/avoid-duplicates.rst @@ -0,0 +1,175 @@ +================ +Avoid duplicates +================ + +This handler implements several strategies to reduce the reception of +duplicate messages. It does this by removing certain recipients from the list +of recipients calculated earlier. + + >>> mlist = create_list('_xtest@example.com') + +Create some members we're going to use. +:: + + >>> from mailman.interfaces.usermanager import IUserManager + >>> from zope.component import getUtility + >>> user_manager = getUtility(IUserManager) + + >>> address_a = user_manager.create_address('aperson@example.com') + >>> address_b = user_manager.create_address('bperson@example.com') + + >>> from mailman.interfaces.member import MemberRole + >>> member_a = mlist.subscribe(address_a, MemberRole.member) + >>> member_b = mlist.subscribe(address_b, MemberRole.member) + >>> # This is the message metadata dictionary as it would be produced by + >>> # the CalcRecips handler. + >>> recips = dict( + ... recipients=['aperson@example.com', 'bperson@example.com']) + + +Short circuiting +================ + +The module short-circuits if there are no recipients. +:: + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... Subject: A message of great import + ... + ... Something + ... """) + >>> msgdata = {} + + >>> handler = config.handlers['avoid-duplicates'] + >>> handler.process(mlist, msg, msgdata) + >>> msgdata + {} + >>> print msg.as_string() + From: aperson@example.com + Subject: A message of great import + + Something + + + +Suppressing the list copy +========================= + +Members can elect not to receive a list copy of any message on which they are +explicitly named as a recipient. This is done by setting their +``receive_list_copy`` preference to ``False``. However, if they aren't +mentioned in one of the recipient headers (i.e. ``To``, ``CC``, ``Resent-To``, +or ``Resent-CC``), then they will get a list copy. + + >>> member_a.preferences.receive_list_copy = False + >>> msg = message_from_string("""\ + ... From: Claire Person + ... + ... Something of great import. + ... """) + >>> msgdata = recips.copy() + >>> handler.process(mlist, msg, msgdata) + >>> sorted(msgdata['recipients']) + [u'aperson@example.com', u'bperson@example.com'] + >>> print msg.as_string() + From: Claire Person + + Something of great import. + + +If they're mentioned on the ``CC`` line, they won't get a list copy. + + >>> msg = message_from_string("""\ + ... From: Claire Person + ... CC: aperson@example.com + ... + ... Something of great import. + ... """) + >>> msgdata = recips.copy() + >>> handler.process(mlist, msg, msgdata) + >>> sorted(msgdata['recipients']) + [u'bperson@example.com'] + >>> print msg.as_string() + From: Claire Person + CC: aperson@example.com + + Something of great import. + + +But if they're mentioned on the ``CC`` line and have ``receive_list_copy`` set +to ``True`` (the default), then they still get a list copy. + + >>> msg = message_from_string("""\ + ... From: Claire Person + ... CC: bperson@example.com + ... + ... Something of great import. + ... """) + >>> msgdata = recips.copy() + >>> handler.process(mlist, msg, msgdata) + >>> sorted(msgdata['recipients']) + [u'aperson@example.com', u'bperson@example.com'] + >>> print msg.as_string() + From: Claire Person + CC: bperson@example.com + + Something of great import. + + +Other headers checked for recipients include the ``To``... + + >>> msg = message_from_string("""\ + ... From: Claire Person + ... To: aperson@example.com + ... + ... Something of great import. + ... """) + >>> msgdata = recips.copy() + >>> handler.process(mlist, msg, msgdata) + >>> sorted(msgdata['recipients']) + [u'bperson@example.com'] + >>> print msg.as_string() + From: Claire Person + To: aperson@example.com + + Something of great import. + + +... ``Resent-To`` ... + + >>> msg = message_from_string("""\ + ... From: Claire Person + ... Resent-To: aperson@example.com + ... + ... Something of great import. + ... """) + >>> msgdata = recips.copy() + >>> handler.process(mlist, msg, msgdata) + >>> sorted(msgdata['recipients']) + [u'bperson@example.com'] + >>> print msg.as_string() + From: Claire Person + Resent-To: aperson@example.com + + Something of great import. + + +...and ``Resent-CC`` headers. + + >>> msg = message_from_string("""\ + ... From: Claire Person + ... Resent-Cc: aperson@example.com + ... + ... Something of great import. + ... """) + >>> msgdata = recips.copy() + >>> handler.process(mlist, msg, msgdata) + >>> sorted(msgdata['recipients']) + [u'bperson@example.com'] + >>> print msg.as_string() + From: Claire Person + Resent-Cc: aperson@example.com + + Something of great import. + diff --git a/src/mailman/handlers/docs/calc-recips.rst b/src/mailman/handlers/docs/calc-recips.rst new file mode 100644 index 000000000..6dca85816 --- /dev/null +++ b/src/mailman/handlers/docs/calc-recips.rst @@ -0,0 +1,114 @@ +====================== +Calculating recipients +====================== + +Every message that makes it through to the list membership gets sent to a set +of recipient addresses. These addresses are calculated by one of the handler +modules and depends on a host of factors. + + >>> mlist = create_list('_xtest@example.com') + +Recipients are calculate from the list members, so add a bunch of members to +start out with. First, create a bunch of addresses... +:: + + >>> from mailman.interfaces.usermanager import IUserManager + >>> from zope.component import getUtility + >>> user_manager = getUtility(IUserManager) + + >>> address_a = user_manager.create_address('aperson@example.com') + >>> address_b = user_manager.create_address('bperson@example.com') + >>> address_c = user_manager.create_address('cperson@example.com') + >>> address_d = user_manager.create_address('dperson@example.com') + >>> address_e = user_manager.create_address('eperson@example.com') + >>> address_f = user_manager.create_address('fperson@example.com') + +...then subscribe these addresses to the mailing list as members... + + >>> from mailman.interfaces.member import MemberRole + >>> member_a = mlist.subscribe(address_a, MemberRole.member) + >>> member_b = mlist.subscribe(address_b, MemberRole.member) + >>> member_c = mlist.subscribe(address_c, MemberRole.member) + >>> member_d = mlist.subscribe(address_d, MemberRole.member) + >>> member_e = mlist.subscribe(address_e, MemberRole.member) + >>> member_f = mlist.subscribe(address_f, MemberRole.member) + +...then make some of the members digest members. + + >>> from mailman.core.constants import DeliveryMode + >>> member_d.preferences.delivery_mode = DeliveryMode.plaintext_digests + >>> member_e.preferences.delivery_mode = DeliveryMode.mime_digests + >>> member_f.preferences.delivery_mode = DeliveryMode.summary_digests + + +Short-circuiting +================ + +Sometimes, the list of recipients already exists in the message metadata. +This can happen for example, when a message was previously delivered to some +but not all of the recipients. +:: + + >>> msg = message_from_string("""\ + ... From: Xavier Person + ... + ... Something of great import. + ... """) + >>> recipients = set(('qperson@example.com', 'zperson@example.com')) + >>> msgdata = dict(recipients=recipients) + + >>> handler = config.handlers['calculate-recipients'] + >>> handler.process(mlist, msg, msgdata) + >>> dump_list(msgdata['recipients']) + qperson@example.com + zperson@example.com + + +Regular delivery recipients +=========================== + +Regular delivery recipients are those people who get messages from the list as +soon as they are posted. In other words, these folks are not digest members. + + >>> msgdata = {} + >>> handler.process(mlist, msg, msgdata) + >>> dump_list(msgdata['recipients']) + aperson@example.com + bperson@example.com + cperson@example.com + +Members can elect not to receive a list copy of their own postings. + + >>> member_c.preferences.receive_own_postings = False + >>> msg = message_from_string("""\ + ... From: Claire Person + ... + ... Something of great import. + ... """) + >>> msgdata = {} + >>> handler.process(mlist, msg, msgdata) + >>> dump_list(msgdata['recipients']) + aperson@example.com + bperson@example.com + +Members can also elect not to receive a list copy of any message on which they +are explicitly named as a recipient. However, see the `avoid duplicates`_ +handler for details. + + +Digest recipients +================= + +XXX Test various digest deliveries. + + +Urgent messages +=============== + +XXX Test various urgent deliveries: + * test_urgent_moderator() + * test_urgent_admin() + * test_urgent_reject() + + +.. _`avoid duplicates`: avoid-duplicates.html diff --git a/src/mailman/handlers/docs/cleanse.rst b/src/mailman/handlers/docs/cleanse.rst new file mode 100644 index 000000000..61dfa8f52 --- /dev/null +++ b/src/mailman/handlers/docs/cleanse.rst @@ -0,0 +1,100 @@ +================= +Cleansing headers +================= + +All messages posted to a list get their headers cleansed. Some headers are +related to additional permissions that can be granted to the message and other +headers can be used to fish for membership. + + >>> mlist = create_list('_xtest@example.com') + +Headers such as ``Approved``, ``Approve``, (as well as their ``X-`` variants) +and ``Urgent`` are used to grant special permissions to individual messages. +All may contain a password; the first two headers are used by list +administrators to pre-approve a message normal held for approval. The latter +header is used to send a regular message to all members, regardless of whether +they get digests or not. Because all three headers contain passwords, they +must be removed from any posted message. :: + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... Approved: foobar + ... Approve: barfoo + ... X-Approved: bazbar + ... X-Approve: barbaz + ... Urgent: notreally + ... Subject: A message of great import + ... + ... Blah blah blah + ... """) + + >>> handler = config.handlers['cleanse'] + >>> handler.process(mlist, msg, {}) + >>> print msg.as_string() + From: aperson@example.com + Subject: A message of great import + + Blah blah blah + + +Other headers can be used by list members to fish the list for membership, so +we don't let them go through. These are a mix of standard headers and custom +headers supported by some mail readers. For example, ``X-PMRC`` is supported +by Pegasus mail. I don't remember what program uses ``X-Confirm-Reading-To`` +though (Some Microsoft product perhaps?). + + >>> msg = message_from_string("""\ + ... From: bperson@example.com + ... Reply-To: bperson@example.org + ... Sender: asystem@example.net + ... Return-Receipt-To: another@example.com + ... Disposition-Notification-To: athird@example.com + ... X-Confirm-Reading-To: afourth@example.com + ... X-PMRQC: afifth@example.com + ... Subject: a message to you + ... + ... How are you doing? + ... """) + >>> handler.process(mlist, msg, {}) + >>> print msg.as_string() + From: bperson@example.com + Reply-To: bperson@example.org + Sender: asystem@example.net + Subject: a message to you + + How are you doing? + + + +Anonymous lists +=============== + +Anonymous mailing lists also try to cleanse certain identifying headers from +the original posting, so that it is at least a bit more difficult to determine +who sent the message. This isn't perfect though, for example, the body of the +messages are never scrubbed (though that might not be a bad idea). The +``From`` and ``Reply-To`` headers in the posted message are taken from list +attributes. + +Hotmail apparently sets ``X-Originating-Email``. + + >>> mlist.anonymous_list = True + >>> mlist.description = 'A Test Mailing List' + >>> mlist.preferred_language = 'en' + >>> msg = message_from_string("""\ + ... From: bperson@example.com + ... Reply-To: bperson@example.org + ... Sender: asystem@example.net + ... X-Originating-Email: cperson@example.com + ... Subject: a message to you + ... + ... How are you doing? + ... """) + >>> handler.process(mlist, msg, {}) + >>> print msg.as_string() + Subject: a message to you + From: A Test Mailing List <_xtest@example.com> + Reply-To: _xtest@example.com + + How are you doing? + diff --git a/src/mailman/handlers/docs/cook-headers.rst b/src/mailman/handlers/docs/cook-headers.rst new file mode 100644 index 000000000..948628d54 --- /dev/null +++ b/src/mailman/handlers/docs/cook-headers.rst @@ -0,0 +1,129 @@ +=============== +Cooking headers +=============== + +Messages that flow through the global pipeline get their headers 'cooked', +which basically means that their headers go through several mostly unrelated +transformations. Some headers get added, others get changed. Some of these +changes depend on mailing list settings and others depend on how the message +is getting sent through the system. We'll take things one-by-one. + + >>> mlist = create_list('test@example.com') + >>> mlist.subject_prefix = '' + + +Saving the original sender +========================== + +Because the original sender headers may get deleted or changed, this handler +will place the sender in the message metadata for safe keeping. +:: + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... + ... A message of great import. + ... """) + >>> msgdata = {} + + >>> from mailman.handlers.cook_headers import process + >>> process(mlist, msg, msgdata) + >>> print msgdata['original_sender'] + aperson@example.com + +But if there was no original sender, then the empty string will be saved. + + >>> msg = message_from_string("""\ + ... Subject: No original sender + ... + ... A message of great import. + ... """) + >>> msgdata = {} + >>> process(mlist, msg, msgdata) + >>> print msgdata['original_sender'] + + + +Mailman version header +====================== + +Mailman will also insert an ``X-Mailman-Version`` header... + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... + ... A message of great import. + ... """) + >>> process(mlist, msg, {}) + >>> from mailman.version import VERSION + >>> msg['x-mailman-version'] == VERSION + True + +...but only if one doesn't already exist. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... X-Mailman-Version: 3000 + ... + ... A message of great import. + ... """) + >>> process(mlist, msg, {}) + >>> print msg['x-mailman-version'] + 3000 + + +Precedence header +================= + +Mailman will insert a ``Precedence`` header, which is a de-facto standard for +telling automatic reply software (e.g. ``vacation(1)``) not to respond to this +message. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... + ... A message of great import. + ... """) + >>> process(mlist, msg, {}) + >>> print msg['precedence'] + list + +But Mailman will only add that header if the original message doesn't already +have one of them. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... Precedence: junk + ... + ... A message of great import. + ... """) + >>> process(mlist, msg, {}) + >>> print msg['precedence'] + junk + + +Personalization +=============== + +The ``To`` field normally contains the list posting address. However when +messages are fully personalized, that header will get overwritten with the +address of the recipient. The list's posting address will be added to one of +the recipient headers so that users will be able to reply back to the list. + + >>> from mailman.interfaces.mailinglist import ( + ... Personalization, ReplyToMunging) + >>> mlist.personalize = Personalization.full + >>> mlist.reply_goes_to_list = ReplyToMunging.no_munging + >>> mlist.description = 'My test mailing list' + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... + ... """) + >>> process(mlist, msg, {}) + >>> print msg.as_string() + From: aperson@example.com + X-Mailman-Version: ... + Precedence: list + Cc: My test mailing list + + diff --git a/src/mailman/handlers/docs/decorate.rst b/src/mailman/handlers/docs/decorate.rst new file mode 100644 index 000000000..eae8ea904 --- /dev/null +++ b/src/mailman/handlers/docs/decorate.rst @@ -0,0 +1,348 @@ +================== +Message decoration +================== + +Message decoration is the process of adding headers and footers to the +original message. A handler module takes care of this based on the settings +of the mailing list and the type of message being processed. + + >>> mlist = create_list('_xtest@example.com') + >>> msg_text = """\ + ... From: aperson@example.org + ... + ... Here is a message. + ... """ + >>> msg = message_from_string(msg_text) + + +Short circuiting +================ + +Digest messages get decorated during the digest creation phase so no extra +decorations are added for digest messages. +:: + + >>> from mailman.handlers.decorate import process + >>> process(mlist, msg, dict(isdigest=True)) + >>> print msg.as_string() + From: aperson@example.org + + Here is a message. + + >>> process(mlist, msg, dict(nodecorate=True)) + >>> print msg.as_string() + From: aperson@example.org + + Here is a message. + + +Simple decorations +================== + +Message decorations are specified by URI and can be specialized by the mailing +list and language. Internal Mailman decorations can be referenced by using +the ``mailman://`` URL scheme. Here we create a simple English header and +footer for all mailing lists in our site. +:: + + >>> import os, tempfile + >>> template_dir = tempfile.mkdtemp() + >>> site_dir = os.path.join(template_dir, 'site', 'en') + >>> os.makedirs(site_dir) + >>> config.push('templates', """ + ... [paths.testing] + ... template_dir: {0} + ... """.format(template_dir)) + + >>> myheader_path = os.path.join(site_dir, 'myheader.txt') + >>> with open(myheader_path, 'w') as fp: + ... print >> fp, 'header' + >>> myfooter_path = os.path.join(site_dir, 'myfooter.txt') + >>> with open(myfooter_path, 'w') as fp: + ... print >> fp, 'footer' + +Setting these attributes on the mailing list causes it to use these +templates. Since these are site-global templates, we can use a shorter path. + + >>> mlist.header_uri = 'mailman:///myheader.txt' + >>> mlist.footer_uri = 'mailman:///myfooter.txt' + +Text messages that have no declared content type are, by default encoded in +ASCII. When the mailing list's preferred language is ``en`` (i.e. English), +the character set of the mailing list and of the message will match, allowing +Mailman to simply prepend the header and append the footer verbatim. + + >>> mlist.preferred_language = 'en' + >>> process(mlist, msg, {}) + >>> print msg.as_string() + From: aperson@example.org + ... + + header + Here is a message. + footer + +Mailman supports a number of interpolation variables, placeholders in the +header and footer for information to be filled in with mailing list specific +data. An example of such information is the mailing list's `real name` (a +short descriptive name for the mailing list). +:: + + >>> with open(myheader_path, 'w') as fp: + ... print >> fp, '$display_name header' + >>> with open(myfooter_path, 'w') as fp: + ... print >> fp, '$display_name footer' + + >>> msg = message_from_string(msg_text) + >>> mlist.display_name = 'XTest' + >>> process(mlist, msg, {}) + >>> print msg.as_string() + From: aperson@example.org + ... + XTest header + Here is a message. + XTest footer + +You can't just pick any interpolation variable though; if you do, the variable +will remain in the header or footer unchanged. +:: + + >>> with open(myheader_path, 'w') as fp: + ... print >> fp, '$dummy header' + >>> with open(myfooter_path, 'w') as fp: + ... print >> fp, '$dummy footer' + + >>> msg = message_from_string(msg_text) + >>> process(mlist, msg, {}) + >>> print msg.as_string() + From: aperson@example.org + ... + $dummy header + Here is a message. + $dummy footer + + +Handling RFC 3676 'format=flowed' parameters +============================================ + +RFC 3676 describes a standard by which text/plain messages can marked by +generating MUAs for better readability in compatible receiving MUAs. The +``format`` parameter on the text/plain ``Content-Type`` header gives hints as +to how the receiving MUA may flow and delete trailing whitespace for better +display in a proportional font. + +When Mailman sees text/plain messages with such RFC 3676 parameters, it +preserves these parameters when it concatenates headers and footers to the +message payload. +:: + + >>> with open(myheader_path, 'w') as fp: + ... print >> fp, 'header' + >>> with open(myfooter_path, 'w') as fp: + ... print >> fp, 'footer' + + >>> mlist.preferred_language = 'en' + >>> msg = message_from_string("""\ + ... From: aperson@example.org + ... Content-Type: text/plain; format=flowed; delsp=no + ... + ... Here is a message\x20 + ... with soft line breaks. + ... """) + >>> process(mlist, msg, {}) + >>> # Don't use 'print' here as above because it won't be obvious from the + >>> # output that the soft-line break space at the end of the 'Here is a + >>> # message' line will be retained in the output. + >>> print msg['content-type'] + text/plain; format="flowed"; delsp="no"; charset="us-ascii" + >>> for line in msg.get_payload().splitlines(): + ... print '>{0}<'.format(line) + >header< + >Here is a message < + >with soft line breaks.< + >footer< + + +Decorating mixed-charset messages +================================= + +When a message has no explicit character set, it is assumed to be ASCII. +However, if the mailing list's preferred language has a different character +set, Mailman will still try to concatenate the header and footer, but it will +convert the text to utf-8 and base-64 encode the message payload. +:: + + # 'ja' = Japanese; charset = 'euc-jp' + >>> mlist.preferred_language = 'ja' + + >>> with open(myheader_path, 'w') as fp: + ... print >> fp, '$description header' + >>> with open(myfooter_path, 'w') as fp: + ... print >> fp, '$description footer' + >>> mlist.description = '\u65e5\u672c\u8a9e' + + >>> from email.message import Message + >>> msg = Message() + >>> msg.set_payload('Fran\xe7aise', 'iso-8859-1') + >>> print msg.as_string() + MIME-Version: 1.0 + Content-Type: text/plain; charset="iso-8859-1" + Content-Transfer-Encoding: quoted-printable + + Fran=E7aise + >>> process(mlist, msg, {}) + >>> print msg.as_string() + MIME-Version: 1.0 + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: base64 + + 5pel5pys6KqeIGhlYWRlcgpGcmFuw6dhaXNlCuaXpeacrOiqniBmb290ZXIK + +Sometimes the message even has an unknown character set. In this case, +Mailman has no choice but to decorate the original message with MIME +attachments. +:: + + >>> mlist.preferred_language = 'en' + >>> with open(myheader_path, 'w') as fp: + ... print >> fp, 'header' + >>> with open(myfooter_path, 'w') as fp: + ... print >> fp, 'footer' + + >>> msg = message_from_string("""\ + ... From: aperson@example.org + ... Content-Type: text/plain; charset=unknown + ... Content-Transfer-Encoding: 7bit + ... + ... Here is a message. + ... """) + + >>> process(mlist, msg, {}) + >>> msg.set_boundary('BOUNDARY') + >>> print msg.as_string() + From: aperson@example.org + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: inline + + header + --BOUNDARY + Content-Type: text/plain; charset=unknown + Content-Transfer-Encoding: 7bit + + Here is a message. + + --BOUNDARY + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: inline + + footer + --BOUNDARY-- + + +Decorating multipart messages +============================= + +Multipart messages have to be decorated differently. The header and footer +cannot be simply concatenated into the payload because that will break the +MIME structure of the message. Instead, the header and footer are attached as +separate MIME subparts. + +When the outer part is ``multipart/mixed``, the header and footer can have a +``Content-Disposition`` of ``inline`` so that MUAs can display these headers +as if they were simply concatenated. + + >>> part_1 = message_from_string("""\ + ... From: aperson@example.org + ... + ... Here is the first message. + ... """) + >>> part_2 = message_from_string("""\ + ... From: bperson@example.com + ... + ... Here is the second message. + ... """) + >>> from email.mime.multipart import MIMEMultipart + >>> msg = MIMEMultipart('mixed', boundary='BOUNDARY', + ... _subparts=(part_1, part_2)) + >>> process(mlist, msg, {}) + >>> print msg.as_string() + Content-Type: multipart/mixed; boundary="BOUNDARY" + MIME-Version: 1.0 + + --BOUNDARY + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: inline + + header + --BOUNDARY + From: aperson@example.org + + Here is the first message. + + --BOUNDARY + From: bperson@example.com + + Here is the second message. + + --BOUNDARY + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: inline + + footer + --BOUNDARY-- + + +Decorating other content types +============================== + +Non-multipart non-text content types will get wrapped in a ``multipart/mixed`` +so that the header and footer can be added as attachments. + + >>> msg = message_from_string("""\ + ... From: aperson@example.org + ... Content-Type: image/x-beautiful + ... + ... IMAGEDATAIMAGEDATAIMAGEDATA + ... """) + >>> process(mlist, msg, {}) + >>> msg.set_boundary('BOUNDARY') + >>> print msg.as_string() + From: aperson@example.org + ... + --BOUNDARY + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: inline + + header + --BOUNDARY + Content-Type: image/x-beautiful + + IMAGEDATAIMAGEDATAIMAGEDATA + + --BOUNDARY + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Disposition: inline + + footer + --BOUNDARY-- + +.. Clean up + + >>> config.pop('templates') + >>> import shutil + >>> shutil.rmtree(template_dir) diff --git a/src/mailman/handlers/docs/digests.rst b/src/mailman/handlers/docs/digests.rst new file mode 100644 index 000000000..d4d563180 --- /dev/null +++ b/src/mailman/handlers/docs/digests.rst @@ -0,0 +1,113 @@ +======= +Digests +======= + +Digests are a way for a user to receive list traffic in collections instead of +as individual messages when immediately posted. There are several forms of +digests, although only two are currently supported: MIME digests and RFC 1153 +(a.k.a. plain text) digests. + + >>> mlist = create_list('xtest@example.com') + +This is a helper function used to iterate through all the accumulated digest +messages, in the order in which they were posted. This makes it easier to +update the tests when we switch to a different mailbox format. +:: + + >>> from mailman.testing.helpers import digest_mbox + >>> from itertools import count + >>> from string import Template + + >>> def message_factory(): + ... for i in count(1): + ... text = Template("""\ + ... From: aperson@example.com + ... To: xtest@example.com + ... Subject: Test message $i + ... + ... Here is message $i + ... """).substitute(i=i) + ... yield message_from_string(text) + >>> message_factory = message_factory() + + +Short circuiting +================ + +When a message is posted to the mailing list, it is generally added to a +mailbox, unless the mailing list does not allow digests. + + >>> mlist.digestable = False + >>> msg = next(message_factory) + >>> process = config.handlers['to-digest'].process + >>> process(mlist, msg, {}) + >>> sum(1 for msg in digest_mbox(mlist)) + 0 + >>> digest_queue = config.switchboards['digest'] + >>> digest_queue.files + [] + +...or they may allow digests but the message is already a digest. + + >>> mlist.digestable = True + >>> process(mlist, msg, dict(isdigest=True)) + >>> sum(1 for msg in digest_mbox(mlist)) + 0 + >>> digest_queue.files + [] + + +Sending a digest +================ + +For messages which are not digests, but which are posted to a digesting +mailing list, the messages will be stored until they reach a criteria +triggering the sending of the digest. If none of those criteria are met, then +the message will just sit in the mailbox for a while. + + >>> mlist.digest_size_threshold = 10000 + >>> process(mlist, msg, {}) + >>> digest_queue.files + [] + >>> digest = digest_mbox(mlist) + >>> sum(1 for msg in digest) + 1 + >>> import os + >>> os.remove(digest._path) + +When the size of the digest mailbox reaches the maximum size threshold, a +marker message is placed into the digest runner's queue. The digest is not +actually crafted by the handler. + + >>> mlist.digest_size_threshold = 1 + >>> mlist.volume = 2 + >>> mlist.next_digest_number = 10 + >>> size = 0 + >>> for msg in message_factory: + ... process(mlist, msg, {}) + ... size += len(str(msg)) + ... if size >= mlist.digest_size_threshold * 1024: + ... break + + >>> sum(1 for msg in digest_mbox(mlist)) + 0 + >>> len(digest_queue.files) + 1 + +The digest has been moved to a unique file. + + >>> from mailman.utilities.mailbox import Mailbox + >>> from mailman.testing.helpers import get_queue_messages + >>> item = get_queue_messages('digest')[0] + >>> for msg in Mailbox(item.msgdata['digest_path']): + ... print msg['subject'] + Test message 2 + Test message 3 + Test message 4 + Test message 5 + Test message 6 + Test message 7 + Test message 8 + Test message 9 + +Digests are actually crafted and sent by a separate digest runner. diff --git a/src/mailman/handlers/docs/file-recips.rst b/src/mailman/handlers/docs/file-recips.rst new file mode 100644 index 000000000..7d157ccc5 --- /dev/null +++ b/src/mailman/handlers/docs/file-recips.rst @@ -0,0 +1,111 @@ +=============== +File recipients +=============== + +Mailman can calculate the recipients for a message from a Sendmail-style +include file. This file must be called ``members.txt`` and it must live in +the list's data directory. + + >>> mlist = create_list('_xtest@example.com') + + +Short circuiting +================ + +If the message's metadata already has recipients, this handler immediately +returns. +:: + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... + ... A message. + ... """) + >>> msgdata = {'recipients': 7} + + >>> handler = config.handlers['file-recipients'] + >>> handler.process(mlist, msg, msgdata) + >>> print msg.as_string() + From: aperson@example.com + + A message. + + >>> dump_msgdata(msgdata) + recipients: 7 + + +Missing file +============ + +The include file must live inside the list's data directory, under the name +``members.txt``. If the file doesn't exist, the list of recipients will be +empty. + + >>> import os + >>> file_path = os.path.join(mlist.data_path, 'members.txt') + >>> open(file_path) + Traceback (most recent call last): + ... + IOError: [Errno ...] + No such file or directory: u'.../_xtest@example.com/members.txt' + >>> msgdata = {} + >>> handler.process(mlist, msg, msgdata) + >>> dump_list(msgdata['recipients']) + *Empty* + + +Existing file +============= + +If the file exists, it contains a list of addresses, one per line. These +addresses are returned as the set of recipients. +:: + + >>> fp = open(file_path, 'w') + >>> try: + ... print >> fp, 'bperson@example.com' + ... print >> fp, 'cperson@example.com' + ... print >> fp, 'dperson@example.com' + ... print >> fp, 'eperson@example.com' + ... print >> fp, 'fperson@example.com' + ... print >> fp, 'gperson@example.com' + ... finally: + ... fp.close() + + >>> msgdata = {} + >>> handler.process(mlist, msg, msgdata) + >>> dump_list(msgdata['recipients']) + bperson@example.com + cperson@example.com + dperson@example.com + eperson@example.com + fperson@example.com + gperson@example.com + +However, if the sender of the original message is a member of the list and +their address is in the include file, the sender's address is *not* included +in the recipients list. +:: + + >>> from mailman.interfaces.usermanager import IUserManager + >>> from zope.component import getUtility + >>> address_1 = getUtility(IUserManager).create_address( + ... 'cperson@example.com') + + >>> from mailman.interfaces.member import MemberRole + >>> mlist.subscribe(address_1, MemberRole.member) + + + >>> msg = message_from_string("""\ + ... From: cperson@example.com + ... + ... A message. + ... """) + >>> msgdata = {} + >>> handler.process(mlist, msg, msgdata) + >>> dump_list(msgdata['recipients']) + bperson@example.com + dperson@example.com + eperson@example.com + fperson@example.com + gperson@example.com diff --git a/src/mailman/handlers/docs/filtering.rst b/src/mailman/handlers/docs/filtering.rst new file mode 100644 index 000000000..fd0b33d3b --- /dev/null +++ b/src/mailman/handlers/docs/filtering.rst @@ -0,0 +1,347 @@ +================= +Content filtering +================= + +Mailman can filter the content of messages posted to a mailing list by +stripping MIME subparts, and possibly reorganizing the MIME structure of a +message. + + >>> mlist = create_list('test@example.com') + +Several mailing list options control content filtering. First, the feature +must be enabled, then there are two options that control which MIME types get +filtered and which get passed. Finally, there is an option to control whether +``text/html`` parts will get converted to plain text. Let's set up some +defaults for these variables, then we'll explain them in more detail below. + + >>> mlist.filter_content = True + >>> mlist.filter_types = [] + >>> mlist.pass_types = [] + >>> mlist.convert_html_to_plaintext = False + + +Filtering the outer content type +================================ + +A simple filtering setting will just search the content types of the messages +parts, discarding all parts with a matching MIME type. If the message's outer +content type matches the filter, the entire message will be discarded. +:: + + >>> from mailman.interfaces.mime import FilterAction + + >>> mlist.filter_types = ['image/jpeg'] + >>> mlist.filter_action = FilterAction.discard + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... Content-Type: image/jpeg + ... MIME-Version: 1.0 + ... + ... xxxxx + ... """) + + >>> process = config.handlers['mime-delete'].process + >>> process(mlist, msg, {}) + Traceback (most recent call last): + ... + DiscardMessage: The message's content type was explicitly disallowed + +However, if we turn off content filtering altogether, then the handler +short-circuits. + + >>> mlist.filter_content = False + >>> msgdata = {} + >>> process(mlist, msg, msgdata) + >>> print msg.as_string() + From: aperson@example.com + Content-Type: image/jpeg + MIME-Version: 1.0 + + xxxxx + >>> msgdata + {} + +Similarly, no content filtering is performed on digest messages, which are +crafted internally by Mailman. + + >>> mlist.filter_content = True + >>> msgdata = {'isdigest': True} + >>> process(mlist, msg, msgdata) + >>> print msg.as_string() + From: aperson@example.com + Content-Type: image/jpeg + MIME-Version: 1.0 + + xxxxx + >>> msgdata + {u'isdigest': True} + + +Simple multipart filtering +========================== + +If one of the subparts in a multipart message matches the filter type, then +just that subpart will be stripped. +:: + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... Content-Type: multipart/mixed; boundary=BOUNDARY + ... MIME-Version: 1.0 + ... + ... --BOUNDARY + ... Content-Type: image/jpeg + ... MIME-Version: 1.0 + ... + ... xxx + ... + ... --BOUNDARY + ... Content-Type: image/gif + ... MIME-Version: 1.0 + ... + ... yyy + ... --BOUNDARY-- + ... """) + + >>> process(mlist, msg, {}) + >>> print msg.as_string() + From: aperson@example.com + Content-Type: multipart/mixed; boundary=BOUNDARY + MIME-Version: 1.0 + X-Content-Filtered-By: Mailman/MimeDel ... + + --BOUNDARY + Content-Type: image/gif + MIME-Version: 1.0 + + yyy + --BOUNDARY-- + + + +Collapsing multipart/alternative messages +========================================= + +When content filtering encounters a ``multipart/alternative`` part, and the +results of filtering leave only one of the subparts, then the +``multipart/alternative`` may be collapsed. For example, in the following +message, the outer content type is a ``multipart/mixed``. Inside this part is +just a single subpart that has a content type of ``multipart/alternative``. +This inner multipart has two subparts, a jpeg and a gif. + +Content filtering will remove the jpeg part, leaving the +``multipart/alternative`` with only a single gif subpart. Because there's +only one subpart left, the MIME structure of the message will be reorganized, +removing the inner ``multipart/alternative`` so that the outer +``multipart/mixed`` has just a single gif subpart. + + >>> mlist.collapse_alternatives = True + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... Content-Type: multipart/mixed; boundary=BOUNDARY + ... MIME-Version: 1.0 + ... + ... --BOUNDARY + ... Content-Type: multipart/alternative; boundary=BOUND2 + ... MIME-Version: 1.0 + ... + ... --BOUND2 + ... Content-Type: image/jpeg + ... MIME-Version: 1.0 + ... + ... xxx + ... + ... --BOUND2 + ... Content-Type: image/gif + ... MIME-Version: 1.0 + ... + ... yyy + ... --BOUND2-- + ... + ... --BOUNDARY-- + ... """) + >>> process(mlist, msg, {}) + >>> print msg.as_string() + From: aperson@example.com + Content-Type: multipart/mixed; boundary=BOUNDARY + MIME-Version: 1.0 + X-Content-Filtered-By: Mailman/MimeDel ... + + --BOUNDARY + Content-Type: image/gif + MIME-Version: 1.0 + + yyy + --BOUNDARY-- + + +When the outer part is a ``multipart/alternative`` and filtering leaves this +outer part with just one subpart, the entire message is converted to the left +over part's content type. In other words, the left over inner part is +promoted to being the outer part. +:: + + >>> mlist.filter_types = ['image/jpeg', 'text/html'] + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... Content-Type: multipart/alternative; boundary=AAA + ... + ... --AAA + ... Content-Type: text/html + ... + ... This is some html + ... --AAA + ... Content-Type: text/plain + ... + ... This is plain text + ... --AAA-- + ... """) + + >>> process(mlist, msg, {}) + >>> print msg.as_string() + From: aperson@example.com + Content-Type: text/plain + X-Content-Filtered-By: Mailman/MimeDel ... + + This is plain text + +Clean up. + + >>> mlist.filter_types = ['image/jpeg'] + + +Conversion to plain text +======================== + +Many mailing lists prohibit HTML email, and in fact, such email can be a +phishing or spam vector. However, many mail readers will send HTML email by +default because users think it looks pretty. One approach to handling this +would be to filter out ``text/html`` parts and rely on +``multipart/alternative`` collapsing to leave just a plain text part. This +works because many mail readers that send HTML email actually send a plain +text part in the second subpart of such ``multipart/alternatives``. + +While this is a good suggestion for plain text-only mailing lists, often a +mail reader will send only a ``text/html`` part with no plain text +alternative. in this case, the site administer can enable ``text/html`` to +``text/plain`` conversion by defining a conversion command. A list +administrator still needs to enable such conversion for their list though. + + >>> mlist.convert_html_to_plaintext = True + +By default, Mailman sends the message through lynx, but since this program is +not guaranteed to exist, we'll craft a simple, but stupid script to simulate +the conversion process. The script expects a single argument, which is the +name of the file containing the message payload to filter. + + >>> import os, sys + >>> script_path = os.path.join(config.DATA_DIR, 'filter.py') + >>> fp = open(script_path, 'w') + >>> try: + ... print >> fp, """\ + ... import sys + ... print 'Converted text/html to text/plain' + ... print 'Filename:', sys.argv[1] + ... """ + ... finally: + ... fp.close() + >>> config.HTML_TO_PLAIN_TEXT_COMMAND = '%s %s %%(filename)s' % ( + ... sys.executable, script_path) + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... Content-Type: text/html + ... MIME-Version: 1.0 + ... + ... + ... + ... """) + >>> process(mlist, msg, {}) + >>> print msg.as_string() + From: aperson@example.com + MIME-Version: 1.0 + Content-Type: text/plain + X-Content-Filtered-By: Mailman/MimeDel ... + + Converted text/html to text/plain + Filename: ... + + + +Discarding empty parts +====================== + +Similarly, if after filtering a multipart section ends up empty, then the +entire multipart is discarded. For example, here's a message where an inner +``multipart/mixed`` contains two jpeg subparts. Both jpegs are filtered out, +so the entire inner ``multipart/mixed`` is discarded. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... Content-Type: multipart/mixed; boundary=AAA + ... + ... --AAA + ... Content-Type: multipart/mixed; boundary=BBB + ... + ... --BBB + ... Content-Type: image/jpeg + ... + ... xxx + ... --BBB + ... Content-Type: image/jpeg + ... + ... yyy + ... --BBB--- + ... --AAA + ... Content-Type: multipart/alternative; boundary=CCC + ... + ... --CCC + ... Content-Type: text/html + ... + ...

This is a header

+ ... + ... --CCC + ... Content-Type: text/plain + ... + ... A different message + ... --CCC-- + ... --AAA + ... Content-Type: image/gif + ... + ... zzz + ... --AAA + ... Content-Type: image/gif + ... + ... aaa + ... --AAA-- + ... """) + >>> process(mlist, msg, {}) + >>> print msg.as_string() + From: aperson@example.com + Content-Type: multipart/mixed; boundary=AAA + X-Content-Filtered-By: Mailman/MimeDel ... + + --AAA + MIME-Version: 1.0 + Content-Type: text/plain + + Converted text/html to text/plain + Filename: ... + + --AAA + Content-Type: image/gif + + zzz + --AAA + Content-Type: image/gif + + aaa + --AAA-- + + + +Passing MIME types +================== + +XXX Describe the pass_mime_types setting and how it interacts with +``filter_mime_types``. diff --git a/src/mailman/handlers/docs/nntp.rst b/src/mailman/handlers/docs/nntp.rst new file mode 100644 index 000000000..874712397 --- /dev/null +++ b/src/mailman/handlers/docs/nntp.rst @@ -0,0 +1,68 @@ +============ +NNTP Gateway +============ + +Mailman has an NNTP gateway, whereby messages posted to the mailing list can +be forwarded onto an NNTP newsgroup. Typically this means Usenet, but since +NNTP is to Usenet as IP is to the web, it's more general than that. + + >>> mlist = create_list('_xtest@example.com') + +Gatewaying from the mailing list to the newsgroup happens through a separate +``nntp`` queue and happen immediately when the message is posted through to +the list. Note that gatewaying from the newsgroup to the list happens via a +cronjob (currently not shown). + +There are several situations which prevent a message from being gatewayed to +the newsgroup. The feature could be disabled, as is the default. +:: + + >>> mlist.gateway_to_news = False + >>> msg = message_from_string("""\ + ... Subject: An important message + ... + ... Something of great import. + ... """) + + >>> handler = config.handlers['to-usenet'] + >>> handler.process(mlist, msg, {}) + + >>> switchboard = config.switchboards['news'] + >>> switchboard.files + [] + +Even if enabled, messages that came from the newsgroup are never gated back to +the newsgroup. + + >>> mlist.gateway_to_news = True + >>> handler.process(mlist, msg, {'fromusenet': True}) + >>> switchboard.files + [] + +Neither are digests ever gated to the newsgroup. + + >>> handler.process(mlist, msg, {'isdigest': True}) + >>> switchboard.files + [] + +However, other posted messages get gated to the newsgroup via the nntp queue. +The list owner can set the linked newsgroup and the nntp host that its +messages are gated to. + + >>> mlist.linked_newsgroup = 'comp.lang.thing' + >>> mlist.nntp_host = 'news.example.com' + >>> handler.process(mlist, msg, {}) + >>> len(switchboard.files) + 1 + >>> filebase = switchboard.files[0] + >>> msg, msgdata = switchboard.dequeue(filebase) + >>> switchboard.finish(filebase) + >>> print msg.as_string() + Subject: An important message + + Something of great import. + + >>> dump_msgdata(msgdata) + _parsemsg: False + listname : _xtest@example.com + version : 3 diff --git a/src/mailman/handlers/docs/reply-to.rst b/src/mailman/handlers/docs/reply-to.rst new file mode 100644 index 000000000..d421e2dc5 --- /dev/null +++ b/src/mailman/handlers/docs/reply-to.rst @@ -0,0 +1,131 @@ +================ +Reply-to munging +================ + +Messages that flow through the global pipeline get their headers *cooked*, +which basically means that their headers go through several mostly unrelated +transformations. Some headers get added, others get changed. Some of these +changes depend on mailing list settings and others depend on how the message +is getting sent through the system. We'll take things one-by-one. + + >>> mlist = create_list('_xtest@example.com') + +*Reply-to munging* refers to the behavior where a mailing list can be +configured to change or augment an existing ``Reply-To`` header in a message +posted to the list. Reply-to munging is fairly controversial, with arguments +made either for or against munging. + +The Mailman developers, and I believe the majority consensus is to do no +reply-to munging, under several principles. Primarily, most reply-to munging +is requested by people who do not have both a `Reply` and `Reply All` button +on their mail reader. If you do not munge ``Reply-To``, then these buttons +will work properly, but if you munge the header, it is impossible for these +buttons to work right, because both will reply to the list. This leads to +unfortunate accidents where a private message is accidentally posted to the +entire list. + +However, Mailman gives list owners the option to do reply-To munging anyway, +mostly as a way to shut up the really vocal minority who seem to insist on +this mis-feature. + + +Reply to list +============= + +A list can be configured to add a ``Reply-To`` header pointing back to the +mailing list's posting address. If there's no ``Reply-To`` header in the +original message, the list's posting address simply gets inserted. +:: + + >>> from mailman.interfaces.mailinglist import ReplyToMunging + >>> mlist.reply_goes_to_list = ReplyToMunging.point_to_list + >>> mlist.preferred_language = 'en' + >>> mlist.description = '' + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... + ... """) + + >>> from mailman.handlers.cook_headers import process + >>> process(mlist, msg, {}) + >>> len(msg.get_all('reply-to')) + 1 + >>> print msg['reply-to'] + _xtest@example.com + +It's also possible to strip any existing ``Reply-To`` header first, before +adding the list's posting address. + + >>> mlist.first_strip_reply_to = True + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... Reply-To: bperson@example.com + ... + ... """) + >>> process(mlist, msg, {}) + >>> len(msg.get_all('reply-to')) + 1 + >>> print msg['reply-to'] + _xtest@example.com + +If you don't first strip the header, then the list's posting address will just +get appended to whatever the original version was. + + >>> mlist.first_strip_reply_to = False + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... Reply-To: bperson@example.com + ... + ... """) + >>> process(mlist, msg, {}) + >>> len(msg.get_all('reply-to')) + 1 + >>> print msg['reply-to'] + bperson@example.com, _xtest@example.com + + +Explicit Reply-To +================= + +The list can also be configured to have an explicit ``Reply-To`` header. + + >>> mlist.reply_goes_to_list = ReplyToMunging.explicit_header + >>> mlist.reply_to_address = 'my-list@example.com' + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... + ... """) + >>> process(mlist, msg, {}) + >>> len(msg.get_all('reply-to')) + 1 + >>> print msg['reply-to'] + my-list@example.com + +And as before, it's possible to either strip any existing ``Reply-To`` +header... + + >>> mlist.first_strip_reply_to = True + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... Reply-To: bperson@example.com + ... + ... """) + >>> process(mlist, msg, {}) + >>> len(msg.get_all('reply-to')) + 1 + >>> print msg['reply-to'] + my-list@example.com + +...or not. + + >>> mlist.first_strip_reply_to = False + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... Reply-To: bperson@example.com + ... + ... """) + >>> process(mlist, msg, {}) + >>> len(msg.get_all('reply-to')) + 1 + >>> print msg['reply-to'] + my-list@example.com, bperson@example.com diff --git a/src/mailman/handlers/docs/replybot.rst b/src/mailman/handlers/docs/replybot.rst new file mode 100644 index 000000000..7cdd7c928 --- /dev/null +++ b/src/mailman/handlers/docs/replybot.rst @@ -0,0 +1,343 @@ +========================== +Automatic response handler +========================== + +Mailman has a autoreply handler that sends automatic responses to messages it +receives on its posting address, owner address, or robot address. Automatic +responses are subject to various conditions, such as headers in the original +message or the amount of time since the last auto-response. + + >>> mlist = create_list('_xtest@example.com') + >>> mlist.display_name = 'XTest' + + +Basic automatic responding +========================== + +Basic automatic responding occurs when the list is set up to respond to either +its ``-owner`` address, its ``-request`` address, or to the posting address, +and a message is sent to one of these addresses. A mailing list also has an +automatic response grace period which specifies how much time must pass before +a second response will be sent, with 0 meaning "there is no grace period". +:: + + >>> import datetime + >>> from mailman.interfaces.autorespond import ResponseAction + + >>> mlist.autorespond_owner = ResponseAction.respond_and_continue + >>> mlist.autoresponse_grace_period = datetime.timedelta() + >>> mlist.autoresponse_owner_text = 'owner autoresponse text' + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest-owner@example.com + ... + ... help + ... """) + +The preceding message to the mailing list's owner will trigger an automatic +response. +:: + + >>> from mailman.testing.helpers import get_queue_messages + + >>> handler = config.handlers['replybot'] + >>> handler.process(mlist, msg, dict(to_owner=True)) + >>> messages = get_queue_messages('virgin') + >>> len(messages) + 1 + + >>> dump_msgdata(messages[0].msgdata) + _parsemsg : False + listname : _xtest@example.com + nodecorate : True + recipients : set([u'aperson@example.com']) + reduced_list_headers: True + version : 3 + + >>> print messages[0].msg.as_string() + MIME-Version: 1.0 + Content-Type: text/plain; charset="us-ascii" + Content-Transfer-Encoding: 7bit + Subject: Auto-response for your message to the "XTest" mailing list + From: _xtest-bounces@example.com + To: aperson@example.com + X-Mailer: The Mailman Replybot + X-Ack: No + Message-ID: <...> + Date: ... + Precedence: bulk + + owner autoresponse text + + +Short circuiting +================ + +Several headers in the original message determine whether an automatic +response should even be sent. For example, if the message has an +``X-Ack: No`` header, no auto-response is sent. +:: + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... X-Ack: No + ... + ... help me + ... """) + + >>> handler.process(mlist, msg, dict(to_owner=True)) + >>> get_queue_messages('virgin') + [] + +Mailman itself can suppress automatic responses for certain types of +internally crafted messages, by setting the ``noack`` metadata key. +:: + + >>> msg = message_from_string("""\ + ... From: mailman@example.com + ... + ... help for you + ... """) + + >>> handler.process(mlist, msg, dict(noack=True, to_owner=True)) + >>> get_queue_messages('virgin') + [] + +If there is a ``Precedence:`` header with any of the values ``bulk``, +``junk``, or ``list``, then the automatic response is also suppressed. +:: + + >>> msg = message_from_string("""\ + ... From: asystem@example.com + ... Precedence: bulk + ... + ... hey! + ... """) + + >>> handler.process(mlist, msg, dict(to_owner=True)) + >>> get_queue_messages('virgin') + [] + + >>> msg.replace_header('precedence', 'junk') + >>> handler.process(mlist, msg, dict(to_owner=True)) + >>> get_queue_messages('virgin') + [] + + >>> msg.replace_header('precedence', 'list') + >>> handler.process(mlist, msg, dict(to_owner=True)) + >>> get_queue_messages('virgin') + [] + +Unless the ``X-Ack:`` header has a value of ``yes``, in which case, the +``Precedence`` header is ignored. +:: + + >>> msg['X-Ack'] = 'yes' + >>> handler.process(mlist, msg, dict(to_owner=True)) + >>> messages = get_queue_messages('virgin') + >>> len(messages) + 1 + + >>> dump_msgdata(messages[0].msgdata) + _parsemsg : False + listname : _xtest@example.com + nodecorate : True + recipients : set([u'asystem@example.com']) + reduced_list_headers: True + version : 3 + + >>> print messages[0].msg.as_string() + MIME-Version: 1.0 + Content-Type: text/plain; charset="us-ascii" + Content-Transfer-Encoding: 7bit + Subject: Auto-response for your message to the "XTest" mailing list + From: _xtest-bounces@example.com + To: asystem@example.com + X-Mailer: The Mailman Replybot + X-Ack: No + Message-ID: <...> + Date: ... + Precedence: bulk + + owner autoresponse text + + +Available auto-responses +======================== + +As shown above, a message sent to the ``-owner`` address will get an +auto-response with the text set for owner responses. Two other types of email +will get auto-responses: those sent to the ``-request`` address... +:: + + >>> mlist.autorespond_requests = ResponseAction.respond_and_continue + >>> mlist.autoresponse_request_text = 'robot autoresponse text' + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest-request@example.com + ... + ... help me + ... """) + + >>> handler.process(mlist, msg, dict(to_request=True)) + >>> messages = get_queue_messages('virgin') + >>> len(messages) + 1 + + >>> print messages[0].msg.as_string() + MIME-Version: 1.0 + Content-Type: text/plain; charset="us-ascii" + Content-Transfer-Encoding: 7bit + Subject: Auto-response for your message to the "XTest" mailing list + From: _xtest-bounces@example.com + To: aperson@example.com + X-Mailer: The Mailman Replybot + X-Ack: No + Message-ID: <...> + Date: ... + Precedence: bulk + + robot autoresponse text + +...and those sent to the posting address. +:: + + >>> mlist.autorespond_postings = ResponseAction.respond_and_continue + >>> mlist.autoresponse_postings_text = 'postings autoresponse text' + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... + ... help me + ... """) + + >>> handler.process(mlist, msg, dict(to_list=True)) + >>> messages = get_queue_messages('virgin') + >>> len(messages) + 1 + + >>> print messages[0].msg.as_string() + MIME-Version: 1.0 + Content-Type: text/plain; charset="us-ascii" + Content-Transfer-Encoding: 7bit + Subject: Auto-response for your message to the "XTest" mailing list + From: _xtest-bounces@example.com + To: aperson@example.com + X-Mailer: The Mailman Replybot + X-Ack: No + Message-ID: <...> + Date: ... + Precedence: bulk + + postings autoresponse text + + +Grace periods +============= + +Automatic responses have a grace period, during which no additional responses +will be sent. This is so as not to bombard the sender with responses. The +grace period is measured in days. + + >>> mlist.autoresponse_grace_period = datetime.timedelta(days=10) + +When a response is sent to a person via any of the owner, request, or postings +addresses, the response date is recorded. The grace period is usually +measured in days. + + >>> msg = message_from_string("""\ + ... From: bperson@example.com + ... To: _xtest-owner@example.com + ... + ... help + ... """) + +This is the first response to bperson, so it gets sent. + + >>> handler.process(mlist, msg, dict(to_owner=True)) + >>> print len(get_queue_messages('virgin')) + 1 + +But with a grace period greater than zero, no subsequent response will be sent +right now. + + >>> handler.process(mlist, msg, dict(to_owner=True)) + >>> print len(get_queue_messages('virgin')) + 0 + +Fast forward 9 days and you still don't get a response. +:: + + >>> from mailman.utilities.datetime import factory + >>> factory.fast_forward(days=9) + + >>> handler.process(mlist, msg, dict(to_owner=True)) + >>> print len(get_queue_messages('virgin')) + 0 + +But tomorrow, the sender will get a new auto-response. + + >>> factory.fast_forward() + >>> handler.process(mlist, msg, dict(to_owner=True)) + >>> print len(get_queue_messages('virgin')) + 1 + +Of course, everything works the same way for messages to the request +address, even if the sender is the same person... +:: + + >>> msg = message_from_string("""\ + ... From: bperson@example.com + ... To: _xtest-request@example.com + ... + ... help + ... """) + + >>> handler.process(mlist, msg, dict(to_request=True)) + >>> print len(get_queue_messages('virgin')) + 1 + + >>> handler.process(mlist, msg, dict(to_request=True)) + >>> print len(get_queue_messages('virgin')) + 0 + + >>> factory.fast_forward(days=9) + >>> handler.process(mlist, msg, dict(to_request=True)) + >>> print len(get_queue_messages('virgin')) + 0 + + >>> factory.fast_forward() + >>> handler.process(mlist, msg, dict(to_request=True)) + >>> print len(get_queue_messages('virgin')) + 1 + +...and for messages to the posting address. +:: + + >>> msg = message_from_string("""\ + ... From: bperson@example.com + ... To: _xtest@example.com + ... + ... help + ... """) + + >>> handler.process(mlist, msg, dict(to_list=True)) + >>> print len(get_queue_messages('virgin')) + 1 + + >>> handler.process(mlist, msg, dict(to_list=True)) + >>> print len(get_queue_messages('virgin')) + 0 + + >>> factory.fast_forward(days=9) + >>> handler.process(mlist, msg, dict(to_list=True)) + >>> print len(get_queue_messages('virgin')) + 0 + + >>> factory.fast_forward() + >>> handler.process(mlist, msg, dict(to_list=True)) + >>> print len(get_queue_messages('virgin')) + 1 diff --git a/src/mailman/handlers/docs/rfc-2369.rst b/src/mailman/handlers/docs/rfc-2369.rst new file mode 100644 index 000000000..0461f27ba --- /dev/null +++ b/src/mailman/handlers/docs/rfc-2369.rst @@ -0,0 +1,206 @@ +========================= +RFC 2919 and 2369 headers +========================= + +`RFC 2919`_ and `RFC 2369`_ define headers for mailing list actions. These +headers generally start with the `List-` prefix. + + >>> mlist = create_list('test@example.com') + >>> mlist.preferred_language = 'en' + >>> mlist.archive = False + +.. + This is a helper function for the following section. + >>> def list_headers(msg, only=None): + ... if isinstance(only, basestring): + ... only = (only.lower(),) + ... elif only is None: + ... only = set(header.lower() for header in msg.keys() + ... if header.lower().startswith('list-')) + ... only.add('archived-at') + ... else: + ... only = set(header.lower() for header in only) + ... print '---start---' + ... for header in sorted(only): + ... for value in sorted(msg.get_all(header, ())): + ... print '%s: %s' % (header, value) + ... print '---end---' + +The `rfc-2369` handler adds the `List-` headers. `List-Id` is always added. + + >>> from mailman.handlers.rfc_2369 import process + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... + ... """) + >>> process(mlist, msg, {}) + >>> list_headers(msg, 'list-id') + ---start--- + list-id: + ---end--- + + +Fewer headers +============= + +Some people don't like these headers because their mail readers aren't good +about hiding them. A list owner can turn these headers off. + + >>> mlist.include_rfc2369_headers = False + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... + ... """) + >>> process(mlist, msg, {}) + >>> list_headers(msg) + ---start--- + ---end--- + +Messages which Mailman generates itself, such as user or owner notifications, +have a reduced set of `List-` headers. Specifically, there is no `List-Post`, +`List-Archive` or `Archived-At` header. + + >>> mlist.include_rfc2369_headers = True + >>> mlist.include_list_post_header = False + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... + ... """) + >>> process(mlist, msg, dict(reduced_list_headers=True)) + >>> list_headers(msg) + ---start--- + list-help: + list-id: + list-subscribe: , + + list-unsubscribe: , + + ---end--- + + +List-Post header +================ + +Discussion lists, to which any subscriber can post, also have a `List-Post` +header which contains the `mailto:` URL used to send messages to the list. + + >>> mlist.include_list_post_header = True + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... + ... """) + >>> process(mlist, msg, {}) + >>> list_headers(msg) + ---start--- + list-help: + list-id: + list-post: + list-subscribe: , + + list-unsubscribe: , + + ---end--- + + +List-Id header +============== + +If the mailing list has a description, then it is included in the ``List-Id`` +header. + + >>> mlist.description = 'My test mailing list' + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... + ... """) + >>> process(mlist, msg, {}) + >>> list_headers(msg) + ---start--- + list-help: + list-id: My test mailing list + list-post: + list-subscribe: , + + list-unsubscribe: , + + ---end--- + +Any existing ``List-Id`` headers are removed from the original message. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... List-ID: <123.456.789> + ... + ... """) + + >>> process(mlist, msg, {}) + >>> list_headers(msg, only='list-id') + ---start--- + list-id: My test mailing list + ---end--- + + +Archive headers +=============== + +When the mailing list is configured to enable archiving, a `List-Archive` +header will be added. + + >>> mlist.archive = True + +`RFC 5064`_ defines the `Archived-At` header which contains the url to the +individual message in the archives. Archivers which don't support +pre-calculation of the archive url cannot add the `Archived-At` header. +However, other archivers can calculate the url, and do add this header. + + >>> config.push('prototype', """ + ... [archiver.prototype] + ... enable: yes + ... [archiver.mail_archive] + ... enable: no + ... [archiver.mhonarc] + ... enable: no + ... [archiver.pipermail] + ... enable: No + ... """) + +The *prototype* archiver can calculate this archive url given a `Message-ID`. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... Message-ID: + ... X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB + ... + ... """) + >>> process(mlist, msg, {}) + >>> list_headers(msg, only=('list-archive', 'archived-at')) + ---start--- + archived-at: http://lists.example.com/4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB + list-archive: + ---end--- + +If the mailing list isn't being archived, neither the `List-Archive` nor +`Archived-At` headers will be added. + + >>> config.pop('prototype') + >>> mlist.archive = False + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... + ... """) + >>> process(mlist, msg, {}) + >>> list_headers(msg) + ---start--- + list-help: + list-id: My test mailing list + list-post: + list-subscribe: , + + list-unsubscribe: , + + ---end--- + + +.. _`RFC 2919`: http://www.faqs.org/rfcs/rfc2919.html +.. _`RFC 2369`: http://www.faqs.org/rfcs/rfc2369.html +.. _`RFC 5064`: http://www.faqs.org/rfcs/rfc5064.html diff --git a/src/mailman/handlers/docs/subject-munging.rst b/src/mailman/handlers/docs/subject-munging.rst new file mode 100644 index 000000000..48cee8e2b --- /dev/null +++ b/src/mailman/handlers/docs/subject-munging.rst @@ -0,0 +1,249 @@ +=============== +Subject munging +=============== + +Messages that flow through the global pipeline get their headers *cooked*, +which basically means that their headers go through several mostly unrelated +transformations. Some headers get added, others get changed. Some of these +changes depend on mailing list settings and others depend on how the message +is getting sent through the system. We'll take things one-by-one. + + >>> mlist = create_list('_xtest@example.com') + + +Inserting a prefix +================== + +Another thing header cooking does is *munge* the ``Subject`` header by +inserting the subject prefix for the list at the front. If there's no subject +header in the original message, Mailman uses a canned default. In order to do +subject munging, a mailing list must have a preferred language. +:: + + >>> mlist.subject_prefix = '[XTest] ' + >>> mlist.preferred_language = 'en' + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... + ... A message of great import. + ... """) + >>> msgdata = {} + + >>> from mailman.handlers.cook_headers import process + >>> process(mlist, msg, msgdata) + +The original subject header is stored in the message metadata. We must print +the new ``Subject`` header because it gets converted from a string to an +``email.header.Header`` instance which has an unhelpful ``repr``. + + >>> msgdata['origsubj'] + u'' + >>> print msg['subject'] + [XTest] (no subject) + +If the original message had a ``Subject`` header, then the prefix is inserted +at the beginning of the header's value. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... Subject: Something important + ... + ... A message of great import. + ... """) + >>> msgdata = {} + >>> process(mlist, msg, msgdata) + >>> print msgdata['origsubj'] + Something important + >>> print msg['subject'] + [XTest] Something important + +``Subject`` headers are not munged for digest messages. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... Subject: Something important + ... + ... A message of great import. + ... """) + >>> process(mlist, msg, dict(isdigest=True)) + >>> print msg['subject'] + Something important + +Nor are they munged for *fast tracked* messages, which are generally defined +as messages that Mailman crafts internally. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... Subject: Something important + ... + ... A message of great import. + ... """) + >>> process(mlist, msg, dict(_fasttrack=True)) + >>> print msg['subject'] + Something important + +If a ``Subject`` header already has a prefix, usually following a ``Re:`` +marker, another one will not be added but the prefix will be moved to the +front of the header text. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... Subject: Re: [XTest] Something important + ... + ... A message of great import. + ... """) + >>> process(mlist, msg, {}) + >>> print msg['subject'] + [XTest] Re: Something important + +If the ``Subject`` header has a prefix at the front of the header text, that's +where it will stay. This is called *new style* prefixing and is the only +option available in Mailman 3. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... Subject: [XTest] Re: Something important + ... + ... A message of great import. + ... """) + >>> process(mlist, msg, {}) + >>> print msg['subject'] + [XTest] Re: Something important + + +Internationalized headers +========================= + +Internationalization adds some interesting twists to the handling of subject +prefixes. Part of what makes this interesting is the encoding of i18n headers +using RFC 2047, and lists whose preferred language is in a different character +set than the encoded header. + + >>> msg = message_from_string("""\ + ... Subject: =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= + ... + ... """) + >>> process(mlist, msg, {}) + >>> print msg['subject'] + [XTest] =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= + >>> unicode(msg['subject']) + u'[XTest] \u30e1\u30fc\u30eb\u30de\u30f3' + + +Prefix numbers +============== + +Subject prefixes support a placeholder for the numeric post id. Every time a +message is posted to the mailing list, a *post id* gets incremented. This is +a purely sequential integer that increases monotonically. By added a ``%d`` +placeholder to the subject prefix, this post id can be included in the prefix. + + >>> mlist.subject_prefix = '[XTest %d] ' + >>> mlist.post_id = 456 + >>> msg = message_from_string("""\ + ... Subject: Something important + ... + ... """) + >>> process(mlist, msg, {}) + >>> print msg['subject'] + [XTest 456] Something important + +This works even when the message is a reply, except that in this case, the +numeric post id in the generated subject prefix is updated with the new post +id. + + >>> msg = message_from_string("""\ + ... Subject: [XTest 123] Re: Something important + ... + ... """) + >>> process(mlist, msg, {}) + >>> print msg['subject'] + [XTest 456] Re: Something important + +If the ``Subject`` header had old style prefixing, the prefix is moved to the +front of the header text. + + >>> msg = message_from_string("""\ + ... Subject: Re: [XTest 123] Something important + ... + ... """) + >>> process(mlist, msg, {}) + >>> print msg['subject'] + [XTest 456] Re: Something important + + +And of course, the proper thing is done when posting id numbers are included +in the subject prefix, and the subject is encoded non-ASCII. + + >>> msg = message_from_string("""\ + ... Subject: =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= + ... + ... """) + >>> process(mlist, msg, {}) + >>> print msg['subject'] + [XTest 456] =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= + >>> unicode(msg['subject']) + u'[XTest 456] \u30e1\u30fc\u30eb\u30de\u30f3' + +Even more fun is when the internationalized ``Subject`` header already has a +prefix, possibly with a different posting number. + + >>> msg = message_from_string("""\ + ... Subject: [XTest 123] Re: =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= + ... + ... """) + >>> process(mlist, msg, {}) + >>> print msg['subject'] + [XTest 456] Re: =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= + +.. + # XXX This requires Python email patch #1681333 to succeed. + # >>> unicode(msg['subject']) + # u'[XTest 456] Re: \u30e1\u30fc\u30eb\u30de\u30f3' + +As before, old style subject prefixes are re-ordered. + + >>> msg = message_from_string("""\ + ... Subject: Re: [XTest 123] =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= + ... + ... """) + >>> process(mlist, msg, {}) + >>> print msg['subject'] + [XTest 456] Re: + =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= + +.. + # XXX This requires Python email patch #1681333 to succeed. + # >>> unicode(msg['subject']) + # u'[XTest 456] Re: \u30e1\u30fc\u30eb\u30de\u30f3' + + +In this test case, we get an extra space between the prefix and the original +subject. It's because the original is *crooked*. Note that a ``Subject`` +starting with '\n ' is generated by some version of Eudora Japanese edition. + + >>> mlist.subject_prefix = '[XTest] ' + >>> msg = message_from_string("""\ + ... Subject: + ... Important message + ... + ... """) + >>> process(mlist, msg, {}) + >>> print msg['subject'] + [XTest] Important message + +And again, with an RFC 2047 encoded header. + + >>> msg = message_from_string("""\ + ... Subject: + ... =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= + ... + ... """) + >>> process(mlist, msg, {}) + +.. + # XXX This one does not appear to work the same way as + # test_subject_munging_prefix_crooked() in the old Python-based tests. I need + # to get Tokio to look at this. + # >>> print msg['subject'] + # [XTest] =?iso-2022-jp?b?IBskQiVhITwlayVeJXMbKEI=?= diff --git a/src/mailman/handlers/docs/tagger.rst b/src/mailman/handlers/docs/tagger.rst new file mode 100644 index 000000000..b64b05c54 --- /dev/null +++ b/src/mailman/handlers/docs/tagger.rst @@ -0,0 +1,238 @@ +============== +Message tagger +============== + +Mailman has a topics system which works like this: a mailing list +administrator sets up one or more topics, which is essentially a named regular +expression. The topic name can be any arbitrary string, and the name serves +double duty as the *topic tag*. Each message that flows the mailing list has +its ``Subject:`` and ``Keywords:`` headers compared against these regular +expressions. The message then gets tagged with the topic names of each hit. + + >>> mlist = create_list('_xtest@example.com') + +Topics must be enabled for Mailman to do any topic matching, even if topics +are defined. +:: + + >>> mlist.topics = [('bar fight', '.*bar.*', 'catch any bars', False)] + >>> mlist.topics_enabled = False + >>> mlist.topics_bodylines_limit = 0 + + >>> msg = message_from_string("""\ + ... Subject: foobar + ... Keywords: barbaz + ... + ... """) + >>> msgdata = {} + + >>> from mailman.handlers.tagger import process + >>> process(mlist, msg, msgdata) + >>> print msg.as_string() + Subject: foobar + Keywords: barbaz + + + >>> msgdata + {} + +However, once topics are enabled, message will be tagged. There are two +artifacts of tagging; an ``X-Topics:`` header is added with the topic name, +and the message metadata gets a key with a list of matching topic names. + + >>> mlist.topics_enabled = True + >>> msg = message_from_string("""\ + ... Subject: foobar + ... Keywords: barbaz + ... + ... """) + >>> msgdata = {} + >>> process(mlist, msg, msgdata) + >>> print msg.as_string() + Subject: foobar + Keywords: barbaz + X-Topics: bar fight + + + >>> msgdata['topichits'] + [u'bar fight'] + + +Scanning body lines +=================== + +The tagger can also look at a certain number of body lines, but only for +``Subject:`` and ``Keyword:`` header-like lines. When set to zero, no body +lines are scanned. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... Subject: nothing + ... Keywords: at all + ... + ... X-Ignore: something else + ... Subject: foobar + ... Keywords: barbaz + ... """) + >>> msgdata = {} + >>> process(mlist, msg, msgdata) + >>> print msg.as_string() + From: aperson@example.com + Subject: nothing + Keywords: at all + + X-Ignore: something else + Subject: foobar + Keywords: barbaz + + >>> msgdata + {} + +But let the tagger scan a few body lines and the matching headers will be +found. + + >>> mlist.topics_bodylines_limit = 5 + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... Subject: nothing + ... Keywords: at all + ... + ... X-Ignore: something else + ... Subject: foobar + ... Keywords: barbaz + ... """) + >>> msgdata = {} + >>> process(mlist, msg, msgdata) + >>> print msg.as_string() + From: aperson@example.com + Subject: nothing + Keywords: at all + X-Topics: bar fight + + X-Ignore: something else + Subject: foobar + Keywords: barbaz + + >>> msgdata['topichits'] + [u'bar fight'] + +However, scanning stops at the first body line that doesn't look like a +header. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... Subject: nothing + ... Keywords: at all + ... + ... This is not a header + ... Subject: foobar + ... Keywords: barbaz + ... """) + >>> msgdata = {} + >>> process(mlist, msg, msgdata) + >>> print msg.as_string() + From: aperson@example.com + Subject: nothing + Keywords: at all + + This is not a header + Subject: foobar + Keywords: barbaz + >>> msgdata + {} + +When set to a negative number, all body lines will be scanned. + + >>> mlist.topics_bodylines_limit = -1 + >>> lots_of_headers = '\n'.join(['X-Ignore: zip'] * 100) + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... Subject: nothing + ... Keywords: at all + ... + ... %s + ... Subject: foobar + ... Keywords: barbaz + ... """ % lots_of_headers) + >>> msgdata = {} + >>> process(mlist, msg, msgdata) + >>> # Rather than print out 100 X-Ignore: headers, let's just prove that + >>> # the X-Topics: header exists, meaning that the tagger did its job. + >>> print msg['x-topics'] + bar fight + >>> msgdata['topichits'] + [u'bar fight'] + + +Scanning sub-parts +================== + +The tagger will also scan the body lines of text subparts in a multipart +message, using the same rules as if all those body lines lived in a single +text payload. + + >>> msg = message_from_string("""\ + ... Subject: Was + ... Keywords: Raw + ... Content-Type: multipart/alternative; boundary="BOUNDARY" + ... + ... --BOUNDARY + ... From: sabo + ... To: obas + ... + ... Subject: farbaw + ... Keywords: barbaz + ... + ... --BOUNDARY-- + ... """) + >>> msgdata = {} + >>> process(mlist, msg, msgdata) + >>> print msg.as_string() + Subject: Was + Keywords: Raw + Content-Type: multipart/alternative; boundary="BOUNDARY" + X-Topics: bar fight + + --BOUNDARY + From: sabo + To: obas + + Subject: farbaw + Keywords: barbaz + + --BOUNDARY-- + + >>> msgdata['topichits'] + [u'bar fight'] + +But the tagger will not descend into non-text parts. + + >>> msg = message_from_string("""\ + ... Subject: Was + ... Keywords: Raw + ... Content-Type: multipart/alternative; boundary=BOUNDARY + ... + ... --BOUNDARY + ... From: sabo + ... To: obas + ... Content-Type: message/rfc822 + ... + ... Subject: farbaw + ... Keywords: barbaz + ... + ... --BOUNDARY + ... From: sabo + ... To: obas + ... Content-Type: message/rfc822 + ... + ... Subject: farbaw + ... Keywords: barbaz + ... + ... --BOUNDARY-- + ... """) + >>> msgdata = {} + >>> process(mlist, msg, msgdata) + >>> print msg['x-topics'] + None + >>> msgdata + {} diff --git a/src/mailman/handlers/docs/to-outgoing.rst b/src/mailman/handlers/docs/to-outgoing.rst new file mode 100644 index 000000000..816aa4ca6 --- /dev/null +++ b/src/mailman/handlers/docs/to-outgoing.rst @@ -0,0 +1,42 @@ +==================== +The outgoing handler +==================== + +Mailman's outgoing queue is used as the wrapper around SMTP delivery to the +upstream mail server. The to-outgoing handler does little more than drop the +message into the outgoing queue. + + >>> mlist = create_list('test@example.com') + +Craft a message destined for the outgoing queue. Include some random metadata +as if this message had passed through some other handlers. +:: + + >>> msg = message_from_string("""\ + ... Subject: Here is a message + ... + ... Something of great import. + ... """) + + >>> msgdata = dict(foo=1, bar=2, verp=True) + >>> handler = config.handlers['to-outgoing'] + >>> handler.process(mlist, msg, msgdata) + +While the queued message will not be changed, the queued metadata will have an +additional key set: the mailing list name. + + >>> from mailman.testing.helpers import get_queue_messages + >>> messages = get_queue_messages('out') + >>> len(messages) + 1 + >>> print messages[0].msg.as_string() + Subject: Here is a message + + Something of great import. + >>> dump_msgdata(messages[0].msgdata) + _parsemsg: False + bar : 2 + foo : 1 + listname : test@example.com + verp : True + version : 3 diff --git a/src/mailman/handlers/file_recipients.py b/src/mailman/handlers/file_recipients.py new file mode 100644 index 000000000..d087ff2bb --- /dev/null +++ b/src/mailman/handlers/file_recipients.py @@ -0,0 +1,64 @@ +# Copyright (C) 2001-2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Get the normal delivery recipients from a Sendmail style :include: file.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'FileRecipients', + ] + + +import os +import errno + +from zope.interface import implements + +from mailman.core.i18n import _ +from mailman.interfaces.handler import IHandler + + + +class FileRecipients: + """Get the normal delivery recipients from an include file.""" + + implements(IHandler) + + name = 'file-recipients' + description = _('Get the normal delivery recipients from an include file.') + + def process(self, mlist, msg, msgdata): + """See `IHandler`.""" + if 'recipients' in msgdata: + return + filename = os.path.join(mlist.data_path, 'members.txt') + try: + with open(filename) as fp: + addrs = set(line.strip() for line in fp) + except IOError as error: + if error.errno != errno.ENOENT: + raise + msgdata['recipients'] = set() + return + # If the sender is a member of the list, remove them from the file + # recipients. + member = mlist.members.get_member(msg.sender) + if member is not None: + addrs.discard(member.address.email) + msgdata['recipients'] = addrs diff --git a/src/mailman/handlers/mime_delete.py b/src/mailman/handlers/mime_delete.py new file mode 100644 index 000000000..c9c1eb408 --- /dev/null +++ b/src/mailman/handlers/mime_delete.py @@ -0,0 +1,302 @@ +# Copyright (C) 2002-2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""MIME-stripping filter for Mailman. + +This module scans a message for MIME content, removing those sections whose +MIME types match one of a list of matches. multipart/alternative sections are +replaced by the first non-empty component, and multipart/mixed sections +wrapping only single sections after other processing are replaced by their +contents. +""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'MIMEDelete', + ] + + +import os +import errno +import logging +import tempfile + +from email.iterators import typed_subpart_iterator +from email.mime.message import MIMEMessage +from email.mime.text import MIMEText +from lazr.config import as_boolean +from os.path import splitext +from zope.interface import implements + +from mailman.config import config +from mailman.core import errors +from mailman.core.i18n import _ +from mailman.email.message import OwnerNotification +from mailman.interfaces.action import FilterAction +from mailman.interfaces.handler import IHandler +from mailman.utilities.string import oneline +from mailman.version import VERSION + + +log = logging.getLogger('mailman.error') + + + +def dispose(mlist, msg, msgdata, why): + if mlist.filter_action is FilterAction.reject: + # Bounce the message to the original author. + raise errors.RejectMessage(why) + elif mlist.filter_action is FilterAction.forward: + # Forward it on to the list moderators. + text=_("""\ +The attached message matched the $mlist.display_name mailing list's content +filtering rules and was prevented from being forwarded on to the list +membership. You are receiving the only remaining copy of the discarded +message. + +""") + subject=_('Content filter message notification') + notice = OwnerNotification(mlist, subject, roster=mlist.moderators) + notice.set_type('multipart/mixed') + notice.attach(MIMEText(text)) + notice.attach(MIMEMessage(msg)) + notice.send(mlist) + # Let this fall through so the original message gets discarded. + elif mlist.filter_action is FilterAction.preserve: + if as_boolean(config.mailman.filtered_messages_are_preservable): + # This is just like discarding the message except that a copy is + # placed in the 'bad' queue should the site administrator want to + # inspect the message. + filebase = config.switchboards['bad'].enqueue(msg, msgdata) + log.info('{0} preserved in file base {1}'.format( + msg.get('message-id', 'n/a'), filebase)) + else: + log.error( + '{1} invalid FilterAction: {0}. Treating as discard'.format( + mlist.fqdn_listname, mlist.filter_action.name)) + # Most cases also discard the message + raise errors.DiscardMessage(why) + + + +def process(mlist, msg, msgdata): + # We also don't care about our own digests or plaintext + ctype = msg.get_content_type() + mtype = msg.get_content_maintype() + # Check to see if the outer type matches one of the filter types + filtertypes = set(mlist.filter_types) + passtypes = set(mlist.pass_types) + if ctype in filtertypes or mtype in filtertypes: + dispose(mlist, msg, msgdata, + _("The message's content type was explicitly disallowed")) + # Check to see if there is a pass types and the outer type doesn't match + # one of these types + if passtypes and not (ctype in passtypes or mtype in passtypes): + dispose(mlist, msg, msgdata, + _("The message's content type was not explicitly allowed")) + # Filter by file extensions + filterexts = set(mlist.filter_extensions) + passexts = set(mlist.pass_extensions) + fext = get_file_ext(msg) + if fext: + if fext in filterexts: + dispose(mlist, msg, msgdata, + _("The message's file extension was explicitly disallowed")) + if passexts and not (fext in passexts): + dispose(mlist, msg, msgdata, + _("The message's file extension was not explicitly allowed")) + numparts = len([subpart for subpart in msg.walk()]) + # If the message is a multipart, filter out matching subparts + if msg.is_multipart(): + # Recursively filter out any subparts that match the filter list + prelen = len(msg.get_payload()) + filter_parts(msg, filtertypes, passtypes, filterexts, passexts) + # If the outer message is now an empty multipart (and it wasn't + # before!) then, again it gets discarded. + postlen = len(msg.get_payload()) + if postlen == 0 and prelen > 0: + dispose(mlist, msg, msgdata, + _("After content filtering, the message was empty")) + # Now replace all multipart/alternatives with just the first non-empty + # alternative. BAW: We have to special case when the outer part is a + # multipart/alternative because we need to retain most of the outer part's + # headers. For now we'll move the subpart's payload into the outer part, + # and then copy over its Content-Type: and Content-Transfer-Encoding: + # headers (any others?). + if mlist.collapse_alternatives: + collapse_multipart_alternatives(msg) + if ctype == 'multipart/alternative': + firstalt = msg.get_payload(0) + reset_payload(msg, firstalt) + # If we removed some parts, make note of this + changedp = 0 + if numparts <> len([subpart for subpart in msg.walk()]): + changedp = 1 + # Now perhaps convert all text/html to text/plain + if mlist.convert_html_to_plaintext and config.HTML_TO_PLAIN_TEXT_COMMAND: + changedp += to_plaintext(msg) + # If we're left with only two parts, an empty body and one attachment, + # recast the message to one of just that part + if msg.is_multipart() and len(msg.get_payload()) == 2: + if msg.get_payload(0).get_payload() == '': + useful = msg.get_payload(1) + reset_payload(msg, useful) + changedp = 1 + if changedp: + msg['X-Content-Filtered-By'] = 'Mailman/MimeDel {0}'.format(VERSION) + + + +def reset_payload(msg, subpart): + # Reset payload of msg to contents of subpart, and fix up content headers + payload = subpart.get_payload() + msg.set_payload(payload) + del msg['content-type'] + del msg['content-transfer-encoding'] + del msg['content-disposition'] + del msg['content-description'] + msg['Content-Type'] = subpart.get('content-type', 'text/plain') + cte = subpart.get('content-transfer-encoding') + if cte: + msg['Content-Transfer-Encoding'] = cte + cdisp = subpart.get('content-disposition') + if cdisp: + msg['Content-Disposition'] = cdisp + cdesc = subpart.get('content-description') + if cdesc: + msg['Content-Description'] = cdesc + + + +def filter_parts(msg, filtertypes, passtypes, filterexts, passexts): + # Look at all the message's subparts, and recursively filter + if not msg.is_multipart(): + return True + payload = msg.get_payload() + prelen = len(payload) + newpayload = [] + for subpart in payload: + keep = filter_parts(subpart, filtertypes, passtypes, + filterexts, passexts) + if not keep: + continue + ctype = subpart.get_content_type() + mtype = subpart.get_content_maintype() + if ctype in filtertypes or mtype in filtertypes: + # Throw this subpart away + continue + if passtypes and not (ctype in passtypes or mtype in passtypes): + # Throw this subpart away + continue + # check file extension + fext = get_file_ext(subpart) + if fext: + if fext in filterexts: + continue + if passexts and not (fext in passexts): + continue + newpayload.append(subpart) + # Check to see if we discarded all the subparts + postlen = len(newpayload) + msg.set_payload(newpayload) + if postlen == 0 and prelen > 0: + # We threw away everything + return False + return True + + + +def collapse_multipart_alternatives(msg): + if not msg.is_multipart(): + return + newpayload = [] + for subpart in msg.get_payload(): + if subpart.get_content_type() == 'multipart/alternative': + try: + firstalt = subpart.get_payload(0) + newpayload.append(firstalt) + except IndexError: + pass + else: + newpayload.append(subpart) + msg.set_payload(newpayload) + + + +def to_plaintext(msg): + changedp = False + for subpart in typed_subpart_iterator(msg, 'text', 'html'): + filename = tempfile.mktemp('.html') + fp = open(filename, 'w') + try: + fp.write(subpart.get_payload(decode=True)) + fp.close() + cmd = os.popen(config.HTML_TO_PLAIN_TEXT_COMMAND % + {'filename': filename}) + plaintext = cmd.read() + rtn = cmd.close() + if rtn: + log.error('HTML->text/plain error: %s', rtn) + finally: + try: + os.unlink(filename) + except OSError, e: + if e.errno <> errno.ENOENT: + raise + # Now replace the payload of the subpart and twiddle the Content-Type: + del subpart['content-transfer-encoding'] + subpart.set_payload(plaintext) + subpart.set_type('text/plain') + changedp = True + return changedp + + + +def get_file_ext(m): + """ + Get filename extension. Caution: some virus don't put filename + in 'Content-Disposition' header. +""" + fext = '' + filename = m.get_filename('') or m.get_param('name', '') + if filename: + fext = splitext(oneline(filename,'utf-8'))[1] + if len(fext) > 1: + fext = fext[1:] + else: + fext = '' + return fext + + + +class MIMEDelete: + """Filter the MIME content of messages.""" + + implements(IHandler) + + name = 'mime-delete' + description = _('Filter the MIME content of messages.') + + def process(self, mlist, msg, msgdata): + # Short-circuits + if not mlist.filter_content: + return + if msgdata.get('isdigest'): + return + process(mlist, msg, msgdata) diff --git a/src/mailman/handlers/owner_recipients.py b/src/mailman/handlers/owner_recipients.py new file mode 100644 index 000000000..9e4bbf174 --- /dev/null +++ b/src/mailman/handlers/owner_recipients.py @@ -0,0 +1,35 @@ +# Copyright (C) 2001-2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Calculate the list owner recipients (includes moderators).""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'process', + ] + + + +def process(mlist, msg, msgdata): + """Add owner recipients.""" + # The recipients are the owner and the moderator + msgdata['recipients'] = mlist.owner + mlist.moderator + # Don't decorate these messages with the header/footers + msgdata['nodecorate'] = True + msgdata['personalize'] = False diff --git a/src/mailman/handlers/replybot.py b/src/mailman/handlers/replybot.py new file mode 100644 index 000000000..83aa40214 --- /dev/null +++ b/src/mailman/handlers/replybot.py @@ -0,0 +1,123 @@ +# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Handler for automatic responses.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'Replybot', + ] + + +import logging + +from zope.component import getUtility +from zope.interface import implements + +from mailman.core.i18n import _ +from mailman.email.message import UserNotification +from mailman.interfaces.autorespond import ( + ALWAYS_REPLY, IAutoResponseSet, Response, ResponseAction) +from mailman.interfaces.handler import IHandler +from mailman.interfaces.usermanager import IUserManager +from mailman.utilities.datetime import today +from mailman.utilities.string import expand, wrap + + +log = logging.getLogger('mailman.error') + + + +class Replybot: + """Send automatic responses.""" + + implements(IHandler) + + name = 'replybot' + description = _('Send automatic responses.') + + def process(self, mlist, msg, msgdata): + """See `IHandler`.""" + # There are several cases where the replybot is short-circuited: + # * the original message has an "X-Ack: No" header + # * the message has a Precedence header with values bulk, junk, or + # list, and there's no explicit "X-Ack: yes" header + # * the message metadata has a true 'noack' key + ack = msg.get('x-ack', '').lower() + if ack == 'no' or msgdata.get('noack'): + return + precedence = msg.get('precedence', '').lower() + if ack != 'yes' and precedence in ('bulk', 'junk', 'list'): + return + # Check to see if the list is even configured to autorespond to this + # email message. Note: the incoming message processors should set the + # destination key in the message data. + if msgdata.get('to_owner'): + if mlist.autorespond_owner is ResponseAction.none: + return + response_type = Response.owner + response_text = mlist.autoresponse_owner_text + elif msgdata.get('to_request'): + if mlist.autorespond_requests is ResponseAction.none: + return + response_type = Response.command + response_text = mlist.autoresponse_request_text + elif msgdata.get('to_list'): + if mlist.autorespond_postings is ResponseAction.none: + return + response_type = Response.postings + response_text = mlist.autoresponse_postings_text + else: + # There are no automatic responses for any other destination. + return + # Now see if we're in the grace period for this sender. grace_period + # = 0 means always automatically respond, as does an "X-Ack: yes" + # header (useful for debugging). + response_set = IAutoResponseSet(mlist) + user_manager = getUtility(IUserManager) + address = user_manager.get_address(msg.sender) + if address is None: + address = user_manager.create_address(msg.sender) + grace_period = mlist.autoresponse_grace_period + if grace_period > ALWAYS_REPLY and ack != 'yes': + last = response_set.last_response(address, response_type) + if last is not None and last.date_sent + grace_period > today(): + return + # Okay, we know we're going to respond to this sender, craft the + # message, send it, and update the database. + display_name = mlist.display_name + subject = _( + 'Auto-response for your message to the "$display_name" ' + 'mailing list') + # Do string interpolation into the autoresponse text + d = dict(list_name = mlist.list_name, + display_name = display_name, + listurl = mlist.script_url('listinfo'), + requestemail = mlist.request_address, + owneremail = mlist.owner_address, + ) + # Interpolation and Wrap the response text. + text = wrap(expand(response_text, d)) + outmsg = UserNotification(msg.sender, mlist.bounces_address, + subject, text, mlist.preferred_language) + outmsg['X-Mailer'] = _('The Mailman Replybot') + # prevent recursions and mail loops! + outmsg['X-Ack'] = 'No' + outmsg.send(mlist) + response_set.response_sent(address, response_type) diff --git a/src/mailman/handlers/rfc_2369.py b/src/mailman/handlers/rfc_2369.py new file mode 100644 index 000000000..ece4e83cb --- /dev/null +++ b/src/mailman/handlers/rfc_2369.py @@ -0,0 +1,113 @@ +# Copyright (C) 2011-2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""RFC 2369 List-* and related headers.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'RFC2369', + ] + + +from email.utils import formataddr +from zope.interface import implements + +from mailman.config import config +from mailman.core.i18n import _ +from mailman.handlers.cook_headers import uheader +from mailman.interfaces.handler import IHandler + + +CONTINUATION = ',\n\t' + + + +def process(mlist, msg, msgdata): + """Add the RFC 2369 List-* and related headers.""" + # Some people really hate the List-* headers. It seems that the free + # version of Eudora (possibly on for some platforms) does not hide these + # headers by default, pissing off their users. Too bad. Fix the MUAs. + if not mlist.include_rfc2369_headers: + return + list_id = '{0.list_name}.{0.mail_host}'.format(mlist) + if mlist.description: + # Don't wrap the header since here we just want to get it properly RFC + # 2047 encoded. + i18ndesc = uheader(mlist, mlist.description, 'List-Id', maxlinelen=998) + listid_h = formataddr((str(i18ndesc), list_id)) + else: + # Without a description, we need to ensure the MUST brackets. + listid_h = '<{0}>'.format(list_id) + # No other agent should add a List-ID header except Mailman. + del msg['list-id'] + msg['List-Id'] = listid_h + # For internally crafted messages, we also add a (nonstandard), + # "X-List-Administrivia: yes" header. For all others (i.e. those coming + # from list posts), we add a bunch of other RFC 2369 headers. + requestaddr = mlist.request_address + subfieldfmt = '<{0}>, ' + listinfo = mlist.script_url('listinfo') + headers = {} + # XXX reduced_list_headers used to suppress List-Help, List-Subject, and + # List-Unsubscribe from UserNotification. That doesn't seem to make sense + # any more, so always add those three headers (others will still be + # suppressed). + headers.update({ + 'List-Help' : ''.format(requestaddr), + 'List-Unsubscribe': subfieldfmt.format(listinfo, mlist.leave_address), + 'List-Subscribe' : subfieldfmt.format(listinfo, mlist.join_address), + }) + if not msgdata.get('reduced_list_headers'): + # List-Post: is controlled by a separate attribute + if mlist.include_list_post_header: + headers['List-Post'] = ''.format(mlist.posting_address) + # Add RFC 2369 and 5064 archiving headers, if archiving is enabled. + if mlist.archive: + for archiver in config.archivers: + headers['List-Archive'] = '<{0}>'.format( + archiver.list_url(mlist)) + permalink = archiver.permalink(mlist, msg) + if permalink is not None: + headers['Archived-At'] = permalink + # XXX RFC 2369 also defines a List-Owner header which we are not currently + # supporting, but should. + for h, v in headers.items(): + # First we delete any pre-existing headers because the RFC permits + # only one copy of each, and we want to be sure it's ours. + del msg[h] + # Wrap these lines if they are too long. 78 character width probably + # shouldn't be hardcoded, but is at least text-MUA friendly. The + # adding of 2 is for the colon-space separator. + if len(h) + 2 + len(v) > 78: + v = CONTINUATION.join(v.split(', ')) + msg[h] = v + + + +class RFC2369: + """Add the RFC 2369 List-* headers.""" + + implements(IHandler) + + name = 'rfc-2369' + description = _('Add the RFC 2369 List-* headers.') + + def process(self, mlist, msg, msgdata): + """See `IHandler`.""" + process(mlist, msg, msgdata) diff --git a/src/mailman/handlers/tagger.py b/src/mailman/handlers/tagger.py new file mode 100644 index 000000000..49e004a12 --- /dev/null +++ b/src/mailman/handlers/tagger.py @@ -0,0 +1,191 @@ +# Copyright (C) 2001-2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Extract topics from the original mail message.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'Tagger', + ] + + +import re +import email.iterators +import email.parser + +from zope.interface import implements + +from mailman.core.i18n import _ +from mailman.interfaces.handler import IHandler + + +OR = '|' +CRNL = '\r\n' +EMPTYBYTES = b'' +NLTAB = '\n\t' + + + +def process(mlist, msg, msgdata): + """Tag the message for topics.""" + if not mlist.topics_enabled: + return + # Extract the Subject:, Keywords:, and possibly body text + matchlines = [] + matchlines.append(msg.get('subject', None)) + matchlines.append(msg.get('keywords', None)) + if mlist.topics_bodylines_limit == 0: + # Don't scan any body lines + pass + elif mlist.topics_bodylines_limit < 0: + # Scan all body lines + matchlines.extend(scanbody(msg)) + else: + # Scan just some of the body lines + matchlines.extend(scanbody(msg, mlist.topics_bodylines_limit)) + # Filter out any 'false' items. + matchlines = [item for item in matchlines if item] + # For each regular expression in the topics list, see if any of the lines + # of interest from the message match the regexp. If so, the message gets + # added to the specific topics bucket. + hits = {} + for name, pattern, desc, emptyflag in mlist.topics: + pattern = OR.join(pattern.splitlines()) + cre = re.compile(pattern, re.IGNORECASE) + for line in matchlines: + if cre.search(line): + hits[name] = 1 + break + if hits: + # Sort the keys and make them available both in the message metadata + # and in a message header. + msgdata['topichits'] = sorted(hits) + msg['X-Topics'] = NLTAB.join(sorted(hits)) + + + +def scanbody(msg, numlines=None): + """Scan the body for keywords.""" + # We only scan the body of the message if it is of MIME type text/plain, + # or if the outer type is multipart/alternative and there is a text/plain + # part. Anything else, and the body is ignored for header-scan purposes. + found = None + if msg.get_content_type() == 'text/plain': + found = msg + elif msg.is_multipart()\ + and msg.get_content_type() == 'multipart/alternative': + for found in msg.get_payload(): + if found.get_content_type() == 'text/plain': + break + else: + found = None + if not found: + return [] + # Now that we have a Message object that meets our criteria, let's extract + # the first numlines of body text. + lines = [] + lineno = 0 + reader = list(email.iterators.body_line_iterator(msg)) + while numlines is None or lineno < numlines: + try: + line = bytes(reader.pop(0)) + except IndexError: + break + # Blank lines don't count + if not line.strip(): + continue + lineno += 1 + lines.append(line) + # Concatenate those body text lines with newlines, and then create a new + # message object from those lines. + p = _ForgivingParser() + msg = p.parsestr(EMPTYBYTES.join(lines)) + return msg.get_all('subject', []) + msg.get_all('keywords', []) + + + +class _ForgivingParser(email.parser.HeaderParser): + """An lax email parser. + + Be a little more forgiving about non-header/continuation lines, since + we'll just read as much as we can from 'header-like' lines in the body. + """ + # BAW: WIBNI we didn't have to cut-n-paste this whole thing just to + # specialize the way it returns? + def _parseheaders(self, container, fp): + """See `email.parser.HeaderParser`.""" + # Parse the headers, returning a list of header/value pairs. None as + # the header means the Unix-From header. + lastheader = '' + lastvalue = [] + lineno = 0 + while 1: + # Don't strip the line before we test for the end condition, + # because whitespace-only header lines are RFC compliant + # continuation lines. + line = fp.readline() + if not line: + break + line = line.splitlines()[0] + if not line: + break + # Ignore the trailing newline + lineno += 1 + # Check for initial Unix From_ line + if line.startswith('From '): + if lineno == 1: + container.set_unixfrom(line) + continue + else: + break + # Header continuation line + if line[0] in ' \t': + if not lastheader: + break + lastvalue.append(line) + continue + # Normal, non-continuation header. BAW: this should check to make + # sure it's a legal header, e.g. doesn't contain spaces. Also, we + # should expose the header matching algorithm in the API, and + # allow for a non-strict parsing mode (that ignores the line + # instead of raising the exception). + i = line.find(':') + if i < 0: + break + if lastheader: + container[lastheader] = NLTAB.join(lastvalue) + lastheader = line[:i] + lastvalue = [line[i+1:].lstrip()] + # Make sure we retain the last header + if lastheader: + container[lastheader] = NLTAB.join(lastvalue) + + + +class Tagger: + """Tag messages with topic matches.""" + + implements(IHandler) + + name = 'tagger' + description = _('Tag messages with topic matches.') + + def process(self, mlist, msg, msgdata): + """See `IHandler`.""" + process(mlist, msg, msgdata) diff --git a/src/mailman/handlers/tests/__init__.py b/src/mailman/handlers/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/mailman/handlers/tests/test_mimedel.py b/src/mailman/handlers/tests/test_mimedel.py new file mode 100644 index 000000000..6ca34b17b --- /dev/null +++ b/src/mailman/handlers/tests/test_mimedel.py @@ -0,0 +1,213 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Test the mime_delete handler.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestDispose', + ] + + +import unittest + +from zope.component import getUtility + +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.core import errors +from mailman.handlers import mime_delete +from mailman.interfaces.action import FilterAction +from mailman.interfaces.member import MemberRole +from mailman.interfaces.usermanager import IUserManager +from mailman.testing.helpers import ( + LogFileMark, + get_queue_messages, + specialized_message_from_string as mfs) +from mailman.testing.layers import ConfigLayer + + + +class TestDispose(unittest.TestCase): + """Test the mime_delete handler.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._msg = mfs("""\ +From: anne@example.com +To: test@example.com +Subject: A disposable message +Message-ID: + +""") + # Python 2.7 has assertMultiLineEqual. Let this work without bounds. + self.maxDiff = None + self.eq = getattr(self, 'assertMultiLineEqual', self.assertEqual) + config.push('dispose', """ + [mailman] + site_owner: noreply@example.com + """) + + def tearDown(self): + config.pop('dispose') + + def test_dispose_discard(self): + self._mlist.filter_action = FilterAction.discard + try: + mime_delete.dispose(self._mlist, self._msg, {}, 'discarding') + except errors.DiscardMessage as error: + pass + else: + raise AssertionError('DiscardMessage exception expected') + self.assertEqual(error.message, 'discarding') + # There should be no messages in the 'bad' queue. + self.assertEqual(len(get_queue_messages('bad')), 0) + + def test_dispose_bounce(self): + self._mlist.filter_action = FilterAction.reject + try: + mime_delete.dispose(self._mlist, self._msg, {}, 'rejecting') + except errors.RejectMessage as error: + pass + else: + raise AssertionError('RejectMessage exception expected') + self.assertEqual(error.message, 'rejecting') + # There should be no messages in the 'bad' queue. + self.assertEqual(len(get_queue_messages('bad')), 0) + + def test_dispose_forward(self): + # The disposed message gets forwarded to the list moderators. So + # first add some moderators. + user_manager = getUtility(IUserManager) + anne = user_manager.create_address('anne@example.com') + bart = user_manager.create_address('bart@example.com') + self._mlist.subscribe(anne, MemberRole.moderator) + self._mlist.subscribe(bart, MemberRole.moderator) + # Now set the filter action and dispose the message. + self._mlist.filter_action = FilterAction.forward + try: + mime_delete.dispose(self._mlist, self._msg, {}, 'forwarding') + except errors.DiscardMessage as error: + pass + else: + raise AssertionError('DiscardMessage exception expected') + self.assertEqual(error.message, 'forwarding') + # There should now be a multipart message in the virgin queue destined + # for the mailing list owners. + messages = get_queue_messages('virgin') + self.assertEqual(len(messages), 1) + message = messages[0].msg + self.assertEqual(message.get_content_type(), 'multipart/mixed') + # Anne and Bart should be recipients of the message, but it will look + # like the message is going to the list owners. + self.assertEqual(message['to'], 'test-owner@example.com') + self.assertEqual(message.recipients, + set(['anne@example.com', 'bart@example.com'])) + # The list owner should be the sender. + self.assertEqual(message['from'], 'noreply@example.com') + self.assertEqual(message['subject'], + 'Content filter message notification') + # The body of the first part provides the moderators some details. + part0 = message.get_payload(0) + self.assertEqual(part0.get_content_type(), 'text/plain') + self.eq(part0.get_payload(), """\ +The attached message matched the Test mailing list's content +filtering rules and was prevented from being forwarded on to the list +membership. You are receiving the only remaining copy of the discarded +message. + +""") + # The second part is the container for the original message. + part1 = message.get_payload(1) + self.assertEqual(part1.get_content_type(), 'message/rfc822') + # And the first part of *that* message will be the original message. + original = part1.get_payload(0) + self.assertEqual(original['subject'], 'A disposable message') + self.assertEqual(original['message-id'], '') + + def test_dispose_non_preservable(self): + # Two actions can happen here, depending on a site-wide setting. If + # the site owner has indicated that filtered messages cannot be + # preserved, then this is the same as discarding them. + self._mlist.filter_action = FilterAction.preserve + config.push('non-preservable', """ + [mailman] + filtered_messages_are_preservable: no + """) + try: + mime_delete.dispose(self._mlist, self._msg, {}, 'not preserved') + except errors.DiscardMessage as error: + pass + else: + raise AssertionError('DiscardMessage exception expected') + finally: + config.pop('non-preservable') + self.assertEqual(error.message, 'not preserved') + # There should be no messages in the 'bad' queue. + self.assertEqual(len(get_queue_messages('bad')), 0) + + def test_dispose_preservable(self): + # Two actions can happen here, depending on a site-wide setting. If + # the site owner has indicated that filtered messages can be + # preserved, then this is similar to discarding the message except + # that a copy is preserved in the 'bad' queue. + self._mlist.filter_action = FilterAction.preserve + config.push('preservable', """ + [mailman] + filtered_messages_are_preservable: yes + """) + try: + mime_delete.dispose(self._mlist, self._msg, {}, 'preserved') + except errors.DiscardMessage as error: + pass + else: + raise AssertionError('DiscardMessage exception expected') + finally: + config.pop('preservable') + self.assertEqual(error.message, 'preserved') + # There should be no messages in the 'bad' queue. + messages = get_queue_messages('bad') + self.assertEqual(len(messages), 1) + message = messages[0].msg + self.assertEqual(message['subject'], 'A disposable message') + self.assertEqual(message['message-id'], '') + + def test_bad_action(self): + # This should never happen, but what if it does? + # FilterAction.accept, FilterAction.hold, and FilterAction.defer are + # not valid. They are treated as discard actions, but the problem is + # also logged. + for action in (FilterAction.accept, + FilterAction.hold, + FilterAction.defer): + self._mlist.filter_action = action + mark = LogFileMark('mailman.error') + try: + mime_delete.dispose(self._mlist, self._msg, {}, 'bad action') + except errors.DiscardMessage as error: + pass + else: + raise AssertionError('DiscardMessage exception expected') + self.assertEqual(error.message, 'bad action') + line = mark.readline()[:-1] + self.assertTrue(line.endswith( + '{0} invalid FilterAction: test@example.com. ' + 'Treating as discard'.format(action.name))) diff --git a/src/mailman/handlers/to_archive.py b/src/mailman/handlers/to_archive.py new file mode 100644 index 000000000..fd5259a14 --- /dev/null +++ b/src/mailman/handlers/to_archive.py @@ -0,0 +1,55 @@ +# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Add the message to the archives.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'ToArchive', + ] + + +from zope.interface import implements + +from mailman.config import config +from mailman.core.i18n import _ +from mailman.interfaces.handler import IHandler + + + +class ToArchive: + """Add the message to the archives.""" + + implements(IHandler) + + name = 'to-archive' + description = _('Add the message to the archives.') + + def process(self, mlist, msg, msgdata): + """See `IHandler`.""" + # Short circuits. + if msgdata.get('isdigest') or not mlist.archive: + return + # Common practice seems to favor "X-No-Archive: yes". No other value + # for this header seems to make sense, so we'll just test for it's + # presence. I'm keeping "X-Archive: no" for backwards compatibility. + if 'x-no-archive' in msg or msg.get('x-archive', '').lower() == 'no': + return + # Send the message to the archiver queue. + config.switchboards['archive'].enqueue(msg, msgdata) diff --git a/src/mailman/handlers/to_digest.py b/src/mailman/handlers/to_digest.py new file mode 100644 index 000000000..698f16e1e --- /dev/null +++ b/src/mailman/handlers/to_digest.py @@ -0,0 +1,123 @@ +# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Add the message to the list's current digest.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'ToDigest', + ] + + +import os +import datetime + +from zope.interface import implements + +from mailman.config import config +from mailman.core.i18n import _ +from mailman.email.message import Message +from mailman.interfaces.digests import DigestFrequency +from mailman.interfaces.handler import IHandler +from mailman.utilities.mailbox import Mailbox + + + +class ToDigest: + """Add the message to the digest, possibly sending it.""" + + implements(IHandler) + + name = 'to-digest' + description = _('Add the message to the digest, possibly sending it.') + + def process(self, mlist, msg, msgdata): + """See `IHandler`.""" + # Short circuit for non-digestable messages. + if not mlist.digestable or msgdata.get('isdigest'): + return + # Open the mailbox that will be used to collect the current digest. + mailbox_path = os.path.join(mlist.data_path, 'digest.mmdf') + # Lock the mailbox and append the message. + with Mailbox(mailbox_path, create=True) as mbox: + mbox.add(msg) + # Calculate the current size of the mailbox file. This will not tell + # us exactly how big the resulting MIME and rfc1153 digest will + # actually be, but it's the most easily available metric to decide + # whether the size threshold has been reached. + size = os.path.getsize(mailbox_path) + if size >= mlist.digest_size_threshold * 1024.0: + # The digest is ready to send. Because we don't want to hold up + # this process with crafting the digest, we're going to move the + # digest file to a safe place, then craft a fake message for the + # DigestRunner as a trigger for it to build and send the digest. + mailbox_dest = os.path.join( + mlist.data_path, + 'digest.{0.volume}.{0.next_digest_number}.mmdf'.format(mlist)) + volume = mlist.volume + digest_number = mlist.next_digest_number + bump_digest_number_and_volume(mlist) + os.rename(mailbox_path, mailbox_dest) + config.switchboards['digest'].enqueue( + Message(), + listname=mlist.fqdn_listname, + digest_path=mailbox_dest, + volume=volume, + digest_number=digest_number) + + + +def bump_digest_number_and_volume(mlist): + """Bump the digest number and volume.""" + now = datetime.datetime.now() + if mlist.digest_last_sent_at is None: + # There has been no previous digest. + bump = False + elif mlist.digest_volume_frequency == DigestFrequency.yearly: + bump = (now.year > mlist.digest_last_sent_at.year) + elif mlist.digest_volume_frequency == DigestFrequency.monthly: + # Monthly. + this_month = now.year * 100 + now.month + digest_month = (mlist.digest_last_sent_at.year * 100 + + mlist.digest_last_sent_at.month) + bump = (this_month > digest_month) + elif mlist.digest_volume_frequency == DigestFrequency.quarterly: + # Quarterly. + this_quarter = now.year * 100 + (now.month - 1) // 4 + digest_quarter = (mlist.digest_last_sent_at.year * 100 + + (mlist.digest_last_sent_at.month - 1) // 4) + bump = (this_quarter > digest_quarter) + elif mlist.digest_volume_frequency == DigestFrequency.weekly: + this_week = now.year * 100 + now.isocalendar()[1] + digest_week = (mlist.digest_last_sent_at.year * 100 + + mlist.digest_last_sent_at.isocalendar()[1]) + bump = (this_week > digest_week) + elif mlist.digest_volume_frequency == DigestFrequency.daily: + bump = (now.toordinal() > mlist.digest_last_sent_at.toordinal()) + else: + raise AssertionError( + 'Bad DigestFrequency: {0}'.format( + mlist.digest_volume_frequency)) + if bump: + mlist.volume += 1 + mlist.next_digest_number = 1 + else: + # Just bump the digest number. + mlist.next_digest_number += 1 + mlist.digest_last_sent_at = now diff --git a/src/mailman/handlers/to_outgoing.py b/src/mailman/handlers/to_outgoing.py new file mode 100644 index 000000000..971f87757 --- /dev/null +++ b/src/mailman/handlers/to_outgoing.py @@ -0,0 +1,52 @@ +# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Re-queue the message to the outgoing queue. + +This module is only for use by the IncomingRunner for delivering messages +posted to the list membership. Anything else that needs to go out to some +recipient should just be placed in the out queue directly. +""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'ToOutgoing', + ] + + +from zope.interface import implements + +from mailman.config import config +from mailman.core.i18n import _ +from mailman.interfaces.handler import IHandler + + + +class ToOutgoing: + """Send the message to the outgoing queue.""" + + implements(IHandler) + + name = 'to-outgoing' + description = _('Send the message to the outgoing queue.') + + def process(self, mlist, msg, msgdata): + """See `IHandler`.""" + config.switchboards['out'].enqueue( + msg, msgdata, listname=mlist.fqdn_listname) diff --git a/src/mailman/handlers/to_usenet.py b/src/mailman/handlers/to_usenet.py new file mode 100644 index 000000000..26a383c64 --- /dev/null +++ b/src/mailman/handlers/to_usenet.py @@ -0,0 +1,69 @@ +# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Move the message to the mail->news queue.""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'ToUsenet', + ] + + +import logging + +from zope.interface import implements + +from mailman.config import config +from mailman.core.i18n import _ +from mailman.interfaces.handler import IHandler + +COMMASPACE = ', ' + +log = logging.getLogger('mailman.error') + + + +class ToUsenet: + """Move the message to the outgoing news queue.""" + + implements(IHandler) + + name = 'to-usenet' + description = _('Move the message to the outgoing news queue.') + + def process(self, mlist, msg, msgdata): + """See `IHandler`.""" + # Short circuits. + if not mlist.gateway_to_news or \ + msgdata.get('isdigest') or \ + msgdata.get('fromusenet'): + return + # sanity checks + error = [] + if not mlist.linked_newsgroup: + error.append('no newsgroup') + if not mlist.nntp_host: + error.append('no NNTP host') + if error: + log.error('NNTP gateway improperly configured: %s', + COMMASPACE.join(error)) + return + # Put the message in the news runner's queue. + config.switchboards['news'].enqueue( + msg, msgdata, listname=mlist.fqdn_listname) diff --git a/src/mailman/pipeline/__init__.py b/src/mailman/pipeline/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/mailman/pipeline/acknowledge.py b/src/mailman/pipeline/acknowledge.py deleted file mode 100644 index 0e0916337..000000000 --- a/src/mailman/pipeline/acknowledge.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see . - -"""Send an acknowledgment of the successful post to the sender. - -This only happens if the sender has set their AcknowledgePosts attribute. -""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'Acknowledge', - ] - - -from zope.component import getUtility -from zope.interface import implements - -from mailman.core.i18n import _ -from mailman.email.message import UserNotification -from mailman.interfaces.handler import IHandler -from mailman.interfaces.languages import ILanguageManager -from mailman.utilities.i18n import make -from mailman.utilities.string import oneline - - - -class Acknowledge: - """Send an acknowledgment.""" - implements(IHandler) - - name = 'acknowledge' - description = _("""Send an acknowledgment of a posting.""") - - def process(self, mlist, msg, msgdata): - """See `IHandler`.""" - # Extract the sender's address and find them in the user database - sender = msgdata.get('original_sender', msg.sender) - member = mlist.members.get_member(sender) - if member is None or not member.acknowledge_posts: - # Either the sender is not a member, in which case we can't know - # whether they want an acknowlegment or not, or they are a member - # who definitely does not want an acknowlegment. - return - # Okay, they are a member that wants an acknowledgment of their post. - # Give them their original subject. BAW: do we want to use the - # decoded header? - original_subject = msgdata.get( - 'origsubj', msg.get('subject', _('(no subject)'))) - # Get the user's preferred language. - language_manager = getUtility(ILanguageManager) - language = (language_manager[msgdata['lang']] - if 'lang' in msgdata - else member.preferred_language) - charset = language_manager[language.code].charset - # Now get the acknowledgement template. - display_name = mlist.display_name - text = make('postack.txt', - mailing_list=mlist, - language=language.code, - wrap=False, - subject=oneline(original_subject, charset), - list_name=mlist.list_name, - display_name=display_name, - listinfo_url=mlist.script_url('listinfo'), - optionsurl=member.options_url, - ) - # Craft the outgoing message, with all headers and attributes - # necessary for general delivery. Then enqueue it to the outgoing - # queue. - subject = _('$display_name post acknowledgment') - usermsg = UserNotification(sender, mlist.bounces_address, - subject, text, language) - usermsg.send(mlist) diff --git a/src/mailman/pipeline/after_delivery.py b/src/mailman/pipeline/after_delivery.py deleted file mode 100644 index 46007092b..000000000 --- a/src/mailman/pipeline/after_delivery.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see . - -"""Perform some bookkeeping after a successful post.""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'AfterDelivery', - ] - - -import datetime - -from zope.interface import implements - -from mailman.core.i18n import _ -from mailman.interfaces.handler import IHandler - - - -class AfterDelivery: - """Perform some bookkeeping after a successful post.""" - - implements(IHandler) - - name = 'after-delivery' - description = _('Perform some bookkeeping after a successful post.') - - def process(self, mlist, msg, msgdata): - """See `IHander`.""" - mlist.last_post_time = datetime.datetime.now() - mlist.post_id += 1 diff --git a/src/mailman/pipeline/avoid_duplicates.py b/src/mailman/pipeline/avoid_duplicates.py deleted file mode 100644 index ffbc80c85..000000000 --- a/src/mailman/pipeline/avoid_duplicates.py +++ /dev/null @@ -1,116 +0,0 @@ -# Copyright (C) 2002-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see . - -"""If the user wishes it, do not send duplicates of the same message. - -This module keeps an in-memory dictionary of Message-ID: and recipient pairs. -If a message with an identical Message-ID: is about to be sent to someone who -has already received a copy, we either drop the message, add a duplicate -warning header, or pass it through, depending on the user's preferences. -""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'AvoidDuplicates', - ] - - -from email.utils import getaddresses, formataddr -from zope.interface import implements - -from mailman.core.i18n import _ -from mailman.interfaces.handler import IHandler - - -COMMASPACE = ', ' - - - -class AvoidDuplicates: - """If the user wishes it, do not send duplicates of the same message.""" - - implements(IHandler) - - name = 'avoid-duplicates' - description = _('Suppress some duplicates of the same message.') - - def process(self, mlist, msg, msgdata): - """See `IHandler`.""" - recips = msgdata.get('recipients') - # Short circuit - if not recips: - return - # Seed this set with addresses we don't care about dup avoiding. - listaddrs = set((mlist.posting_address, - mlist.bounces_address, - mlist.owner_address, - mlist.request_address)) - explicit_recips = listaddrs.copy() - # Figure out the set of explicit recipients. - cc_addresses = {} - for header in ('to', 'cc', 'resent-to', 'resent-cc'): - addrs = getaddresses(msg.get_all(header, [])) - header_addresses = dict((addr, formataddr((name, addr))) - for name, addr in addrs - if addr) - if header == 'cc': - # Yes, it's possible that an address is mentioned in multiple - # CC headers using different names. In that case, the last - # real name will win, but that doesn't seem like such a big - # deal. Besides, how else would you chose? - cc_addresses.update(header_addresses) - # Ignore the list addresses for purposes of dup avoidance. - explicit_recips |= set(header_addresses) - # Now strip out the list addresses. - explicit_recips -= listaddrs - if not explicit_recips: - # No one was explicitly addressed, so we can't do any dup - # collapsing - return - newrecips = set() - for r in recips: - # If this recipient is explicitly addressed... - if r in explicit_recips: - send_duplicate = True - # If the member wants to receive duplicates, or if the - # recipient is not a member at all, they will get a copy. - # header. - member = mlist.members.get_member(r) - if member and not member.receive_list_copy: - send_duplicate = False - # We'll send a duplicate unless the user doesn't wish it. If - # personalization is enabled, the add-dupe-header flag will - # add a X-Mailman-Duplicate: yes header for this user's - # message. - if send_duplicate: - msgdata.setdefault('add-dup-header', set()).add(r) - newrecips.add(r) - elif r in cc_addresses: - del cc_addresses[r] - else: - # Otherwise, this is the first time they've been in the recips - # list. Add them to the newrecips list and flag them as - # having received this message. - newrecips.add(r) - # Set the new list of recipients. XXX recips should always be a set. - msgdata['recipients'] = list(newrecips) - # RFC 2822 specifies zero or one CC header - if cc_addresses: - del msg['cc'] - msg['CC'] = COMMASPACE.join(cc_addresses.values()) diff --git a/src/mailman/pipeline/calculate_recipients.py b/src/mailman/pipeline/calculate_recipients.py deleted file mode 100644 index 15be07ecd..000000000 --- a/src/mailman/pipeline/calculate_recipients.py +++ /dev/null @@ -1,149 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see . - -"""Calculate the regular (i.e. non-digest) recipients of the message. - -This module calculates the non-digest recipients for the message based on the -list's membership and configuration options. It places the list of recipients -on the `recipients' attribute of the message. This attribute is used by the -SendmailDeliver and BulkDeliver modules. -""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'CalculateRecipients', - ] - -from zope.interface import implements - -from mailman.config import config -from mailman.core import errors -from mailman.core.i18n import _ -from mailman.interfaces.handler import IHandler -from mailman.interfaces.member import DeliveryStatus -from mailman.utilities.string import wrap - - - -class CalculateRecipients: - """Calculate the regular (i.e. non-digest) recipients of the message.""" - - implements(IHandler) - - name = 'calculate-recipients' - description = _('Calculate the regular recipients of the message.') - - def process(self, mlist, msg, msgdata): - """See `IHandler`.""" - # Short circuit if we've already calculated the recipients list, - # regardless of whether the list is empty or not. - if 'recipients' in msgdata: - return - # Should the original sender should be included in the recipients list? - include_sender = True - member = mlist.members.get_member(msg.sender) - if member and not member.receive_own_postings: - include_sender = False - # Support for urgent messages, which bypasses digests and disabled - # delivery and forces an immediate delivery to all members Right Now. - # We are specifically /not/ allowing the site admins password to work - # here because we want to discourage the practice of sending the site - # admin password through email in the clear. (see also Approve.py) - # - # XXX This is broken. - missing = object() - password = msg.get('urgent', missing) - if password is not missing: - if mlist.Authenticate((config.AuthListModerator, - config.AuthListAdmin), - password): - recipients = mlist.getMemberCPAddresses( - mlist.getRegularMemberKeys() + - mlist.getDigestMemberKeys()) - msgdata['recipients'] = recipients - return - else: - # Bad Urgent: password, so reject it instead of passing it on. - # I think it's better that the sender know they screwed up - # than to deliver it normally. - listname = mlist.display_name - text = _("""\ -Your urgent message to the $listname mailing list was not authorized for -delivery. The original message as received by Mailman is attached. -""") - raise errors.RejectMessage(wrap(text)) - # Calculate the regular recipients of the message - recipients = set(member.address.email - for member in mlist.regular_members.members - if member.delivery_status == DeliveryStatus.enabled) - # Remove the sender if they don't want to receive their own posts - if not include_sender and member.address.email in recipients: - recipients.remove(member.address.email) - # Handle topic classifications - do_topic_filters(mlist, msg, msgdata, recipients) - # Bookkeeping - msgdata['recipients'] = recipients - - - -def do_topic_filters(mlist, msg, msgdata, recipients): - """Filter out recipients based on topics.""" - if not mlist.topics_enabled: - # MAS: if topics are currently disabled for the list, send to all - # regardless of ReceiveNonmatchingTopics - return - hits = msgdata.get('topichits') - zap_recipients = [] - if hits: - # The message hit some topics, so only deliver this message to those - # who are interested in one of the hit topics. - for user in recipients: - utopics = mlist.getMemberTopics(user) - if not utopics: - # This user is not interested in any topics, so they get all - # postings. - continue - # BAW: Slow, first-match, set intersection! - for topic in utopics: - if topic in hits: - # The user wants this message - break - else: - # The user was interested in topics, but not any of the ones - # this message matched, so zap him. - zap_recipients.append(user) - else: - # The semantics for a message that did not hit any of the pre-canned - # topics is to troll through the membership list, looking for users - # who selected at least one topic of interest, but turned on - # ReceiveNonmatchingTopics. - for user in recipients: - if not mlist.getMemberTopics(user): - # The user did not select any topics of interest, so he gets - # this message by default. - continue - if not mlist.getMemberOption( - user, config.ReceiveNonmatchingTopics): - # The user has interest in some topics, but elects not to - # receive message that match no topics, so zap him. - zap_recipients.append(user) - # Otherwise, the user wants non-matching messages. - # Prune out the non-receiving users - for user in zap_recipients: - recipients.remove(user) diff --git a/src/mailman/pipeline/cleanse.py b/src/mailman/pipeline/cleanse.py deleted file mode 100644 index 90f2a892a..000000000 --- a/src/mailman/pipeline/cleanse.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see . - -"""Cleanse certain headers from all messages.""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'Cleanse', - ] - - -import logging - -from email.utils import formataddr -from zope.interface import implements - -from mailman.core.i18n import _ -from mailman.interfaces.handler import IHandler -from mailman.pipeline.cook_headers import uheader - - -log = logging.getLogger('mailman.post') - - - -class Cleanse: - """Cleanse certain headers from all messages.""" - - implements(IHandler) - - name = 'cleanse' - description = _('Cleanse certain headers from all messages.') - - def process(self, mlist, msg, msgdata): - """See `IHandler`.""" - # Remove headers that could contain passwords. - del msg['approved'] - del msg['approve'] - del msg['x-approved'] - del msg['x-approve'] - del msg['urgent'] - # We remove other headers from anonymous lists. - if mlist.anonymous_list: - log.info('post to %s from %s anonymized', - mlist.fqdn_listname, msg.get('from')) - del msg['from'] - del msg['reply-to'] - del msg['sender'] - # Hotmail sets this one - del msg['x-originating-email'] - i18ndesc = str(uheader(mlist, mlist.description, 'From')) - msg['From'] = formataddr((i18ndesc, mlist.posting_address)) - msg['Reply-To'] = mlist.posting_address - # Some headers can be used to fish for membership. - del msg['return-receipt-to'] - del msg['disposition-notification-to'] - del msg['x-confirm-reading-to'] - # Pegasus mail uses this one... sigh. - del msg['x-pmrqc'] - # Don't let this header be spoofed. See RFC 5064. - del msg['archived-at'] diff --git a/src/mailman/pipeline/cleanse_dkim.py b/src/mailman/pipeline/cleanse_dkim.py deleted file mode 100644 index d2cd32636..000000000 --- a/src/mailman/pipeline/cleanse_dkim.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (C) 2006-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see . - -"""Remove any 'DomainKeys' (or similar) headers. - -The values contained in these header lines are intended to be used by the -recipient to detect forgery or tampering in transit, and the modifications -made by Mailman to the headers and body of the message will cause these keys -to appear invalid. Removing them will at least avoid this misleading result, -and it will also give the MTA the opportunity to regenerate valid keys -originating at the Mailman server for the outgoing message. -""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'CleanseDKIM', - ] - - -from lazr.config import as_boolean -from zope.interface import implements - -from mailman.config import config -from mailman.core.i18n import _ -from mailman.interfaces.handler import IHandler - - - -class CleanseDKIM: - """Remove DomainKeys headers.""" - - implements(IHandler) - - name = 'cleanse-dkim' - description = _('Remove DomainKeys headers.') - - def process(self, mlist, msg, msgdata): - """See `IHandler`.""" - if as_boolean(config.mta.remove_dkim_headers): - del msg['domainkey-signature'] - del msg['dkim-signature'] - del msg['authentication-results'] diff --git a/src/mailman/pipeline/cook_headers.py b/src/mailman/pipeline/cook_headers.py deleted file mode 100644 index 2d117429c..000000000 --- a/src/mailman/pipeline/cook_headers.py +++ /dev/null @@ -1,292 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see . - -"""Cook a message's headers.""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'CookHeaders', - ] - - -import re - -from email.errors import HeaderParseError -from email.header import Header, decode_header, make_header -from email.utils import parseaddr, formataddr, getaddresses -from zope.interface import implements - -from mailman.core.i18n import _ -from mailman.interfaces.handler import IHandler -from mailman.interfaces.mailinglist import Personalization, ReplyToMunging -from mailman.version import VERSION - - -COMMASPACE = ', ' -MAXLINELEN = 78 - -nonascii = re.compile('[^\s!-~]') - - - -def uheader(mlist, s, header_name=None, continuation_ws='\t', maxlinelen=None): - """Get the charset to encode the string in. - - Then search if there is any non-ascii character is in the string. If - there is and the charset is us-ascii then we use iso-8859-1 instead. If - the string is ascii only we use 'us-ascii' if another charset is - specified. - """ - charset = mlist.preferred_language.charset - if nonascii.search(s): - # use list charset but ... - if charset == 'us-ascii': - charset = 'iso-8859-1' - else: - # there is no nonascii so ... - charset = 'us-ascii' - return Header(s, charset, maxlinelen, header_name, continuation_ws) - - - -def process(mlist, msg, msgdata): - """Process the headers of the message.""" - # Set the "X-Ack: no" header if noack flag is set. - if msgdata.get('noack'): - del msg['x-ack'] - msg['X-Ack'] = 'no' - # Because we're going to modify various important headers in the email - # message, we want to save some of the information in the msgdata - # dictionary for later. Specifically, the sender header will get waxed, - # but we need it for the Acknowledge module later. - msgdata['original_sender'] = msg.sender - # VirginRunner sets _fasttrack for internally crafted messages. - fasttrack = msgdata.get('_fasttrack') - if not msgdata.get('isdigest') and not fasttrack: - try: - prefix_subject(mlist, msg, msgdata) - except (UnicodeError, ValueError): - # TK: Sometimes subject header is not MIME encoded for 8bit - # simply abort prefixing. - pass - # Add Precedence: and other useful headers. None of these are standard - # and finding information on some of them are fairly difficult. Some are - # just common practice, and we'll add more here as they become necessary. - # Good places to look are: - # - # http://www.dsv.su.se/~jpalme/ietf/jp-ietf-home.html - # http://www.faqs.org/rfcs/rfc2076.html - # - # None of these headers are added if they already exist. BAW: some - # consider the advertising of this a security breach. I.e. if there are - # known exploits in a particular version of Mailman and we know a site is - # using such an old version, they may be vulnerable. It's too easy to - # edit the code to add a configuration variable to handle this. - if 'x-mailman-version' not in msg: - msg['X-Mailman-Version'] = VERSION - # We set "Precedence: list" because this is the recommendation from the - # sendmail docs, the most authoritative source of this header's semantics. - if 'precedence' not in msg: - msg['Precedence'] = 'list' - # Reply-To: munging. Do not do this if the message is "fast tracked", - # meaning it is internally crafted and delivered to a specific user. BAW: - # Yuck, I really hate this feature but I've caved under the sheer pressure - # of the (very vocal) folks want it. OTOH, RFC 2822 allows Reply-To: to - # be a list of addresses, so instead of replacing the original, simply - # augment it. RFC 2822 allows max one Reply-To: header so collapse them - # if we're adding a value, otherwise don't touch it. (Should we collapse - # in all cases?) - if not fasttrack: - # A convenience function, requires nested scopes. pair is (name, addr) - new = [] - d = {} - def add(pair): - lcaddr = pair[1].lower() - if lcaddr in d: - return - d[lcaddr] = pair - new.append(pair) - # List admin wants an explicit Reply-To: added - if mlist.reply_goes_to_list == ReplyToMunging.explicit_header: - add(parseaddr(mlist.reply_to_address)) - # If we're not first stripping existing Reply-To: then we need to add - # the original Reply-To:'s to the list we're building up. In both - # cases we'll zap the existing field because RFC 2822 says max one is - # allowed. - if not mlist.first_strip_reply_to: - orig = msg.get_all('reply-to', []) - for pair in getaddresses(orig): - add(pair) - # Set Reply-To: header to point back to this list. Add this last - # because some folks think that some MUAs make it easier to delete - # addresses from the right than from the left. - if mlist.reply_goes_to_list == ReplyToMunging.point_to_list: - i18ndesc = uheader(mlist, mlist.description, 'Reply-To') - add((str(i18ndesc), mlist.posting_address)) - del msg['reply-to'] - # Don't put Reply-To: back if there's nothing to add! - if new: - # Preserve order - msg['Reply-To'] = COMMASPACE.join( - [formataddr(pair) for pair in new]) - # The To field normally contains the list posting address. However - # when messages are fully personalized, that header will get - # overwritten with the address of the recipient. We need to get the - # posting address in one of the recipient headers or they won't be - # able to reply back to the list. It's possible the posting address - # was munged into the Reply-To header, but if not, we'll add it to a - # Cc header. BAW: should we force it into a Reply-To header in the - # above code? - # Also skip Cc if this is an anonymous list as list posting address - # is already in From and Reply-To in this case. - if (mlist.personalize == Personalization.full and - mlist.reply_goes_to_list != ReplyToMunging.point_to_list and - not mlist.anonymous_list): - # Watch out for existing Cc headers, merge, and remove dups. Note - # that RFC 2822 says only zero or one Cc header is allowed. - new = [] - d = {} - for pair in getaddresses(msg.get_all('cc', [])): - add(pair) - i18ndesc = uheader(mlist, mlist.description, 'Cc') - add((str(i18ndesc), mlist.posting_address)) - del msg['Cc'] - msg['Cc'] = COMMASPACE.join([formataddr(pair) for pair in new]) - - - -def prefix_subject(mlist, msg, msgdata): - """Maybe add a subject prefix. - - Add the subject prefix unless the message is a digest or is being fast - tracked (e.g. internally crafted, delivered to a single user such as the - list admin). - """ - if not mlist.subject_prefix.strip(): - return - prefix = mlist.subject_prefix - subject = msg.get('subject', '') - # Try to figure out what the continuation_ws is for the header - if isinstance(subject, Header): - lines = str(subject).splitlines() - else: - lines = subject.splitlines() - ws = '\t' - if len(lines) > 1 and lines[1] and lines[1][0] in ' \t': - ws = lines[1][0] - msgdata['origsubj'] = subject - # The subject may be multilingual but we take the first charset as major - # one and try to decode. If it is decodable, returned subject is in one - # line and cset is properly set. If fail, subject is mime-encoded and - # cset is set as us-ascii. See detail for ch_oneline() (CookHeaders one - # line function). - subject, cset = ch_oneline(subject) - # TK: Python interpreter has evolved to be strict on ascii charset code - # range. It is safe to use unicode string when manupilating header - # contents with re module. It would be best to return unicode in - # ch_oneline() but here is temporary solution. - subject = unicode(subject, cset) - # If the subject_prefix contains '%d', it is replaced with the - # mailing list sequential number. Sequential number format allows - # '%d' or '%05d' like pattern. - prefix_pattern = re.escape(prefix) - # unescape '%' :-< - prefix_pattern = '%'.join(prefix_pattern.split(r'\%')) - p = re.compile('%\d*d') - if p.search(prefix, 1): - # prefix have number, so we should search prefix w/number in subject. - # Also, force new style. - prefix_pattern = p.sub(r'\s*\d+\s*', prefix_pattern) - subject = re.sub(prefix_pattern, '', subject) - rematch = re.match('((RE|AW|SV|VS)(\[\d+\])?:\s*)+', subject, re.I) - if rematch: - subject = subject[rematch.end():] - recolon = 'Re:' - else: - recolon = '' - # At this point, subject may become null if someone post mail with - # subject: [subject prefix] - if subject.strip() == '': - subject = _('(no subject)') - cset = mlist.preferred_language.charset - # and substitute %d in prefix with post_id - try: - prefix = prefix % mlist.post_id - except TypeError: - pass - # Get the header as a Header instance, with proper unicode conversion - if not recolon: - h = uheader(mlist, prefix, 'Subject', continuation_ws=ws) - else: - h = uheader(mlist, prefix, 'Subject', continuation_ws=ws) - h.append(recolon) - # TK: Subject is concatenated and unicode string. - subject = subject.encode(cset, 'replace') - h.append(subject, cset) - del msg['subject'] - msg['Subject'] = h - ss = uheader(mlist, recolon, 'Subject', continuation_ws=ws) - ss.append(subject, cset) - msgdata['stripped_subject'] = ss - - - -def ch_oneline(headerstr): - # Decode header string in one line and convert into single charset. - # Return (string, cset) tuple as check for failure. - try: - d = decode_header(headerstr) - # At this point, we should rstrip() every string because some - # MUA deliberately add trailing spaces when composing return - # message. - d = [(s.rstrip(), c) for (s, c) in d] - # Find all charsets in the original header. We use 'utf-8' rather - # than using the first charset (in mailman 2.1.x) if multiple - # charsets are used. - csets = [] - for (s, c) in d: - if c and c not in csets: - csets.append(c) - if len(csets) == 0: - cset = 'us-ascii' - elif len(csets) == 1: - cset = csets[0] - else: - cset = 'utf-8' - h = make_header(d) - ustr = unicode(h) - oneline = ''.join(ustr.splitlines()) - return oneline.encode(cset, 'replace'), cset - except (LookupError, UnicodeError, ValueError, HeaderParseError): - # possibly charset problem. return with undecoded string in one line. - return ''.join(headerstr.splitlines()), 'us-ascii' - - - -class CookHeaders: - """Modify message headers.""" - - implements(IHandler) - - name = 'cook-headers' - description = _('Modify message headers.') - - def process(self, mlist, msg, msgdata): - """See `IHandler`.""" - process(mlist, msg, msgdata) diff --git a/src/mailman/pipeline/decorate.py b/src/mailman/pipeline/decorate.py deleted file mode 100644 index d6d156048..000000000 --- a/src/mailman/pipeline/decorate.py +++ /dev/null @@ -1,245 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see . - -"""Decorate a message by sticking the header and footer around it.""" - -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type -__all__ = [ - 'Decorate', - ] - - -import re -import logging - -from email.mime.text import MIMEText -from urllib2 import URLError -from zope.component import getUtility -from zope.interface import implements - -from mailman.core.i18n import _ -from mailman.email.message import Message -from mailman.interfaces.handler import IHandler -from mailman.interfaces.templates import ITemplateLoader -from mailman.utilities.string import expand - - -log = logging.getLogger('mailman.error') - - - -def process(mlist, msg, msgdata): - """Decorate the message with headers and footers.""" - # Digests and Mailman-craft messages should not get additional headers. - if msgdata.get('isdigest') or msgdata.get('nodecorate'): - return - d = {} - member = msgdata.get('member') - if member is not None: - # Calculate the extra personalization dictionary. - recipient = msgdata.get('recipient', member.address.original_email) - d['user_address'] = recipient - d['user_delivered_to'] = member.address.original_email - d['user_language'] = member.preferred_language.description - d['user_name'] = (member.user.display_name - if member.user.display_name - else member.address.original_email) - d['user_optionsurl'] = member.options_url - # These strings are descriptive for the log file and shouldn't be i18n'd - d.update(msgdata.get('decoration-data', {})) - try: - header = decorate(mlist, mlist.header_uri, d) - except URLError: - log.exception('Header decorator URI not found ({0}): {1}'.format( - mlist.fqdn_listname, mlist.header_uri)) - try: - footer = decorate(mlist, mlist.footer_uri, d) - except URLError: - log.exception('Footer decorator URI not found ({0}): {1}'.format( - mlist.fqdn_listname, mlist.footer_uri)) - # Escape hatch if both the footer and header are empty - if not header and not footer: - return - # Be MIME smart here. We only attach the header and footer by - # concatenation when the message is a non-multipart of type text/plain. - # Otherwise, if it is not a multipart, we make it a multipart, and then we - # add the header and footer as text/plain parts. - # - # BJG: In addition, only add the footer if the message's character set - # matches the charset of the list's preferred language. This is a - # suboptimal solution, and should be solved by allowing a list to have - # multiple headers/footers, for each language the list supports. - # - # Also, if the list's preferred charset is us-ascii, we can always - # safely add the header/footer to a plain text message since all - # charsets Mailman supports are strict supersets of us-ascii -- - # no, UTF-16 emails are not supported yet. - # - # TK: Message with 'charset=' cause trouble. So, instead of - # mgs.get_content_charset('us-ascii') ... - mcset = msg.get_content_charset() or 'us-ascii' - lcset = mlist.preferred_language.charset - msgtype = msg.get_content_type() - # BAW: If the charsets don't match, should we add the header and footer by - # MIME multipart chroming the message? - wrap = True - if not msg.is_multipart() and msgtype == 'text/plain': - # Save the RFC-3676 format parameters. - format_param = msg.get_param('format') - delsp = msg.get_param('delsp') - # Save 'Content-Transfer-Encoding' header in case decoration fails. - cte = msg.get('content-transfer-encoding') - # header/footer is now in unicode (2.2) - try: - oldpayload = unicode(msg.get_payload(decode=True), mcset) - del msg['content-transfer-encoding'] - frontsep = endsep = '' - if header and not header.endswith('\n'): - frontsep = '\n' - if footer and not oldpayload.endswith('\n'): - endsep = '\n' - payload = header + frontsep + oldpayload + endsep + footer - # When setting the payload for the message, try various charset - # encodings until one does not produce a UnicodeError. We'll try - # charsets in this order: the list's charset, the message's - # charset, then utf-8. It's okay if some of these are duplicates. - for cset in (lcset, mcset, 'utf-8'): - try: - msg.set_payload(payload.encode(cset), cset) - except UnicodeError: - pass - else: - if format_param: - msg.set_param('format', format_param) - if delsp: - msg.set_param('delsp', delsp) - wrap = False - break - except (LookupError, UnicodeError): - if cte: - # Restore the original c-t-e. - del msg['content-transfer-encoding'] - msg['Content-Transfer-Encoding'] = cte - elif msg.get_content_type() == 'multipart/mixed': - # The next easiest thing to do is just prepend the header and append - # the footer as additional subparts - payload = msg.get_payload() - if not isinstance(payload, list): - payload = [payload] - if footer: - mimeftr = MIMEText(footer.encode(lcset), 'plain', lcset) - mimeftr['Content-Disposition'] = 'inline' - payload.append(mimeftr) - if header: - mimehdr = MIMEText(header.encode(lcset), 'plain', lcset) - mimehdr['Content-Disposition'] = 'inline' - payload.insert(0, mimehdr) - msg.set_payload(payload) - wrap = False - # If we couldn't add the header or footer in a less intrusive way, we can - # at least do it by MIME encapsulation. We want to keep as much of the - # outer chrome as possible. - if not wrap: - return - # Because of the way Message objects are passed around to process(), we - # need to play tricks with the outer message -- i.e. the outer one must - # remain the same instance. So we're going to create a clone of the outer - # message, with all the header chrome intact, then copy the payload to it. - # This will give us a clone of the original message, and it will form the - # basis of the interior, wrapped Message. - inner = Message() - # Which headers to copy? Let's just do the Content-* headers - for h, v in msg.items(): - if h.lower().startswith('content-'): - inner[h] = v - inner.set_payload(msg.get_payload()) - # For completeness - inner.set_unixfrom(msg.get_unixfrom()) - inner.preamble = msg.preamble - inner.epilogue = msg.epilogue - # Don't copy get_charset, as this might be None, even if - # get_content_charset isn't. However, do make sure there is a default - # content-type, even if the original message was not MIME. - inner.set_default_type(msg.get_default_type()) - # BAW: HACK ALERT. - if hasattr(msg, '__version__'): - inner.__version__ = msg.__version__ - # Now, play games with the outer message to make it contain three - # subparts: the header (if any), the wrapped message, and the footer (if - # any). - payload = [inner] - if header: - mimehdr = MIMEText(header.encode(lcset), 'plain', lcset) - mimehdr['Content-Disposition'] = 'inline' - payload.insert(0, mimehdr) - if footer: - mimeftr = MIMEText(footer.encode(lcset), 'plain', lcset) - mimeftr['Content-Disposition'] = 'inline' - payload.append(mimeftr) - msg.set_payload(payload) - del msg['content-type'] - del msg['content-transfer-encoding'] - del msg['content-disposition'] - msg['Content-Type'] = 'multipart/mixed' - - - -def decorate(mlist, uri, extradict=None): - """Expand the decoration template.""" - if uri is None: - return '' - # Get the decorator template. - loader = getUtility(ITemplateLoader) - template_uri = expand(uri, dict( - listname=mlist.fqdn_listname, - language=mlist.preferred_language.code, - )) - template = loader.get(template_uri) - # Create a dictionary which includes the default set of interpolation - # variables allowed in headers and footers. These will be augmented by - # any key/value pairs in the extradict. - substitutions = dict( - fqdn_listname = mlist.fqdn_listname, - list_name = mlist.list_name, - host_name = mlist.mail_host, - display_name = mlist.display_name, - listinfo_uri = mlist.script_url('listinfo'), - list_requests = mlist.request_address, - description = mlist.description, - info = mlist.info, - ) - if extradict is not None: - substitutions.update(extradict) - text = expand(template, substitutions) - # Turn any \r\n line endings into just \n - return re.sub(r' *\r?\n', r'\n', text) - - - -class Decorate: - """Decorate a message with headers and footers.""" - - implements(IHandler) - - name = 'decorate' - description = _('Decorate a message with headers and footers.') - - def process(self, mlist, msg, msgdata): - "See `IHandler`.""" - process(mlist, msg, msgdata) diff --git a/src/mailman/pipeline/docs/ack-headers.rst b/src/mailman/pipeline/docs/ack-headers.rst deleted file mode 100644 index dba2169e2..000000000 --- a/src/mailman/pipeline/docs/ack-headers.rst +++ /dev/null @@ -1,43 +0,0 @@ -====================== -Acknowledgment headers -====================== - -Messages that flow through the global pipeline get their headers `cooked`, -which basically means that their headers go through several mostly unrelated -transformations. Some headers get added, others get changed. Some of these -changes depend on mailing list settings and others depend on how the message -is getting sent through the system. We'll take things one-by-one. - - >>> mlist = create_list('_xtest@example.com') - >>> mlist.subject_prefix = '' - -When the message's metadata has a `noack` key set, an ``X-Ack: no`` header is -added. -:: - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... - ... A message of great import. - ... """) - - >>> from mailman.pipeline.cook_headers import process - >>> process(mlist, msg, dict(noack=True)) - >>> print msg.as_string() - From: aperson@example.com - X-Ack: no - ... - -Any existing ``X-Ack`` header in the original message is removed. - - >>> msg = message_from_string("""\ - ... X-Ack: yes - ... From: aperson@example.com - ... - ... A message of great import. - ... """) - >>> process(mlist, msg, dict(noack=True)) - >>> print msg.as_string() - From: aperson@example.com - X-Ack: no - ... diff --git a/src/mailman/pipeline/docs/acknowledge.rst b/src/mailman/pipeline/docs/acknowledge.rst deleted file mode 100644 index 479aa4ea6..000000000 --- a/src/mailman/pipeline/docs/acknowledge.rst +++ /dev/null @@ -1,174 +0,0 @@ -====================== -Message acknowledgment -====================== - -When a user posts a message to a mailing list, and that user has chosen to -receive acknowledgments of their postings, Mailman will sent them such an -acknowledgment. -:: - - >>> mlist = create_list('test@example.com') - >>> mlist.display_name = 'Test' - >>> mlist.preferred_language = 'en' - >>> # XXX This will almost certainly change once we've worked out the web - >>> # space layout for mailing lists now. - - >>> # Ensure that the virgin queue is empty, since we'll be checking this - >>> # for new auto-response messages. - >>> from mailman.testing.helpers import get_queue_messages - >>> get_queue_messages('virgin') - [] - -Subscribe a user to the mailing list. -:: - - >>> from mailman.interfaces.usermanager import IUserManager - >>> from zope.component import getUtility - >>> user_manager = getUtility(IUserManager) - - >>> from mailman.interfaces.member import MemberRole - >>> user_1 = user_manager.create_user('aperson@example.com') - >>> address_1 = list(user_1.addresses)[0] - >>> mlist.subscribe(address_1, MemberRole.member) - - - -Non-member posts -================ - -Non-members can't get acknowledgments of their posts to the mailing list. -:: - - >>> msg = message_from_string("""\ - ... From: bperson@example.com - ... - ... """) - - >>> handler = config.handlers['acknowledge'] - >>> handler.process(mlist, msg, {}) - >>> get_queue_messages('virgin') - [] - -We can also specify the original sender in the message's metadata. If that -person is also not a member, no acknowledgment will be sent either. - - >>> msg = message_from_string("""\ - ... From: bperson@example.com - ... - ... """) - >>> handler.process(mlist, msg, - ... dict(original_sender='cperson@example.com')) - >>> get_queue_messages('virgin') - [] - - -No acknowledgment requested -=========================== - -Unless the user has requested acknowledgments, they will not get one. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... - ... """) - >>> handler.process(mlist, msg, {}) - >>> get_queue_messages('virgin') - [] - -Similarly if the original sender is specified in the message metadata, and -that sender is a member but not one who has requested acknowledgments, none -will be sent. -:: - - >>> user_2 = user_manager.create_user('dperson@example.com') - >>> address_2 = list(user_2.addresses)[0] - >>> mlist.subscribe(address_2, MemberRole.member) - - - >>> handler.process(mlist, msg, - ... dict(original_sender='dperson@example.com')) - >>> get_queue_messages('virgin') - [] - - -Requested acknowledgments -========================= - -If the member requests acknowledgments, Mailman will send them one when they -post to the mailing list. - - >>> user_1.preferences.acknowledge_posts = True - -The receipt will include the original message's subject in the response body, - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Subject: Something witty and insightful - ... - ... """) - >>> handler.process(mlist, msg, {}) - >>> messages = get_queue_messages('virgin') - >>> len(messages) - 1 - >>> dump_msgdata(messages[0].msgdata) - _parsemsg : False - listname : test@example.com - nodecorate : True - recipients : set([u'aperson@example.com']) - reduced_list_headers: True - ... - >>> print messages[0].msg.as_string() - ... - MIME-Version: 1.0 - ... - Subject: Test post acknowledgment - From: test-bounces@example.com - To: aperson@example.com - ... - Precedence: bulk - - Your message entitled - - Something witty and insightful - - was successfully received by the Test mailing list. - - List info page: http://lists.example.com/listinfo/test@example.com - Your preferences: http://example.com/aperson@example.com - - -If there is no subject, then the receipt will use a generic message. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... - ... """) - >>> handler.process(mlist, msg, {}) - >>> messages = get_queue_messages('virgin') - >>> len(messages) - 1 - >>> dump_msgdata(messages[0].msgdata) - _parsemsg : False - listname : test@example.com - nodecorate : True - recipients : set([u'aperson@example.com']) - reduced_list_headers: True - ... - >>> print messages[0].msg.as_string() - MIME-Version: 1.0 - ... - Subject: Test post acknowledgment - From: test-bounces@example.com - To: aperson@example.com - ... - Precedence: bulk - - Your message entitled - - (no subject) - - was successfully received by the Test mailing list. - - List info page: http://lists.example.com/listinfo/test@example.com - Your preferences: http://example.com/aperson@example.com - diff --git a/src/mailman/pipeline/docs/after-delivery.rst b/src/mailman/pipeline/docs/after-delivery.rst deleted file mode 100644 index c3e393cf2..000000000 --- a/src/mailman/pipeline/docs/after-delivery.rst +++ /dev/null @@ -1,30 +0,0 @@ -============== -After delivery -============== - -After a message is delivered, or more correctly, after it has been processed -by the rest of the handlers in the incoming queue pipeline, a couple of -bookkeeping pieces of information are updated. - - >>> import datetime - >>> mlist = create_list('_xtest@example.com') - >>> post_time = datetime.datetime.now() - datetime.timedelta(minutes=10) - >>> mlist.last_post_time = post_time - >>> mlist.post_id = 10 - -Processing a message with this handler updates the last_post_time and post_id -attributes. -:: - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... - ... Something interesting. - ... """) - - >>> handler = config.handlers['after-delivery'] - >>> handler.process(mlist, msg, {}) - >>> mlist.last_post_time > post_time - True - >>> mlist.post_id - 11 diff --git a/src/mailman/pipeline/docs/archives.rst b/src/mailman/pipeline/docs/archives.rst deleted file mode 100644 index 323d121e8..000000000 --- a/src/mailman/pipeline/docs/archives.rst +++ /dev/null @@ -1,133 +0,0 @@ -======== -Archives -======== - -Updating the archives with posted messages is handled by a separate queue, -which allows for better memory management and prevents blocking the main -delivery processes while messages are archived. This also allows external -archivers to work in a separate process from the main Mailman delivery -processes. - - >>> handler = config.handlers['to-archive'] - >>> mlist = create_list('_xtest@example.com') - >>> switchboard = config.switchboards['archive'] - -A helper function. - - >>> def clear(): - ... for filebase in switchboard.files: - ... msg, msgdata = switchboard.dequeue(filebase) - ... switchboard.finish(filebase) - -The purpose of this handler is to make a simple decision as to whether the -message should get archived and if so, to drop the message in the archiving -queue. Really the most important things are to determine when a message -should *not* get archived. - -For example, no digests should ever get archived. - - >>> mlist.archive = True - >>> msg = message_from_string("""\ - ... Subject: A sample message - ... - ... A message of great import. - ... """) - >>> handler.process(mlist, msg, dict(isdigest=True)) - >>> switchboard.files - [] - -If the mailing list is not configured to archive, then even regular deliveries -won't be archived. - - >>> mlist.archive = False - >>> handler.process(mlist, msg, {}) - >>> switchboard.files - [] - -There are two de-facto standards for a message to indicate that it does not -want to be archived. We've seen both in the wild so both are supported. The -``X-No-Archive:`` header can be used to indicate that the message should not -be archived. Confusingly, this header's value is actually ignored. - - >>> mlist.archive = True - >>> msg = message_from_string("""\ - ... Subject: A sample message - ... X-No-Archive: YES - ... - ... A message of great import. - ... """) - >>> handler.process(mlist, msg, dict(isdigest=True)) - >>> switchboard.files - [] - -Even a ``no`` value will stop the archiving of the message. - - >>> msg = message_from_string("""\ - ... Subject: A sample message - ... X-No-Archive: No - ... - ... A message of great import. - ... """) - >>> handler.process(mlist, msg, dict(isdigest=True)) - >>> switchboard.files - [] - -Another header that's been observed is the ``X-Archive:`` header. Here, the -header's case folded value must be ``no`` in order to prevent archiving. - - >>> msg = message_from_string("""\ - ... Subject: A sample message - ... X-Archive: No - ... - ... A message of great import. - ... """) - >>> handler.process(mlist, msg, dict(isdigest=True)) - >>> switchboard.files - [] - -But if the value is ``yes``, then the message will be archived. - - >>> msg = message_from_string("""\ - ... Subject: A sample message - ... X-Archive: Yes - ... - ... A message of great import. - ... """) - >>> handler.process(mlist, msg, {}) - >>> len(switchboard.files) - 1 - >>> filebase = switchboard.files[0] - >>> qmsg, qdata = switchboard.dequeue(filebase) - >>> switchboard.finish(filebase) - >>> print qmsg.as_string() - Subject: A sample message - X-Archive: Yes - - A message of great import. - - >>> dump_msgdata(qdata) - _parsemsg: False - version : 3 - -Without either archiving header, and all other things being the same, the -message will get archived. - - >>> msg = message_from_string("""\ - ... Subject: A sample message - ... - ... A message of great import. - ... """) - >>> handler.process(mlist, msg, {}) - >>> len(switchboard.files) - 1 - >>> filebase = switchboard.files[0] - >>> qmsg, qdata = switchboard.dequeue(filebase) - >>> switchboard.finish(filebase) - >>> print qmsg.as_string() - Subject: A sample message - - A message of great import. - - >>> dump_msgdata(qdata) - _parsemsg: False - version : 3 diff --git a/src/mailman/pipeline/docs/avoid-duplicates.rst b/src/mailman/pipeline/docs/avoid-duplicates.rst deleted file mode 100644 index 1e46793c2..000000000 --- a/src/mailman/pipeline/docs/avoid-duplicates.rst +++ /dev/null @@ -1,175 +0,0 @@ -================ -Avoid duplicates -================ - -This handler implements several strategies to reduce the reception of -duplicate messages. It does this by removing certain recipients from the list -of recipients calculated earlier. - - >>> mlist = create_list('_xtest@example.com') - -Create some members we're going to use. -:: - - >>> from mailman.interfaces.usermanager import IUserManager - >>> from zope.component import getUtility - >>> user_manager = getUtility(IUserManager) - - >>> address_a = user_manager.create_address('aperson@example.com') - >>> address_b = user_manager.create_address('bperson@example.com') - - >>> from mailman.interfaces.member import MemberRole - >>> member_a = mlist.subscribe(address_a, MemberRole.member) - >>> member_b = mlist.subscribe(address_b, MemberRole.member) - >>> # This is the message metadata dictionary as it would be produced by - >>> # the CalcRecips handler. - >>> recips = dict( - ... recipients=['aperson@example.com', 'bperson@example.com']) - - -Short circuiting -================ - -The module short-circuits if there are no recipients. -:: - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Subject: A message of great import - ... - ... Something - ... """) - >>> msgdata = {} - - >>> handler = config.handlers['avoid-duplicates'] - >>> handler.process(mlist, msg, msgdata) - >>> msgdata - {} - >>> print msg.as_string() - From: aperson@example.com - Subject: A message of great import - - Something - - - -Suppressing the list copy -========================= - -Members can elect not to receive a list copy of any message on which they are -explicitly named as a recipient. This is done by setting their -``receive_list_copy`` preference to ``False``. However, if they aren't -mentioned in one of the recipient headers (i.e. ``To``, ``CC``, ``Resent-To``, -or ``Resent-CC``), then they will get a list copy. - - >>> member_a.preferences.receive_list_copy = False - >>> msg = message_from_string("""\ - ... From: Claire Person - ... - ... Something of great import. - ... """) - >>> msgdata = recips.copy() - >>> handler.process(mlist, msg, msgdata) - >>> sorted(msgdata['recipients']) - [u'aperson@example.com', u'bperson@example.com'] - >>> print msg.as_string() - From: Claire Person - - Something of great import. - - -If they're mentioned on the ``CC`` line, they won't get a list copy. - - >>> msg = message_from_string("""\ - ... From: Claire Person - ... CC: aperson@example.com - ... - ... Something of great import. - ... """) - >>> msgdata = recips.copy() - >>> handler.process(mlist, msg, msgdata) - >>> sorted(msgdata['recipients']) - [u'bperson@example.com'] - >>> print msg.as_string() - From: Claire Person - CC: aperson@example.com - - Something of great import. - - -But if they're mentioned on the ``CC`` line and have ``receive_list_copy`` set -to ``True`` (the default), then they still get a list copy. - - >>> msg = message_from_string("""\ - ... From: Claire Person - ... CC: bperson@example.com - ... - ... Something of great import. - ... """) - >>> msgdata = recips.copy() - >>> handler.process(mlist, msg, msgdata) - >>> sorted(msgdata['recipients']) - [u'aperson@example.com', u'bperson@example.com'] - >>> print msg.as_string() - From: Claire Person - CC: bperson@example.com - - Something of great import. - - -Other headers checked for recipients include the ``To``... - - >>> msg = message_from_string("""\ - ... From: Claire Person - ... To: aperson@example.com - ... - ... Something of great import. - ... """) - >>> msgdata = recips.copy() - >>> handler.process(mlist, msg, msgdata) - >>> sorted(msgdata['recipients']) - [u'bperson@example.com'] - >>> print msg.as_string() - From: Claire Person - To: aperson@example.com - - Something of great import. - - -... ``Resent-To`` ... - - >>> msg = message_from_string("""\ - ... From: Claire Person - ... Resent-To: aperson@example.com - ... - ... Something of great import. - ... """) - >>> msgdata = recips.copy() - >>> handler.process(mlist, msg, msgdata) - >>> sorted(msgdata['recipients']) - [u'bperson@example.com'] - >>> print msg.as_string() - From: Claire Person - Resent-To: aperson@example.com - - Something of great import. - - -...and ``Resent-CC`` headers. - - >>> msg = message_from_string("""\ - ... From: Claire Person - ... Resent-Cc: aperson@example.com - ... - ... Something of great import. - ... """) - >>> msgdata = recips.copy() - >>> handler.process(mlist, msg, msgdata) - >>> sorted(msgdata['recipients']) - [u'bperson@example.com'] - >>> print msg.as_string() - From: Claire Person - Resent-Cc: aperson@example.com - - Something of great import. - diff --git a/src/mailman/pipeline/docs/calc-recips.rst b/src/mailman/pipeline/docs/calc-recips.rst deleted file mode 100644 index 6dca85816..000000000 --- a/src/mailman/pipeline/docs/calc-recips.rst +++ /dev/null @@ -1,114 +0,0 @@ -====================== -Calculating recipients -====================== - -Every message that makes it through to the list membership gets sent to a set -of recipient addresses. These addresses are calculated by one of the handler -modules and depends on a host of factors. - - >>> mlist = create_list('_xtest@example.com') - -Recipients are calculate from the list members, so add a bunch of members to -start out with. First, create a bunch of addresses... -:: - - >>> from mailman.interfaces.usermanager import IUserManager - >>> from zope.component import getUtility - >>> user_manager = getUtility(IUserManager) - - >>> address_a = user_manager.create_address('aperson@example.com') - >>> address_b = user_manager.create_address('bperson@example.com') - >>> address_c = user_manager.create_address('cperson@example.com') - >>> address_d = user_manager.create_address('dperson@example.com') - >>> address_e = user_manager.create_address('eperson@example.com') - >>> address_f = user_manager.create_address('fperson@example.com') - -...then subscribe these addresses to the mailing list as members... - - >>> from mailman.interfaces.member import MemberRole - >>> member_a = mlist.subscribe(address_a, MemberRole.member) - >>> member_b = mlist.subscribe(address_b, MemberRole.member) - >>> member_c = mlist.subscribe(address_c, MemberRole.member) - >>> member_d = mlist.subscribe(address_d, MemberRole.member) - >>> member_e = mlist.subscribe(address_e, MemberRole.member) - >>> member_f = mlist.subscribe(address_f, MemberRole.member) - -...then make some of the members digest members. - - >>> from mailman.core.constants import DeliveryMode - >>> member_d.preferences.delivery_mode = DeliveryMode.plaintext_digests - >>> member_e.preferences.delivery_mode = DeliveryMode.mime_digests - >>> member_f.preferences.delivery_mode = DeliveryMode.summary_digests - - -Short-circuiting -================ - -Sometimes, the list of recipients already exists in the message metadata. -This can happen for example, when a message was previously delivered to some -but not all of the recipients. -:: - - >>> msg = message_from_string("""\ - ... From: Xavier Person - ... - ... Something of great import. - ... """) - >>> recipients = set(('qperson@example.com', 'zperson@example.com')) - >>> msgdata = dict(recipients=recipients) - - >>> handler = config.handlers['calculate-recipients'] - >>> handler.process(mlist, msg, msgdata) - >>> dump_list(msgdata['recipients']) - qperson@example.com - zperson@example.com - - -Regular delivery recipients -=========================== - -Regular delivery recipients are those people who get messages from the list as -soon as they are posted. In other words, these folks are not digest members. - - >>> msgdata = {} - >>> handler.process(mlist, msg, msgdata) - >>> dump_list(msgdata['recipients']) - aperson@example.com - bperson@example.com - cperson@example.com - -Members can elect not to receive a list copy of their own postings. - - >>> member_c.preferences.receive_own_postings = False - >>> msg = message_from_string("""\ - ... From: Claire Person - ... - ... Something of great import. - ... """) - >>> msgdata = {} - >>> handler.process(mlist, msg, msgdata) - >>> dump_list(msgdata['recipients']) - aperson@example.com - bperson@example.com - -Members can also elect not to receive a list copy of any message on which they -are explicitly named as a recipient. However, see the `avoid duplicates`_ -handler for details. - - -Digest recipients -================= - -XXX Test various digest deliveries. - - -Urgent messages -=============== - -XXX Test various urgent deliveries: - * test_urgent_moderator() - * test_urgent_admin() - * test_urgent_reject() - - -.. _`avoid duplicates`: avoid-duplicates.html diff --git a/src/mailman/pipeline/docs/cleanse.rst b/src/mailman/pipeline/docs/cleanse.rst deleted file mode 100644 index 61dfa8f52..000000000 --- a/src/mailman/pipeline/docs/cleanse.rst +++ /dev/null @@ -1,100 +0,0 @@ -================= -Cleansing headers -================= - -All messages posted to a list get their headers cleansed. Some headers are -related to additional permissions that can be granted to the message and other -headers can be used to fish for membership. - - >>> mlist = create_list('_xtest@example.com') - -Headers such as ``Approved``, ``Approve``, (as well as their ``X-`` variants) -and ``Urgent`` are used to grant special permissions to individual messages. -All may contain a password; the first two headers are used by list -administrators to pre-approve a message normal held for approval. The latter -header is used to send a regular message to all members, regardless of whether -they get digests or not. Because all three headers contain passwords, they -must be removed from any posted message. :: - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Approved: foobar - ... Approve: barfoo - ... X-Approved: bazbar - ... X-Approve: barbaz - ... Urgent: notreally - ... Subject: A message of great import - ... - ... Blah blah blah - ... """) - - >>> handler = config.handlers['cleanse'] - >>> handler.process(mlist, msg, {}) - >>> print msg.as_string() - From: aperson@example.com - Subject: A message of great import - - Blah blah blah - - -Other headers can be used by list members to fish the list for membership, so -we don't let them go through. These are a mix of standard headers and custom -headers supported by some mail readers. For example, ``X-PMRC`` is supported -by Pegasus mail. I don't remember what program uses ``X-Confirm-Reading-To`` -though (Some Microsoft product perhaps?). - - >>> msg = message_from_string("""\ - ... From: bperson@example.com - ... Reply-To: bperson@example.org - ... Sender: asystem@example.net - ... Return-Receipt-To: another@example.com - ... Disposition-Notification-To: athird@example.com - ... X-Confirm-Reading-To: afourth@example.com - ... X-PMRQC: afifth@example.com - ... Subject: a message to you - ... - ... How are you doing? - ... """) - >>> handler.process(mlist, msg, {}) - >>> print msg.as_string() - From: bperson@example.com - Reply-To: bperson@example.org - Sender: asystem@example.net - Subject: a message to you - - How are you doing? - - - -Anonymous lists -=============== - -Anonymous mailing lists also try to cleanse certain identifying headers from -the original posting, so that it is at least a bit more difficult to determine -who sent the message. This isn't perfect though, for example, the body of the -messages are never scrubbed (though that might not be a bad idea). The -``From`` and ``Reply-To`` headers in the posted message are taken from list -attributes. - -Hotmail apparently sets ``X-Originating-Email``. - - >>> mlist.anonymous_list = True - >>> mlist.description = 'A Test Mailing List' - >>> mlist.preferred_language = 'en' - >>> msg = message_from_string("""\ - ... From: bperson@example.com - ... Reply-To: bperson@example.org - ... Sender: asystem@example.net - ... X-Originating-Email: cperson@example.com - ... Subject: a message to you - ... - ... How are you doing? - ... """) - >>> handler.process(mlist, msg, {}) - >>> print msg.as_string() - Subject: a message to you - From: A Test Mailing List <_xtest@example.com> - Reply-To: _xtest@example.com - - How are you doing? - diff --git a/src/mailman/pipeline/docs/cook-headers.rst b/src/mailman/pipeline/docs/cook-headers.rst deleted file mode 100644 index e0313f53a..000000000 --- a/src/mailman/pipeline/docs/cook-headers.rst +++ /dev/null @@ -1,129 +0,0 @@ -=============== -Cooking headers -=============== - -Messages that flow through the global pipeline get their headers 'cooked', -which basically means that their headers go through several mostly unrelated -transformations. Some headers get added, others get changed. Some of these -changes depend on mailing list settings and others depend on how the message -is getting sent through the system. We'll take things one-by-one. - - >>> mlist = create_list('test@example.com') - >>> mlist.subject_prefix = '' - - -Saving the original sender -========================== - -Because the original sender headers may get deleted or changed, this handler -will place the sender in the message metadata for safe keeping. -:: - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... - ... A message of great import. - ... """) - >>> msgdata = {} - - >>> from mailman.pipeline.cook_headers import process - >>> process(mlist, msg, msgdata) - >>> print msgdata['original_sender'] - aperson@example.com - -But if there was no original sender, then the empty string will be saved. - - >>> msg = message_from_string("""\ - ... Subject: No original sender - ... - ... A message of great import. - ... """) - >>> msgdata = {} - >>> process(mlist, msg, msgdata) - >>> print msgdata['original_sender'] - - - -Mailman version header -====================== - -Mailman will also insert an ``X-Mailman-Version`` header... - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... - ... A message of great import. - ... """) - >>> process(mlist, msg, {}) - >>> from mailman.version import VERSION - >>> msg['x-mailman-version'] == VERSION - True - -...but only if one doesn't already exist. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... X-Mailman-Version: 3000 - ... - ... A message of great import. - ... """) - >>> process(mlist, msg, {}) - >>> print msg['x-mailman-version'] - 3000 - - -Precedence header -================= - -Mailman will insert a ``Precedence`` header, which is a de-facto standard for -telling automatic reply software (e.g. ``vacation(1)``) not to respond to this -message. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... - ... A message of great import. - ... """) - >>> process(mlist, msg, {}) - >>> print msg['precedence'] - list - -But Mailman will only add that header if the original message doesn't already -have one of them. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Precedence: junk - ... - ... A message of great import. - ... """) - >>> process(mlist, msg, {}) - >>> print msg['precedence'] - junk - - -Personalization -=============== - -The ``To`` field normally contains the list posting address. However when -messages are fully personalized, that header will get overwritten with the -address of the recipient. The list's posting address will be added to one of -the recipient headers so that users will be able to reply back to the list. - - >>> from mailman.interfaces.mailinglist import ( - ... Personalization, ReplyToMunging) - >>> mlist.personalize = Personalization.full - >>> mlist.reply_goes_to_list = ReplyToMunging.no_munging - >>> mlist.description = 'My test mailing list' - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... - ... """) - >>> process(mlist, msg, {}) - >>> print msg.as_string() - From: aperson@example.com - X-Mailman-Version: ... - Precedence: list - Cc: My test mailing list - - diff --git a/src/mailman/pipeline/docs/decorate.rst b/src/mailman/pipeline/docs/decorate.rst deleted file mode 100644 index 6fa8212ac..000000000 --- a/src/mailman/pipeline/docs/decorate.rst +++ /dev/null @@ -1,348 +0,0 @@ -================== -Message decoration -================== - -Message decoration is the process of adding headers and footers to the -original message. A handler module takes care of this based on the settings -of the mailing list and the type of message being processed. - - >>> mlist = create_list('_xtest@example.com') - >>> msg_text = """\ - ... From: aperson@example.org - ... - ... Here is a message. - ... """ - >>> msg = message_from_string(msg_text) - - -Short circuiting -================ - -Digest messages get decorated during the digest creation phase so no extra -decorations are added for digest messages. -:: - - >>> from mailman.pipeline.decorate import process - >>> process(mlist, msg, dict(isdigest=True)) - >>> print msg.as_string() - From: aperson@example.org - - Here is a message. - - >>> process(mlist, msg, dict(nodecorate=True)) - >>> print msg.as_string() - From: aperson@example.org - - Here is a message. - - -Simple decorations -================== - -Message decorations are specified by URI and can be specialized by the mailing -list and language. Internal Mailman decorations can be referenced by using -the ``mailman://`` URL scheme. Here we create a simple English header and -footer for all mailing lists in our site. -:: - - >>> import os, tempfile - >>> template_dir = tempfile.mkdtemp() - >>> site_dir = os.path.join(template_dir, 'site', 'en') - >>> os.makedirs(site_dir) - >>> config.push('templates', """ - ... [paths.testing] - ... template_dir: {0} - ... """.format(template_dir)) - - >>> myheader_path = os.path.join(site_dir, 'myheader.txt') - >>> with open(myheader_path, 'w') as fp: - ... print >> fp, 'header' - >>> myfooter_path = os.path.join(site_dir, 'myfooter.txt') - >>> with open(myfooter_path, 'w') as fp: - ... print >> fp, 'footer' - -Setting these attributes on the mailing list causes it to use these -templates. Since these are site-global templates, we can use a shorter path. - - >>> mlist.header_uri = 'mailman:///myheader.txt' - >>> mlist.footer_uri = 'mailman:///myfooter.txt' - -Text messages that have no declared content type are, by default encoded in -ASCII. When the mailing list's preferred language is ``en`` (i.e. English), -the character set of the mailing list and of the message will match, allowing -Mailman to simply prepend the header and append the footer verbatim. - - >>> mlist.preferred_language = 'en' - >>> process(mlist, msg, {}) - >>> print msg.as_string() - From: aperson@example.org - ... - - header - Here is a message. - footer - -Mailman supports a number of interpolation variables, placeholders in the -header and footer for information to be filled in with mailing list specific -data. An example of such information is the mailing list's `real name` (a -short descriptive name for the mailing list). -:: - - >>> with open(myheader_path, 'w') as fp: - ... print >> fp, '$display_name header' - >>> with open(myfooter_path, 'w') as fp: - ... print >> fp, '$display_name footer' - - >>> msg = message_from_string(msg_text) - >>> mlist.display_name = 'XTest' - >>> process(mlist, msg, {}) - >>> print msg.as_string() - From: aperson@example.org - ... - XTest header - Here is a message. - XTest footer - -You can't just pick any interpolation variable though; if you do, the variable -will remain in the header or footer unchanged. -:: - - >>> with open(myheader_path, 'w') as fp: - ... print >> fp, '$dummy header' - >>> with open(myfooter_path, 'w') as fp: - ... print >> fp, '$dummy footer' - - >>> msg = message_from_string(msg_text) - >>> process(mlist, msg, {}) - >>> print msg.as_string() - From: aperson@example.org - ... - $dummy header - Here is a message. - $dummy footer - - -Handling RFC 3676 'format=flowed' parameters -============================================ - -RFC 3676 describes a standard by which text/plain messages can marked by -generating MUAs for better readability in compatible receiving MUAs. The -``format`` parameter on the text/plain ``Content-Type`` header gives hints as -to how the receiving MUA may flow and delete trailing whitespace for better -display in a proportional font. - -When Mailman sees text/plain messages with such RFC 3676 parameters, it -preserves these parameters when it concatenates headers and footers to the -message payload. -:: - - >>> with open(myheader_path, 'w') as fp: - ... print >> fp, 'header' - >>> with open(myfooter_path, 'w') as fp: - ... print >> fp, 'footer' - - >>> mlist.preferred_language = 'en' - >>> msg = message_from_string("""\ - ... From: aperson@example.org - ... Content-Type: text/plain; format=flowed; delsp=no - ... - ... Here is a message\x20 - ... with soft line breaks. - ... """) - >>> process(mlist, msg, {}) - >>> # Don't use 'print' here as above because it won't be obvious from the - >>> # output that the soft-line break space at the end of the 'Here is a - >>> # message' line will be retained in the output. - >>> print msg['content-type'] - text/plain; format="flowed"; delsp="no"; charset="us-ascii" - >>> for line in msg.get_payload().splitlines(): - ... print '>{0}<'.format(line) - >header< - >Here is a message < - >with soft line breaks.< - >footer< - - -Decorating mixed-charset messages -================================= - -When a message has no explicit character set, it is assumed to be ASCII. -However, if the mailing list's preferred language has a different character -set, Mailman will still try to concatenate the header and footer, but it will -convert the text to utf-8 and base-64 encode the message payload. -:: - - # 'ja' = Japanese; charset = 'euc-jp' - >>> mlist.preferred_language = 'ja' - - >>> with open(myheader_path, 'w') as fp: - ... print >> fp, '$description header' - >>> with open(myfooter_path, 'w') as fp: - ... print >> fp, '$description footer' - >>> mlist.description = '\u65e5\u672c\u8a9e' - - >>> from email.message import Message - >>> msg = Message() - >>> msg.set_payload('Fran\xe7aise', 'iso-8859-1') - >>> print msg.as_string() - MIME-Version: 1.0 - Content-Type: text/plain; charset="iso-8859-1" - Content-Transfer-Encoding: quoted-printable - - Fran=E7aise - >>> process(mlist, msg, {}) - >>> print msg.as_string() - MIME-Version: 1.0 - Content-Type: text/plain; charset="utf-8" - Content-Transfer-Encoding: base64 - - 5pel5pys6KqeIGhlYWRlcgpGcmFuw6dhaXNlCuaXpeacrOiqniBmb290ZXIK - -Sometimes the message even has an unknown character set. In this case, -Mailman has no choice but to decorate the original message with MIME -attachments. -:: - - >>> mlist.preferred_language = 'en' - >>> with open(myheader_path, 'w') as fp: - ... print >> fp, 'header' - >>> with open(myfooter_path, 'w') as fp: - ... print >> fp, 'footer' - - >>> msg = message_from_string("""\ - ... From: aperson@example.org - ... Content-Type: text/plain; charset=unknown - ... Content-Transfer-Encoding: 7bit - ... - ... Here is a message. - ... """) - - >>> process(mlist, msg, {}) - >>> msg.set_boundary('BOUNDARY') - >>> print msg.as_string() - From: aperson@example.org - Content-Type: multipart/mixed; boundary="BOUNDARY" - - --BOUNDARY - Content-Type: text/plain; charset="us-ascii" - MIME-Version: 1.0 - Content-Transfer-Encoding: 7bit - Content-Disposition: inline - - header - --BOUNDARY - Content-Type: text/plain; charset=unknown - Content-Transfer-Encoding: 7bit - - Here is a message. - - --BOUNDARY - Content-Type: text/plain; charset="us-ascii" - MIME-Version: 1.0 - Content-Transfer-Encoding: 7bit - Content-Disposition: inline - - footer - --BOUNDARY-- - - -Decorating multipart messages -============================= - -Multipart messages have to be decorated differently. The header and footer -cannot be simply concatenated into the payload because that will break the -MIME structure of the message. Instead, the header and footer are attached as -separate MIME subparts. - -When the outer part is ``multipart/mixed``, the header and footer can have a -``Content-Disposition`` of ``inline`` so that MUAs can display these headers -as if they were simply concatenated. - - >>> part_1 = message_from_string("""\ - ... From: aperson@example.org - ... - ... Here is the first message. - ... """) - >>> part_2 = message_from_string("""\ - ... From: bperson@example.com - ... - ... Here is the second message. - ... """) - >>> from email.mime.multipart import MIMEMultipart - >>> msg = MIMEMultipart('mixed', boundary='BOUNDARY', - ... _subparts=(part_1, part_2)) - >>> process(mlist, msg, {}) - >>> print msg.as_string() - Content-Type: multipart/mixed; boundary="BOUNDARY" - MIME-Version: 1.0 - - --BOUNDARY - Content-Type: text/plain; charset="us-ascii" - MIME-Version: 1.0 - Content-Transfer-Encoding: 7bit - Content-Disposition: inline - - header - --BOUNDARY - From: aperson@example.org - - Here is the first message. - - --BOUNDARY - From: bperson@example.com - - Here is the second message. - - --BOUNDARY - Content-Type: text/plain; charset="us-ascii" - MIME-Version: 1.0 - Content-Transfer-Encoding: 7bit - Content-Disposition: inline - - footer - --BOUNDARY-- - - -Decorating other content types -============================== - -Non-multipart non-text content types will get wrapped in a ``multipart/mixed`` -so that the header and footer can be added as attachments. - - >>> msg = message_from_string("""\ - ... From: aperson@example.org - ... Content-Type: image/x-beautiful - ... - ... IMAGEDATAIMAGEDATAIMAGEDATA - ... """) - >>> process(mlist, msg, {}) - >>> msg.set_boundary('BOUNDARY') - >>> print msg.as_string() - From: aperson@example.org - ... - --BOUNDARY - Content-Type: text/plain; charset="us-ascii" - MIME-Version: 1.0 - Content-Transfer-Encoding: 7bit - Content-Disposition: inline - - header - --BOUNDARY - Content-Type: image/x-beautiful - - IMAGEDATAIMAGEDATAIMAGEDATA - - --BOUNDARY - Content-Type: text/plain; charset="us-ascii" - MIME-Version: 1.0 - Content-Transfer-Encoding: 7bit - Content-Disposition: inline - - footer - --BOUNDARY-- - -.. Clean up - - >>> config.pop('templates') - >>> import shutil - >>> shutil.rmtree(template_dir) diff --git a/src/mailman/pipeline/docs/digests.rst b/src/mailman/pipeline/docs/digests.rst deleted file mode 100644 index d4d563180..000000000 --- a/src/mailman/pipeline/docs/digests.rst +++ /dev/null @@ -1,113 +0,0 @@ -======= -Digests -======= - -Digests are a way for a user to receive list traffic in collections instead of -as individual messages when immediately posted. There are several forms of -digests, although only two are currently supported: MIME digests and RFC 1153 -(a.k.a. plain text) digests. - - >>> mlist = create_list('xtest@example.com') - -This is a helper function used to iterate through all the accumulated digest -messages, in the order in which they were posted. This makes it easier to -update the tests when we switch to a different mailbox format. -:: - - >>> from mailman.testing.helpers import digest_mbox - >>> from itertools import count - >>> from string import Template - - >>> def message_factory(): - ... for i in count(1): - ... text = Template("""\ - ... From: aperson@example.com - ... To: xtest@example.com - ... Subject: Test message $i - ... - ... Here is message $i - ... """).substitute(i=i) - ... yield message_from_string(text) - >>> message_factory = message_factory() - - -Short circuiting -================ - -When a message is posted to the mailing list, it is generally added to a -mailbox, unless the mailing list does not allow digests. - - >>> mlist.digestable = False - >>> msg = next(message_factory) - >>> process = config.handlers['to-digest'].process - >>> process(mlist, msg, {}) - >>> sum(1 for msg in digest_mbox(mlist)) - 0 - >>> digest_queue = config.switchboards['digest'] - >>> digest_queue.files - [] - -...or they may allow digests but the message is already a digest. - - >>> mlist.digestable = True - >>> process(mlist, msg, dict(isdigest=True)) - >>> sum(1 for msg in digest_mbox(mlist)) - 0 - >>> digest_queue.files - [] - - -Sending a digest -================ - -For messages which are not digests, but which are posted to a digesting -mailing list, the messages will be stored until they reach a criteria -triggering the sending of the digest. If none of those criteria are met, then -the message will just sit in the mailbox for a while. - - >>> mlist.digest_size_threshold = 10000 - >>> process(mlist, msg, {}) - >>> digest_queue.files - [] - >>> digest = digest_mbox(mlist) - >>> sum(1 for msg in digest) - 1 - >>> import os - >>> os.remove(digest._path) - -When the size of the digest mailbox reaches the maximum size threshold, a -marker message is placed into the digest runner's queue. The digest is not -actually crafted by the handler. - - >>> mlist.digest_size_threshold = 1 - >>> mlist.volume = 2 - >>> mlist.next_digest_number = 10 - >>> size = 0 - >>> for msg in message_factory: - ... process(mlist, msg, {}) - ... size += len(str(msg)) - ... if size >= mlist.digest_size_threshold * 1024: - ... break - - >>> sum(1 for msg in digest_mbox(mlist)) - 0 - >>> len(digest_queue.files) - 1 - -The digest has been moved to a unique file. - - >>> from mailman.utilities.mailbox import Mailbox - >>> from mailman.testing.helpers import get_queue_messages - >>> item = get_queue_messages('digest')[0] - >>> for msg in Mailbox(item.msgdata['digest_path']): - ... print msg['subject'] - Test message 2 - Test message 3 - Test message 4 - Test message 5 - Test message 6 - Test message 7 - Test message 8 - Test message 9 - -Digests are actually crafted and sent by a separate digest runner. diff --git a/src/mailman/pipeline/docs/file-recips.rst b/src/mailman/pipeline/docs/file-recips.rst deleted file mode 100644 index 7d157ccc5..000000000 --- a/src/mailman/pipeline/docs/file-recips.rst +++ /dev/null @@ -1,111 +0,0 @@ -=============== -File recipients -=============== - -Mailman can calculate the recipients for a message from a Sendmail-style -include file. This file must be called ``members.txt`` and it must live in -the list's data directory. - - >>> mlist = create_list('_xtest@example.com') - - -Short circuiting -================ - -If the message's metadata already has recipients, this handler immediately -returns. -:: - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... - ... A message. - ... """) - >>> msgdata = {'recipients': 7} - - >>> handler = config.handlers['file-recipients'] - >>> handler.process(mlist, msg, msgdata) - >>> print msg.as_string() - From: aperson@example.com - - A message. - - >>> dump_msgdata(msgdata) - recipients: 7 - - -Missing file -============ - -The include file must live inside the list's data directory, under the name -``members.txt``. If the file doesn't exist, the list of recipients will be -empty. - - >>> import os - >>> file_path = os.path.join(mlist.data_path, 'members.txt') - >>> open(file_path) - Traceback (most recent call last): - ... - IOError: [Errno ...] - No such file or directory: u'.../_xtest@example.com/members.txt' - >>> msgdata = {} - >>> handler.process(mlist, msg, msgdata) - >>> dump_list(msgdata['recipients']) - *Empty* - - -Existing file -============= - -If the file exists, it contains a list of addresses, one per line. These -addresses are returned as the set of recipients. -:: - - >>> fp = open(file_path, 'w') - >>> try: - ... print >> fp, 'bperson@example.com' - ... print >> fp, 'cperson@example.com' - ... print >> fp, 'dperson@example.com' - ... print >> fp, 'eperson@example.com' - ... print >> fp, 'fperson@example.com' - ... print >> fp, 'gperson@example.com' - ... finally: - ... fp.close() - - >>> msgdata = {} - >>> handler.process(mlist, msg, msgdata) - >>> dump_list(msgdata['recipients']) - bperson@example.com - cperson@example.com - dperson@example.com - eperson@example.com - fperson@example.com - gperson@example.com - -However, if the sender of the original message is a member of the list and -their address is in the include file, the sender's address is *not* included -in the recipients list. -:: - - >>> from mailman.interfaces.usermanager import IUserManager - >>> from zope.component import getUtility - >>> address_1 = getUtility(IUserManager).create_address( - ... 'cperson@example.com') - - >>> from mailman.interfaces.member import MemberRole - >>> mlist.subscribe(address_1, MemberRole.member) - - - >>> msg = message_from_string("""\ - ... From: cperson@example.com - ... - ... A message. - ... """) - >>> msgdata = {} - >>> handler.process(mlist, msg, msgdata) - >>> dump_list(msgdata['recipients']) - bperson@example.com - dperson@example.com - eperson@example.com - fperson@example.com - gperson@example.com diff --git a/src/mailman/pipeline/docs/filtering.rst b/src/mailman/pipeline/docs/filtering.rst deleted file mode 100644 index fd0b33d3b..000000000 --- a/src/mailman/pipeline/docs/filtering.rst +++ /dev/null @@ -1,347 +0,0 @@ -================= -Content filtering -================= - -Mailman can filter the content of messages posted to a mailing list by -stripping MIME subparts, and possibly reorganizing the MIME structure of a -message. - - >>> mlist = create_list('test@example.com') - -Several mailing list options control content filtering. First, the feature -must be enabled, then there are two options that control which MIME types get -filtered and which get passed. Finally, there is an option to control whether -``text/html`` parts will get converted to plain text. Let's set up some -defaults for these variables, then we'll explain them in more detail below. - - >>> mlist.filter_content = True - >>> mlist.filter_types = [] - >>> mlist.pass_types = [] - >>> mlist.convert_html_to_plaintext = False - - -Filtering the outer content type -================================ - -A simple filtering setting will just search the content types of the messages -parts, discarding all parts with a matching MIME type. If the message's outer -content type matches the filter, the entire message will be discarded. -:: - - >>> from mailman.interfaces.mime import FilterAction - - >>> mlist.filter_types = ['image/jpeg'] - >>> mlist.filter_action = FilterAction.discard - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Content-Type: image/jpeg - ... MIME-Version: 1.0 - ... - ... xxxxx - ... """) - - >>> process = config.handlers['mime-delete'].process - >>> process(mlist, msg, {}) - Traceback (most recent call last): - ... - DiscardMessage: The message's content type was explicitly disallowed - -However, if we turn off content filtering altogether, then the handler -short-circuits. - - >>> mlist.filter_content = False - >>> msgdata = {} - >>> process(mlist, msg, msgdata) - >>> print msg.as_string() - From: aperson@example.com - Content-Type: image/jpeg - MIME-Version: 1.0 - - xxxxx - >>> msgdata - {} - -Similarly, no content filtering is performed on digest messages, which are -crafted internally by Mailman. - - >>> mlist.filter_content = True - >>> msgdata = {'isdigest': True} - >>> process(mlist, msg, msgdata) - >>> print msg.as_string() - From: aperson@example.com - Content-Type: image/jpeg - MIME-Version: 1.0 - - xxxxx - >>> msgdata - {u'isdigest': True} - - -Simple multipart filtering -========================== - -If one of the subparts in a multipart message matches the filter type, then -just that subpart will be stripped. -:: - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Content-Type: multipart/mixed; boundary=BOUNDARY - ... MIME-Version: 1.0 - ... - ... --BOUNDARY - ... Content-Type: image/jpeg - ... MIME-Version: 1.0 - ... - ... xxx - ... - ... --BOUNDARY - ... Content-Type: image/gif - ... MIME-Version: 1.0 - ... - ... yyy - ... --BOUNDARY-- - ... """) - - >>> process(mlist, msg, {}) - >>> print msg.as_string() - From: aperson@example.com - Content-Type: multipart/mixed; boundary=BOUNDARY - MIME-Version: 1.0 - X-Content-Filtered-By: Mailman/MimeDel ... - - --BOUNDARY - Content-Type: image/gif - MIME-Version: 1.0 - - yyy - --BOUNDARY-- - - - -Collapsing multipart/alternative messages -========================================= - -When content filtering encounters a ``multipart/alternative`` part, and the -results of filtering leave only one of the subparts, then the -``multipart/alternative`` may be collapsed. For example, in the following -message, the outer content type is a ``multipart/mixed``. Inside this part is -just a single subpart that has a content type of ``multipart/alternative``. -This inner multipart has two subparts, a jpeg and a gif. - -Content filtering will remove the jpeg part, leaving the -``multipart/alternative`` with only a single gif subpart. Because there's -only one subpart left, the MIME structure of the message will be reorganized, -removing the inner ``multipart/alternative`` so that the outer -``multipart/mixed`` has just a single gif subpart. - - >>> mlist.collapse_alternatives = True - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Content-Type: multipart/mixed; boundary=BOUNDARY - ... MIME-Version: 1.0 - ... - ... --BOUNDARY - ... Content-Type: multipart/alternative; boundary=BOUND2 - ... MIME-Version: 1.0 - ... - ... --BOUND2 - ... Content-Type: image/jpeg - ... MIME-Version: 1.0 - ... - ... xxx - ... - ... --BOUND2 - ... Content-Type: image/gif - ... MIME-Version: 1.0 - ... - ... yyy - ... --BOUND2-- - ... - ... --BOUNDARY-- - ... """) - >>> process(mlist, msg, {}) - >>> print msg.as_string() - From: aperson@example.com - Content-Type: multipart/mixed; boundary=BOUNDARY - MIME-Version: 1.0 - X-Content-Filtered-By: Mailman/MimeDel ... - - --BOUNDARY - Content-Type: image/gif - MIME-Version: 1.0 - - yyy - --BOUNDARY-- - - -When the outer part is a ``multipart/alternative`` and filtering leaves this -outer part with just one subpart, the entire message is converted to the left -over part's content type. In other words, the left over inner part is -promoted to being the outer part. -:: - - >>> mlist.filter_types = ['image/jpeg', 'text/html'] - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Content-Type: multipart/alternative; boundary=AAA - ... - ... --AAA - ... Content-Type: text/html - ... - ... This is some html - ... --AAA - ... Content-Type: text/plain - ... - ... This is plain text - ... --AAA-- - ... """) - - >>> process(mlist, msg, {}) - >>> print msg.as_string() - From: aperson@example.com - Content-Type: text/plain - X-Content-Filtered-By: Mailman/MimeDel ... - - This is plain text - -Clean up. - - >>> mlist.filter_types = ['image/jpeg'] - - -Conversion to plain text -======================== - -Many mailing lists prohibit HTML email, and in fact, such email can be a -phishing or spam vector. However, many mail readers will send HTML email by -default because users think it looks pretty. One approach to handling this -would be to filter out ``text/html`` parts and rely on -``multipart/alternative`` collapsing to leave just a plain text part. This -works because many mail readers that send HTML email actually send a plain -text part in the second subpart of such ``multipart/alternatives``. - -While this is a good suggestion for plain text-only mailing lists, often a -mail reader will send only a ``text/html`` part with no plain text -alternative. in this case, the site administer can enable ``text/html`` to -``text/plain`` conversion by defining a conversion command. A list -administrator still needs to enable such conversion for their list though. - - >>> mlist.convert_html_to_plaintext = True - -By default, Mailman sends the message through lynx, but since this program is -not guaranteed to exist, we'll craft a simple, but stupid script to simulate -the conversion process. The script expects a single argument, which is the -name of the file containing the message payload to filter. - - >>> import os, sys - >>> script_path = os.path.join(config.DATA_DIR, 'filter.py') - >>> fp = open(script_path, 'w') - >>> try: - ... print >> fp, """\ - ... import sys - ... print 'Converted text/html to text/plain' - ... print 'Filename:', sys.argv[1] - ... """ - ... finally: - ... fp.close() - >>> config.HTML_TO_PLAIN_TEXT_COMMAND = '%s %s %%(filename)s' % ( - ... sys.executable, script_path) - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Content-Type: text/html - ... MIME-Version: 1.0 - ... - ... - ... - ... """) - >>> process(mlist, msg, {}) - >>> print msg.as_string() - From: aperson@example.com - MIME-Version: 1.0 - Content-Type: text/plain - X-Content-Filtered-By: Mailman/MimeDel ... - - Converted text/html to text/plain - Filename: ... - - - -Discarding empty parts -====================== - -Similarly, if after filtering a multipart section ends up empty, then the -entire multipart is discarded. For example, here's a message where an inner -``multipart/mixed`` contains two jpeg subparts. Both jpegs are filtered out, -so the entire inner ``multipart/mixed`` is discarded. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Content-Type: multipart/mixed; boundary=AAA - ... - ... --AAA - ... Content-Type: multipart/mixed; boundary=BBB - ... - ... --BBB - ... Content-Type: image/jpeg - ... - ... xxx - ... --BBB - ... Content-Type: image/jpeg - ... - ... yyy - ... --BBB--- - ... --AAA - ... Content-Type: multipart/alternative; boundary=CCC - ... - ... --CCC - ... Content-Type: text/html - ... - ...

This is a header

- ... - ... --CCC - ... Content-Type: text/plain - ... - ... A different message - ... --CCC-- - ... --AAA - ... Content-Type: image/gif - ... - ... zzz - ... --AAA - ... Content-Type: image/gif - ... - ... aaa - ... --AAA-- - ... """) - >>> process(mlist, msg, {}) - >>> print msg.as_string() - From: aperson@example.com - Content-Type: multipart/mixed; boundary=AAA - X-Content-Filtered-By: Mailman/MimeDel ... - - --AAA - MIME-Version: 1.0 - Content-Type: text/plain - - Converted text/html to text/plain - Filename: ... - - --AAA - Content-Type: image/gif - - zzz - --AAA - Content-Type: image/gif - - aaa - --AAA-- - - - -Passing MIME types -================== - -XXX Describe the pass_mime_types setting and how it interacts with -``filter_mime_types``. diff --git a/src/mailman/pipeline/docs/nntp.rst b/src/mailman/pipeline/docs/nntp.rst deleted file mode 100644 index 874712397..000000000 --- a/src/mailman/pipeline/docs/nntp.rst +++ /dev/null @@ -1,68 +0,0 @@ -============ -NNTP Gateway -============ - -Mailman has an NNTP gateway, whereby messages posted to the mailing list can -be forwarded onto an NNTP newsgroup. Typically this means Usenet, but since -NNTP is to Usenet as IP is to the web, it's more general than that. - - >>> mlist = create_list('_xtest@example.com') - -Gatewaying from the mailing list to the newsgroup happens through a separate -``nntp`` queue and happen immediately when the message is posted through to -the list. Note that gatewaying from the newsgroup to the list happens via a -cronjob (currently not shown). - -There are several situations which prevent a message from being gatewayed to -the newsgroup. The feature could be disabled, as is the default. -:: - - >>> mlist.gateway_to_news = False - >>> msg = message_from_string("""\ - ... Subject: An important message - ... - ... Something of great import. - ... """) - - >>> handler = config.handlers['to-usenet'] - >>> handler.process(mlist, msg, {}) - - >>> switchboard = config.switchboards['news'] - >>> switchboard.files - [] - -Even if enabled, messages that came from the newsgroup are never gated back to -the newsgroup. - - >>> mlist.gateway_to_news = True - >>> handler.process(mlist, msg, {'fromusenet': True}) - >>> switchboard.files - [] - -Neither are digests ever gated to the newsgroup. - - >>> handler.process(mlist, msg, {'isdigest': True}) - >>> switchboard.files - [] - -However, other posted messages get gated to the newsgroup via the nntp queue. -The list owner can set the linked newsgroup and the nntp host that its -messages are gated to. - - >>> mlist.linked_newsgroup = 'comp.lang.thing' - >>> mlist.nntp_host = 'news.example.com' - >>> handler.process(mlist, msg, {}) - >>> len(switchboard.files) - 1 - >>> filebase = switchboard.files[0] - >>> msg, msgdata = switchboard.dequeue(filebase) - >>> switchboard.finish(filebase) - >>> print msg.as_string() - Subject: An important message - - Something of great import. - - >>> dump_msgdata(msgdata) - _parsemsg: False - listname : _xtest@example.com - version : 3 diff --git a/src/mailman/pipeline/docs/reply-to.rst b/src/mailman/pipeline/docs/reply-to.rst deleted file mode 100644 index e08fea81d..000000000 --- a/src/mailman/pipeline/docs/reply-to.rst +++ /dev/null @@ -1,131 +0,0 @@ -================ -Reply-to munging -================ - -Messages that flow through the global pipeline get their headers *cooked*, -which basically means that their headers go through several mostly unrelated -transformations. Some headers get added, others get changed. Some of these -changes depend on mailing list settings and others depend on how the message -is getting sent through the system. We'll take things one-by-one. - - >>> mlist = create_list('_xtest@example.com') - -*Reply-to munging* refers to the behavior where a mailing list can be -configured to change or augment an existing ``Reply-To`` header in a message -posted to the list. Reply-to munging is fairly controversial, with arguments -made either for or against munging. - -The Mailman developers, and I believe the majority consensus is to do no -reply-to munging, under several principles. Primarily, most reply-to munging -is requested by people who do not have both a `Reply` and `Reply All` button -on their mail reader. If you do not munge ``Reply-To``, then these buttons -will work properly, but if you munge the header, it is impossible for these -buttons to work right, because both will reply to the list. This leads to -unfortunate accidents where a private message is accidentally posted to the -entire list. - -However, Mailman gives list owners the option to do reply-To munging anyway, -mostly as a way to shut up the really vocal minority who seem to insist on -this mis-feature. - - -Reply to list -============= - -A list can be configured to add a ``Reply-To`` header pointing back to the -mailing list's posting address. If there's no ``Reply-To`` header in the -original message, the list's posting address simply gets inserted. -:: - - >>> from mailman.interfaces.mailinglist import ReplyToMunging - >>> mlist.reply_goes_to_list = ReplyToMunging.point_to_list - >>> mlist.preferred_language = 'en' - >>> mlist.description = '' - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... - ... """) - - >>> from mailman.pipeline.cook_headers import process - >>> process(mlist, msg, {}) - >>> len(msg.get_all('reply-to')) - 1 - >>> print msg['reply-to'] - _xtest@example.com - -It's also possible to strip any existing ``Reply-To`` header first, before -adding the list's posting address. - - >>> mlist.first_strip_reply_to = True - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Reply-To: bperson@example.com - ... - ... """) - >>> process(mlist, msg, {}) - >>> len(msg.get_all('reply-to')) - 1 - >>> print msg['reply-to'] - _xtest@example.com - -If you don't first strip the header, then the list's posting address will just -get appended to whatever the original version was. - - >>> mlist.first_strip_reply_to = False - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Reply-To: bperson@example.com - ... - ... """) - >>> process(mlist, msg, {}) - >>> len(msg.get_all('reply-to')) - 1 - >>> print msg['reply-to'] - bperson@example.com, _xtest@example.com - - -Explicit Reply-To -================= - -The list can also be configured to have an explicit ``Reply-To`` header. - - >>> mlist.reply_goes_to_list = ReplyToMunging.explicit_header - >>> mlist.reply_to_address = 'my-list@example.com' - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... - ... """) - >>> process(mlist, msg, {}) - >>> len(msg.get_all('reply-to')) - 1 - >>> print msg['reply-to'] - my-list@example.com - -And as before, it's possible to either strip any existing ``Reply-To`` -header... - - >>> mlist.first_strip_reply_to = True - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Reply-To: bperson@example.com - ... - ... """) - >>> process(mlist, msg, {}) - >>> len(msg.get_all('reply-to')) - 1 - >>> print msg['reply-to'] - my-list@example.com - -...or not. - - >>> mlist.first_strip_reply_to = False - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Reply-To: bperson@example.com - ... - ... """) - >>> process(mlist, msg, {}) - >>> len(msg.get_all('reply-to')) - 1 - >>> print msg['reply-to'] - my-list@example.com, bperson@example.com diff --git a/src/mailman/pipeline/docs/replybot.rst b/src/mailman/pipeline/docs/replybot.rst deleted file mode 100644 index 7cdd7c928..000000000 --- a/src/mailman/pipeline/docs/replybot.rst +++ /dev/null @@ -1,343 +0,0 @@ -========================== -Automatic response handler -========================== - -Mailman has a autoreply handler that sends automatic responses to messages it -receives on its posting address, owner address, or robot address. Automatic -responses are subject to various conditions, such as headers in the original -message or the amount of time since the last auto-response. - - >>> mlist = create_list('_xtest@example.com') - >>> mlist.display_name = 'XTest' - - -Basic automatic responding -========================== - -Basic automatic responding occurs when the list is set up to respond to either -its ``-owner`` address, its ``-request`` address, or to the posting address, -and a message is sent to one of these addresses. A mailing list also has an -automatic response grace period which specifies how much time must pass before -a second response will be sent, with 0 meaning "there is no grace period". -:: - - >>> import datetime - >>> from mailman.interfaces.autorespond import ResponseAction - - >>> mlist.autorespond_owner = ResponseAction.respond_and_continue - >>> mlist.autoresponse_grace_period = datetime.timedelta() - >>> mlist.autoresponse_owner_text = 'owner autoresponse text' - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: _xtest-owner@example.com - ... - ... help - ... """) - -The preceding message to the mailing list's owner will trigger an automatic -response. -:: - - >>> from mailman.testing.helpers import get_queue_messages - - >>> handler = config.handlers['replybot'] - >>> handler.process(mlist, msg, dict(to_owner=True)) - >>> messages = get_queue_messages('virgin') - >>> len(messages) - 1 - - >>> dump_msgdata(messages[0].msgdata) - _parsemsg : False - listname : _xtest@example.com - nodecorate : True - recipients : set([u'aperson@example.com']) - reduced_list_headers: True - version : 3 - - >>> print messages[0].msg.as_string() - MIME-Version: 1.0 - Content-Type: text/plain; charset="us-ascii" - Content-Transfer-Encoding: 7bit - Subject: Auto-response for your message to the "XTest" mailing list - From: _xtest-bounces@example.com - To: aperson@example.com - X-Mailer: The Mailman Replybot - X-Ack: No - Message-ID: <...> - Date: ... - Precedence: bulk - - owner autoresponse text - - -Short circuiting -================ - -Several headers in the original message determine whether an automatic -response should even be sent. For example, if the message has an -``X-Ack: No`` header, no auto-response is sent. -:: - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... X-Ack: No - ... - ... help me - ... """) - - >>> handler.process(mlist, msg, dict(to_owner=True)) - >>> get_queue_messages('virgin') - [] - -Mailman itself can suppress automatic responses for certain types of -internally crafted messages, by setting the ``noack`` metadata key. -:: - - >>> msg = message_from_string("""\ - ... From: mailman@example.com - ... - ... help for you - ... """) - - >>> handler.process(mlist, msg, dict(noack=True, to_owner=True)) - >>> get_queue_messages('virgin') - [] - -If there is a ``Precedence:`` header with any of the values ``bulk``, -``junk``, or ``list``, then the automatic response is also suppressed. -:: - - >>> msg = message_from_string("""\ - ... From: asystem@example.com - ... Precedence: bulk - ... - ... hey! - ... """) - - >>> handler.process(mlist, msg, dict(to_owner=True)) - >>> get_queue_messages('virgin') - [] - - >>> msg.replace_header('precedence', 'junk') - >>> handler.process(mlist, msg, dict(to_owner=True)) - >>> get_queue_messages('virgin') - [] - - >>> msg.replace_header('precedence', 'list') - >>> handler.process(mlist, msg, dict(to_owner=True)) - >>> get_queue_messages('virgin') - [] - -Unless the ``X-Ack:`` header has a value of ``yes``, in which case, the -``Precedence`` header is ignored. -:: - - >>> msg['X-Ack'] = 'yes' - >>> handler.process(mlist, msg, dict(to_owner=True)) - >>> messages = get_queue_messages('virgin') - >>> len(messages) - 1 - - >>> dump_msgdata(messages[0].msgdata) - _parsemsg : False - listname : _xtest@example.com - nodecorate : True - recipients : set([u'asystem@example.com']) - reduced_list_headers: True - version : 3 - - >>> print messages[0].msg.as_string() - MIME-Version: 1.0 - Content-Type: text/plain; charset="us-ascii" - Content-Transfer-Encoding: 7bit - Subject: Auto-response for your message to the "XTest" mailing list - From: _xtest-bounces@example.com - To: asystem@example.com - X-Mailer: The Mailman Replybot - X-Ack: No - Message-ID: <...> - Date: ... - Precedence: bulk - - owner autoresponse text - - -Available auto-responses -======================== - -As shown above, a message sent to the ``-owner`` address will get an -auto-response with the text set for owner responses. Two other types of email -will get auto-responses: those sent to the ``-request`` address... -:: - - >>> mlist.autorespond_requests = ResponseAction.respond_and_continue - >>> mlist.autoresponse_request_text = 'robot autoresponse text' - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: _xtest-request@example.com - ... - ... help me - ... """) - - >>> handler.process(mlist, msg, dict(to_request=True)) - >>> messages = get_queue_messages('virgin') - >>> len(messages) - 1 - - >>> print messages[0].msg.as_string() - MIME-Version: 1.0 - Content-Type: text/plain; charset="us-ascii" - Content-Transfer-Encoding: 7bit - Subject: Auto-response for your message to the "XTest" mailing list - From: _xtest-bounces@example.com - To: aperson@example.com - X-Mailer: The Mailman Replybot - X-Ack: No - Message-ID: <...> - Date: ... - Precedence: bulk - - robot autoresponse text - -...and those sent to the posting address. -:: - - >>> mlist.autorespond_postings = ResponseAction.respond_and_continue - >>> mlist.autoresponse_postings_text = 'postings autoresponse text' - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: _xtest@example.com - ... - ... help me - ... """) - - >>> handler.process(mlist, msg, dict(to_list=True)) - >>> messages = get_queue_messages('virgin') - >>> len(messages) - 1 - - >>> print messages[0].msg.as_string() - MIME-Version: 1.0 - Content-Type: text/plain; charset="us-ascii" - Content-Transfer-Encoding: 7bit - Subject: Auto-response for your message to the "XTest" mailing list - From: _xtest-bounces@example.com - To: aperson@example.com - X-Mailer: The Mailman Replybot - X-Ack: No - Message-ID: <...> - Date: ... - Precedence: bulk - - postings autoresponse text - - -Grace periods -============= - -Automatic responses have a grace period, during which no additional responses -will be sent. This is so as not to bombard the sender with responses. The -grace period is measured in days. - - >>> mlist.autoresponse_grace_period = datetime.timedelta(days=10) - -When a response is sent to a person via any of the owner, request, or postings -addresses, the response date is recorded. The grace period is usually -measured in days. - - >>> msg = message_from_string("""\ - ... From: bperson@example.com - ... To: _xtest-owner@example.com - ... - ... help - ... """) - -This is the first response to bperson, so it gets sent. - - >>> handler.process(mlist, msg, dict(to_owner=True)) - >>> print len(get_queue_messages('virgin')) - 1 - -But with a grace period greater than zero, no subsequent response will be sent -right now. - - >>> handler.process(mlist, msg, dict(to_owner=True)) - >>> print len(get_queue_messages('virgin')) - 0 - -Fast forward 9 days and you still don't get a response. -:: - - >>> from mailman.utilities.datetime import factory - >>> factory.fast_forward(days=9) - - >>> handler.process(mlist, msg, dict(to_owner=True)) - >>> print len(get_queue_messages('virgin')) - 0 - -But tomorrow, the sender will get a new auto-response. - - >>> factory.fast_forward() - >>> handler.process(mlist, msg, dict(to_owner=True)) - >>> print len(get_queue_messages('virgin')) - 1 - -Of course, everything works the same way for messages to the request -address, even if the sender is the same person... -:: - - >>> msg = message_from_string("""\ - ... From: bperson@example.com - ... To: _xtest-request@example.com - ... - ... help - ... """) - - >>> handler.process(mlist, msg, dict(to_request=True)) - >>> print len(get_queue_messages('virgin')) - 1 - - >>> handler.process(mlist, msg, dict(to_request=True)) - >>> print len(get_queue_messages('virgin')) - 0 - - >>> factory.fast_forward(days=9) - >>> handler.process(mlist, msg, dict(to_request=True)) - >>> print len(get_queue_messages('virgin')) - 0 - - >>> factory.fast_forward() - >>> handler.process(mlist, msg, dict(to_request=True)) - >>> print len(get_queue_messages('virgin')) - 1 - -...and for messages to the posting address. -:: - - >>> msg = message_from_string("""\ - ... From: bperson@example.com - ... To: _xtest@example.com - ... - ... help - ... """) - - >>> handler.process(mlist, msg, dict(to_list=True)) - >>> print len(get_queue_messages('virgin')) - 1 - - >>> handler.process(mlist, msg, dict(to_list=True)) - >>> print len(get_queue_messages('virgin')) - 0 - - >>> factory.fast_forward(days=9) - >>> handler.process(mlist, msg, dict(to_list=True)) - >>> print len(get_queue_messages('virgin')) - 0 - - >>> factory.fast_forward() - >>> handler.process(mlist, msg, dict(to_list=True)) - >>> print len(get_queue_messages('virgin')) - 1 diff --git a/src/mailman/pipeline/docs/rfc-2369.rst b/src/mailman/pipeline/docs/rfc-2369.rst deleted file mode 100644 index 1b89f2354..000000000 --- a/src/mailman/pipeline/docs/rfc-2369.rst +++ /dev/null @@ -1,206 +0,0 @@ -========================= -RFC 2919 and 2369 headers -========================= - -`RFC 2919`_ and `RFC 2369`_ define headers for mailing list actions. These -headers generally start with the `List-` prefix. - - >>> mlist = create_list('test@example.com') - >>> mlist.preferred_language = 'en' - >>> mlist.archive = False - -.. - This is a helper function for the following section. - >>> def list_headers(msg, only=None): - ... if isinstance(only, basestring): - ... only = (only.lower(),) - ... elif only is None: - ... only = set(header.lower() for header in msg.keys() - ... if header.lower().startswith('list-')) - ... only.add('archived-at') - ... else: - ... only = set(header.lower() for header in only) - ... print '---start---' - ... for header in sorted(only): - ... for value in sorted(msg.get_all(header, ())): - ... print '%s: %s' % (header, value) - ... print '---end---' - -The `rfc-2369` handler adds the `List-` headers. `List-Id` is always added. - - >>> from mailman.pipeline.rfc_2369 import process - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... - ... """) - >>> process(mlist, msg, {}) - >>> list_headers(msg, 'list-id') - ---start--- - list-id: - ---end--- - - -Fewer headers -============= - -Some people don't like these headers because their mail readers aren't good -about hiding them. A list owner can turn these headers off. - - >>> mlist.include_rfc2369_headers = False - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... - ... """) - >>> process(mlist, msg, {}) - >>> list_headers(msg) - ---start--- - ---end--- - -Messages which Mailman generates itself, such as user or owner notifications, -have a reduced set of `List-` headers. Specifically, there is no `List-Post`, -`List-Archive` or `Archived-At` header. - - >>> mlist.include_rfc2369_headers = True - >>> mlist.include_list_post_header = False - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... - ... """) - >>> process(mlist, msg, dict(reduced_list_headers=True)) - >>> list_headers(msg) - ---start--- - list-help: - list-id: - list-subscribe: , - - list-unsubscribe: , - - ---end--- - - -List-Post header -================ - -Discussion lists, to which any subscriber can post, also have a `List-Post` -header which contains the `mailto:` URL used to send messages to the list. - - >>> mlist.include_list_post_header = True - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... - ... """) - >>> process(mlist, msg, {}) - >>> list_headers(msg) - ---start--- - list-help: - list-id: - list-post: - list-subscribe: , - - list-unsubscribe: , - - ---end--- - - -List-Id header -============== - -If the mailing list has a description, then it is included in the ``List-Id`` -header. - - >>> mlist.description = 'My test mailing list' - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... - ... """) - >>> process(mlist, msg, {}) - >>> list_headers(msg) - ---start--- - list-help: - list-id: My test mailing list - list-post: - list-subscribe: , - - list-unsubscribe: , - - ---end--- - -Any existing ``List-Id`` headers are removed from the original message. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... List-ID: <123.456.789> - ... - ... """) - - >>> process(mlist, msg, {}) - >>> list_headers(msg, only='list-id') - ---start--- - list-id: My test mailing list - ---end--- - - -Archive headers -=============== - -When the mailing list is configured to enable archiving, a `List-Archive` -header will be added. - - >>> mlist.archive = True - -`RFC 5064`_ defines the `Archived-At` header which contains the url to the -individual message in the archives. Archivers which don't support -pre-calculation of the archive url cannot add the `Archived-At` header. -However, other archivers can calculate the url, and do add this header. - - >>> config.push('prototype', """ - ... [archiver.prototype] - ... enable: yes - ... [archiver.mail_archive] - ... enable: no - ... [archiver.mhonarc] - ... enable: no - ... [archiver.pipermail] - ... enable: No - ... """) - -The *prototype* archiver can calculate this archive url given a `Message-ID`. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Message-ID: - ... X-Message-ID-Hash: 4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB - ... - ... """) - >>> process(mlist, msg, {}) - >>> list_headers(msg, only=('list-archive', 'archived-at')) - ---start--- - archived-at: http://lists.example.com/4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB - list-archive: - ---end--- - -If the mailing list isn't being archived, neither the `List-Archive` nor -`Archived-At` headers will be added. - - >>> config.pop('prototype') - >>> mlist.archive = False - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... - ... """) - >>> process(mlist, msg, {}) - >>> list_headers(msg) - ---start--- - list-help: - list-id: My test mailing list - list-post: - list-subscribe: , - - list-unsubscribe: , - - ---end--- - - -.. _`RFC 2919`: http://www.faqs.org/rfcs/rfc2919.html -.. _`RFC 2369`: http://www.faqs.org/rfcs/rfc2369.html -.. _`RFC 5064`: http://www.faqs.org/rfcs/rfc5064.html diff --git a/src/mailman/pipeline/docs/subject-munging.rst b/src/mailman/pipeline/docs/subject-munging.rst deleted file mode 100644 index e7a6553ce..000000000 --- a/src/mailman/pipeline/docs/subject-munging.rst +++ /dev/null @@ -1,249 +0,0 @@ -=============== -Subject munging -=============== - -Messages that flow through the global pipeline get their headers *cooked*, -which basically means that their headers go through several mostly unrelated -transformations. Some headers get added, others get changed. Some of these -changes depend on mailing list settings and others depend on how the message -is getting sent through the system. We'll take things one-by-one. - - >>> mlist = create_list('_xtest@example.com') - - -Inserting a prefix -================== - -Another thing header cooking does is *munge* the ``Subject`` header by -inserting the subject prefix for the list at the front. If there's no subject -header in the original message, Mailman uses a canned default. In order to do -subject munging, a mailing list must have a preferred language. -:: - - >>> mlist.subject_prefix = '[XTest] ' - >>> mlist.preferred_language = 'en' - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... - ... A message of great import. - ... """) - >>> msgdata = {} - - >>> from mailman.pipeline.cook_headers import process - >>> process(mlist, msg, msgdata) - -The original subject header is stored in the message metadata. We must print -the new ``Subject`` header because it gets converted from a string to an -``email.header.Header`` instance which has an unhelpful ``repr``. - - >>> msgdata['origsubj'] - u'' - >>> print msg['subject'] - [XTest] (no subject) - -If the original message had a ``Subject`` header, then the prefix is inserted -at the beginning of the header's value. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Subject: Something important - ... - ... A message of great import. - ... """) - >>> msgdata = {} - >>> process(mlist, msg, msgdata) - >>> print msgdata['origsubj'] - Something important - >>> print msg['subject'] - [XTest] Something important - -``Subject`` headers are not munged for digest messages. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Subject: Something important - ... - ... A message of great import. - ... """) - >>> process(mlist, msg, dict(isdigest=True)) - >>> print msg['subject'] - Something important - -Nor are they munged for *fast tracked* messages, which are generally defined -as messages that Mailman crafts internally. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Subject: Something important - ... - ... A message of great import. - ... """) - >>> process(mlist, msg, dict(_fasttrack=True)) - >>> print msg['subject'] - Something important - -If a ``Subject`` header already has a prefix, usually following a ``Re:`` -marker, another one will not be added but the prefix will be moved to the -front of the header text. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Subject: Re: [XTest] Something important - ... - ... A message of great import. - ... """) - >>> process(mlist, msg, {}) - >>> print msg['subject'] - [XTest] Re: Something important - -If the ``Subject`` header has a prefix at the front of the header text, that's -where it will stay. This is called *new style* prefixing and is the only -option available in Mailman 3. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Subject: [XTest] Re: Something important - ... - ... A message of great import. - ... """) - >>> process(mlist, msg, {}) - >>> print msg['subject'] - [XTest] Re: Something important - - -Internationalized headers -========================= - -Internationalization adds some interesting twists to the handling of subject -prefixes. Part of what makes this interesting is the encoding of i18n headers -using RFC 2047, and lists whose preferred language is in a different character -set than the encoded header. - - >>> msg = message_from_string("""\ - ... Subject: =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= - ... - ... """) - >>> process(mlist, msg, {}) - >>> print msg['subject'] - [XTest] =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= - >>> unicode(msg['subject']) - u'[XTest] \u30e1\u30fc\u30eb\u30de\u30f3' - - -Prefix numbers -============== - -Subject prefixes support a placeholder for the numeric post id. Every time a -message is posted to the mailing list, a *post id* gets incremented. This is -a purely sequential integer that increases monotonically. By added a ``%d`` -placeholder to the subject prefix, this post id can be included in the prefix. - - >>> mlist.subject_prefix = '[XTest %d] ' - >>> mlist.post_id = 456 - >>> msg = message_from_string("""\ - ... Subject: Something important - ... - ... """) - >>> process(mlist, msg, {}) - >>> print msg['subject'] - [XTest 456] Something important - -This works even when the message is a reply, except that in this case, the -numeric post id in the generated subject prefix is updated with the new post -id. - - >>> msg = message_from_string("""\ - ... Subject: [XTest 123] Re: Something important - ... - ... """) - >>> process(mlist, msg, {}) - >>> print msg['subject'] - [XTest 456] Re: Something important - -If the ``Subject`` header had old style prefixing, the prefix is moved to the -front of the header text. - - >>> msg = message_from_string("""\ - ... Subject: Re: [XTest 123] Something important - ... - ... """) - >>> process(mlist, msg, {}) - >>> print msg['subject'] - [XTest 456] Re: Something important - - -And of course, the proper thing is done when posting id numbers are included -in the subject prefix, and the subject is encoded non-ASCII. - - >>> msg = message_from_string("""\ - ... Subject: =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= - ... - ... """) - >>> process(mlist, msg, {}) - >>> print msg['subject'] - [XTest 456] =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= - >>> unicode(msg['subject']) - u'[XTest 456] \u30e1\u30fc\u30eb\u30de\u30f3' - -Even more fun is when the internationalized ``Subject`` header already has a -prefix, possibly with a different posting number. - - >>> msg = message_from_string("""\ - ... Subject: [XTest 123] Re: =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= - ... - ... """) - >>> process(mlist, msg, {}) - >>> print msg['subject'] - [XTest 456] Re: =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= - -.. - # XXX This requires Python email patch #1681333 to succeed. - # >>> unicode(msg['subject']) - # u'[XTest 456] Re: \u30e1\u30fc\u30eb\u30de\u30f3' - -As before, old style subject prefixes are re-ordered. - - >>> msg = message_from_string("""\ - ... Subject: Re: [XTest 123] =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= - ... - ... """) - >>> process(mlist, msg, {}) - >>> print msg['subject'] - [XTest 456] Re: - =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= - -.. - # XXX This requires Python email patch #1681333 to succeed. - # >>> unicode(msg['subject']) - # u'[XTest 456] Re: \u30e1\u30fc\u30eb\u30de\u30f3' - - -In this test case, we get an extra space between the prefix and the original -subject. It's because the original is *crooked*. Note that a ``Subject`` -starting with '\n ' is generated by some version of Eudora Japanese edition. - - >>> mlist.subject_prefix = '[XTest] ' - >>> msg = message_from_string("""\ - ... Subject: - ... Important message - ... - ... """) - >>> process(mlist, msg, {}) - >>> print msg['subject'] - [XTest] Important message - -And again, with an RFC 2047 encoded header. - - >>> msg = message_from_string("""\ - ... Subject: - ... =?iso-2022-jp?b?GyRCJWEhPCVrJV4lcxsoQg==?= - ... - ... """) - >>> process(mlist, msg, {}) - -.. - # XXX This one does not appear to work the same way as - # test_subject_munging_prefix_crooked() in the old Python-based tests. I need - # to get Tokio to look at this. - # >>> print msg['subject'] - # [XTest] =?iso-2022-jp?b?IBskQiVhITwlayVeJXMbKEI=?= diff --git a/src/mailman/pipeline/docs/tagger.rst b/src/mailman/pipeline/docs/tagger.rst deleted file mode 100644 index 80e682119..000000000 --- a/src/mailman/pipeline/docs/tagger.rst +++ /dev/null @@ -1,238 +0,0 @@ -============== -Message tagger -============== - -Mailman has a topics system which works like this: a mailing list -administrator sets up one or more topics, which is essentially a named regular -expression. The topic name can be any arbitrary string, and the name serves -double duty as the *topic tag*. Each message that flows the mailing list has -its ``Subject:`` and ``Keywords:`` headers compared against these regular -expressions. The message then gets tagged with the topic names of each hit. - - >>> mlist = create_list('_xtest@example.com') - -Topics must be enabled for Mailman to do any topic matching, even if topics -are defined. -:: - - >>> mlist.topics = [('bar fight', '.*bar.*', 'catch any bars', False)] - >>> mlist.topics_enabled = False - >>> mlist.topics_bodylines_limit = 0 - - >>> msg = message_from_string("""\ - ... Subject: foobar - ... Keywords: barbaz - ... - ... """) - >>> msgdata = {} - - >>> from mailman.pipeline.tagger import process - >>> process(mlist, msg, msgdata) - >>> print msg.as_string() - Subject: foobar - Keywords: barbaz - - - >>> msgdata - {} - -However, once topics are enabled, message will be tagged. There are two -artifacts of tagging; an ``X-Topics:`` header is added with the topic name, -and the message metadata gets a key with a list of matching topic names. - - >>> mlist.topics_enabled = True - >>> msg = message_from_string("""\ - ... Subject: foobar - ... Keywords: barbaz - ... - ... """) - >>> msgdata = {} - >>> process(mlist, msg, msgdata) - >>> print msg.as_string() - Subject: foobar - Keywords: barbaz - X-Topics: bar fight - - - >>> msgdata['topichits'] - [u'bar fight'] - - -Scanning body lines -=================== - -The tagger can also look at a certain number of body lines, but only for -``Subject:`` and ``Keyword:`` header-like lines. When set to zero, no body -lines are scanned. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Subject: nothing - ... Keywords: at all - ... - ... X-Ignore: something else - ... Subject: foobar - ... Keywords: barbaz - ... """) - >>> msgdata = {} - >>> process(mlist, msg, msgdata) - >>> print msg.as_string() - From: aperson@example.com - Subject: nothing - Keywords: at all - - X-Ignore: something else - Subject: foobar - Keywords: barbaz - - >>> msgdata - {} - -But let the tagger scan a few body lines and the matching headers will be -found. - - >>> mlist.topics_bodylines_limit = 5 - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Subject: nothing - ... Keywords: at all - ... - ... X-Ignore: something else - ... Subject: foobar - ... Keywords: barbaz - ... """) - >>> msgdata = {} - >>> process(mlist, msg, msgdata) - >>> print msg.as_string() - From: aperson@example.com - Subject: nothing - Keywords: at all - X-Topics: bar fight - - X-Ignore: something else - Subject: foobar - Keywords: barbaz - - >>> msgdata['topichits'] - [u'bar fight'] - -However, scanning stops at the first body line that doesn't look like a -header. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Subject: nothing - ... Keywords: at all - ... - ... This is not a header - ... Subject: foobar - ... Keywords: barbaz - ... """) - >>> msgdata = {} - >>> process(mlist, msg, msgdata) - >>> print msg.as_string() - From: aperson@example.com - Subject: nothing - Keywords: at all - - This is not a header - Subject: foobar - Keywords: barbaz - >>> msgdata - {} - -When set to a negative number, all body lines will be scanned. - - >>> mlist.topics_bodylines_limit = -1 - >>> lots_of_headers = '\n'.join(['X-Ignore: zip'] * 100) - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... Subject: nothing - ... Keywords: at all - ... - ... %s - ... Subject: foobar - ... Keywords: barbaz - ... """ % lots_of_headers) - >>> msgdata = {} - >>> process(mlist, msg, msgdata) - >>> # Rather than print out 100 X-Ignore: headers, let's just prove that - >>> # the X-Topics: header exists, meaning that the tagger did its job. - >>> print msg['x-topics'] - bar fight - >>> msgdata['topichits'] - [u'bar fight'] - - -Scanning sub-parts -================== - -The tagger will also scan the body lines of text subparts in a multipart -message, using the same rules as if all those body lines lived in a single -text payload. - - >>> msg = message_from_string("""\ - ... Subject: Was - ... Keywords: Raw - ... Content-Type: multipart/alternative; boundary="BOUNDARY" - ... - ... --BOUNDARY - ... From: sabo - ... To: obas - ... - ... Subject: farbaw - ... Keywords: barbaz - ... - ... --BOUNDARY-- - ... """) - >>> msgdata = {} - >>> process(mlist, msg, msgdata) - >>> print msg.as_string() - Subject: Was - Keywords: Raw - Content-Type: multipart/alternative; boundary="BOUNDARY" - X-Topics: bar fight - - --BOUNDARY - From: sabo - To: obas - - Subject: farbaw - Keywords: barbaz - - --BOUNDARY-- - - >>> msgdata['topichits'] - [u'bar fight'] - -But the tagger will not descend into non-text parts. - - >>> msg = message_from_string("""\ - ... Subject: Was - ... Keywords: Raw - ... Content-Type: multipart/alternative; boundary=BOUNDARY - ... - ... --BOUNDARY - ... From: sabo - ... To: obas - ... Content-Type: message/rfc822 - ... - ... Subject: farbaw - ... Keywords: barbaz - ... - ... --BOUNDARY - ... From: sabo - ... To: obas - ... Content-Type: message/rfc822 - ... - ... Subject: farbaw - ... Keywords: barbaz - ... - ... --BOUNDARY-- - ... """) - >>> msgdata = {} - >>> process(mlist, msg, msgdata) - >>> print msg['x-topics'] - None - >>> msgdata - {} diff --git a/src/mailman/pipeline/docs/to-outgoing.rst b/src/mailman/pipeline/docs/to-outgoing.rst deleted file mode 100644 index 816aa4ca6..000000000 --- a/src/mailman/pipeline/docs/to-outgoing.rst +++ /dev/null @@ -1,42 +0,0 @@ -==================== -The outgoing handler -==================== - -Mailman's outgoing queue is used as the wrapper around SMTP delivery to the -upstream mail server. The to-outgoing handler does little more than drop the -message into the outgoing queue. - - >>> mlist = create_list('test@example.com') - -Craft a message destined for the outgoing queue. Include some random metadata -as if this message had passed through some other handlers. -:: - - >>> msg = message_from_string("""\ - ... Subject: Here is a message - ... - ... Something of great import. - ... """) - - >>> msgdata = dict(foo=1, bar=2, verp=True) - >>> handler = config.handlers['to-outgoing'] - >>> handler.process(mlist, msg, msgdata) - -While the queued message will not be changed, the queued metadata will have an -additional key set: the mailing list name. - - >>> from mailman.testing.helpers import get_queue_messages - >>> messages = get_queue_messages('out') - >>> len(messages) - 1 - >>> print messages[0].msg.as_string() - Subject: Here is a message - - Something of great import. - >>> dump_msgdata(messages[0].msgdata) - _parsemsg: False - bar : 2 - foo : 1 - listname : test@example.com - verp : True - version : 3 diff --git a/src/mailman/pipeline/file_recipients.py b/src/mailman/pipeline/file_recipients.py deleted file mode 100644 index d087ff2bb..000000000 --- a/src/mailman/pipeline/file_recipients.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright (C) 2001-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see . - -"""Get the normal delivery recipients from a Sendmail style :include: file.""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'FileRecipients', - ] - - -import os -import errno - -from zope.interface import implements - -from mailman.core.i18n import _ -from mailman.interfaces.handler import IHandler - - - -class FileRecipients: - """Get the normal delivery recipients from an include file.""" - - implements(IHandler) - - name = 'file-recipients' - description = _('Get the normal delivery recipients from an include file.') - - def process(self, mlist, msg, msgdata): - """See `IHandler`.""" - if 'recipients' in msgdata: - return - filename = os.path.join(mlist.data_path, 'members.txt') - try: - with open(filename) as fp: - addrs = set(line.strip() for line in fp) - except IOError as error: - if error.errno != errno.ENOENT: - raise - msgdata['recipients'] = set() - return - # If the sender is a member of the list, remove them from the file - # recipients. - member = mlist.members.get_member(msg.sender) - if member is not None: - addrs.discard(member.address.email) - msgdata['recipients'] = addrs diff --git a/src/mailman/pipeline/mime_delete.py b/src/mailman/pipeline/mime_delete.py deleted file mode 100644 index c9c1eb408..000000000 --- a/src/mailman/pipeline/mime_delete.py +++ /dev/null @@ -1,302 +0,0 @@ -# Copyright (C) 2002-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see . - -"""MIME-stripping filter for Mailman. - -This module scans a message for MIME content, removing those sections whose -MIME types match one of a list of matches. multipart/alternative sections are -replaced by the first non-empty component, and multipart/mixed sections -wrapping only single sections after other processing are replaced by their -contents. -""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'MIMEDelete', - ] - - -import os -import errno -import logging -import tempfile - -from email.iterators import typed_subpart_iterator -from email.mime.message import MIMEMessage -from email.mime.text import MIMEText -from lazr.config import as_boolean -from os.path import splitext -from zope.interface import implements - -from mailman.config import config -from mailman.core import errors -from mailman.core.i18n import _ -from mailman.email.message import OwnerNotification -from mailman.interfaces.action import FilterAction -from mailman.interfaces.handler import IHandler -from mailman.utilities.string import oneline -from mailman.version import VERSION - - -log = logging.getLogger('mailman.error') - - - -def dispose(mlist, msg, msgdata, why): - if mlist.filter_action is FilterAction.reject: - # Bounce the message to the original author. - raise errors.RejectMessage(why) - elif mlist.filter_action is FilterAction.forward: - # Forward it on to the list moderators. - text=_("""\ -The attached message matched the $mlist.display_name mailing list's content -filtering rules and was prevented from being forwarded on to the list -membership. You are receiving the only remaining copy of the discarded -message. - -""") - subject=_('Content filter message notification') - notice = OwnerNotification(mlist, subject, roster=mlist.moderators) - notice.set_type('multipart/mixed') - notice.attach(MIMEText(text)) - notice.attach(MIMEMessage(msg)) - notice.send(mlist) - # Let this fall through so the original message gets discarded. - elif mlist.filter_action is FilterAction.preserve: - if as_boolean(config.mailman.filtered_messages_are_preservable): - # This is just like discarding the message except that a copy is - # placed in the 'bad' queue should the site administrator want to - # inspect the message. - filebase = config.switchboards['bad'].enqueue(msg, msgdata) - log.info('{0} preserved in file base {1}'.format( - msg.get('message-id', 'n/a'), filebase)) - else: - log.error( - '{1} invalid FilterAction: {0}. Treating as discard'.format( - mlist.fqdn_listname, mlist.filter_action.name)) - # Most cases also discard the message - raise errors.DiscardMessage(why) - - - -def process(mlist, msg, msgdata): - # We also don't care about our own digests or plaintext - ctype = msg.get_content_type() - mtype = msg.get_content_maintype() - # Check to see if the outer type matches one of the filter types - filtertypes = set(mlist.filter_types) - passtypes = set(mlist.pass_types) - if ctype in filtertypes or mtype in filtertypes: - dispose(mlist, msg, msgdata, - _("The message's content type was explicitly disallowed")) - # Check to see if there is a pass types and the outer type doesn't match - # one of these types - if passtypes and not (ctype in passtypes or mtype in passtypes): - dispose(mlist, msg, msgdata, - _("The message's content type was not explicitly allowed")) - # Filter by file extensions - filterexts = set(mlist.filter_extensions) - passexts = set(mlist.pass_extensions) - fext = get_file_ext(msg) - if fext: - if fext in filterexts: - dispose(mlist, msg, msgdata, - _("The message's file extension was explicitly disallowed")) - if passexts and not (fext in passexts): - dispose(mlist, msg, msgdata, - _("The message's file extension was not explicitly allowed")) - numparts = len([subpart for subpart in msg.walk()]) - # If the message is a multipart, filter out matching subparts - if msg.is_multipart(): - # Recursively filter out any subparts that match the filter list - prelen = len(msg.get_payload()) - filter_parts(msg, filtertypes, passtypes, filterexts, passexts) - # If the outer message is now an empty multipart (and it wasn't - # before!) then, again it gets discarded. - postlen = len(msg.get_payload()) - if postlen == 0 and prelen > 0: - dispose(mlist, msg, msgdata, - _("After content filtering, the message was empty")) - # Now replace all multipart/alternatives with just the first non-empty - # alternative. BAW: We have to special case when the outer part is a - # multipart/alternative because we need to retain most of the outer part's - # headers. For now we'll move the subpart's payload into the outer part, - # and then copy over its Content-Type: and Content-Transfer-Encoding: - # headers (any others?). - if mlist.collapse_alternatives: - collapse_multipart_alternatives(msg) - if ctype == 'multipart/alternative': - firstalt = msg.get_payload(0) - reset_payload(msg, firstalt) - # If we removed some parts, make note of this - changedp = 0 - if numparts <> len([subpart for subpart in msg.walk()]): - changedp = 1 - # Now perhaps convert all text/html to text/plain - if mlist.convert_html_to_plaintext and config.HTML_TO_PLAIN_TEXT_COMMAND: - changedp += to_plaintext(msg) - # If we're left with only two parts, an empty body and one attachment, - # recast the message to one of just that part - if msg.is_multipart() and len(msg.get_payload()) == 2: - if msg.get_payload(0).get_payload() == '': - useful = msg.get_payload(1) - reset_payload(msg, useful) - changedp = 1 - if changedp: - msg['X-Content-Filtered-By'] = 'Mailman/MimeDel {0}'.format(VERSION) - - - -def reset_payload(msg, subpart): - # Reset payload of msg to contents of subpart, and fix up content headers - payload = subpart.get_payload() - msg.set_payload(payload) - del msg['content-type'] - del msg['content-transfer-encoding'] - del msg['content-disposition'] - del msg['content-description'] - msg['Content-Type'] = subpart.get('content-type', 'text/plain') - cte = subpart.get('content-transfer-encoding') - if cte: - msg['Content-Transfer-Encoding'] = cte - cdisp = subpart.get('content-disposition') - if cdisp: - msg['Content-Disposition'] = cdisp - cdesc = subpart.get('content-description') - if cdesc: - msg['Content-Description'] = cdesc - - - -def filter_parts(msg, filtertypes, passtypes, filterexts, passexts): - # Look at all the message's subparts, and recursively filter - if not msg.is_multipart(): - return True - payload = msg.get_payload() - prelen = len(payload) - newpayload = [] - for subpart in payload: - keep = filter_parts(subpart, filtertypes, passtypes, - filterexts, passexts) - if not keep: - continue - ctype = subpart.get_content_type() - mtype = subpart.get_content_maintype() - if ctype in filtertypes or mtype in filtertypes: - # Throw this subpart away - continue - if passtypes and not (ctype in passtypes or mtype in passtypes): - # Throw this subpart away - continue - # check file extension - fext = get_file_ext(subpart) - if fext: - if fext in filterexts: - continue - if passexts and not (fext in passexts): - continue - newpayload.append(subpart) - # Check to see if we discarded all the subparts - postlen = len(newpayload) - msg.set_payload(newpayload) - if postlen == 0 and prelen > 0: - # We threw away everything - return False - return True - - - -def collapse_multipart_alternatives(msg): - if not msg.is_multipart(): - return - newpayload = [] - for subpart in msg.get_payload(): - if subpart.get_content_type() == 'multipart/alternative': - try: - firstalt = subpart.get_payload(0) - newpayload.append(firstalt) - except IndexError: - pass - else: - newpayload.append(subpart) - msg.set_payload(newpayload) - - - -def to_plaintext(msg): - changedp = False - for subpart in typed_subpart_iterator(msg, 'text', 'html'): - filename = tempfile.mktemp('.html') - fp = open(filename, 'w') - try: - fp.write(subpart.get_payload(decode=True)) - fp.close() - cmd = os.popen(config.HTML_TO_PLAIN_TEXT_COMMAND % - {'filename': filename}) - plaintext = cmd.read() - rtn = cmd.close() - if rtn: - log.error('HTML->text/plain error: %s', rtn) - finally: - try: - os.unlink(filename) - except OSError, e: - if e.errno <> errno.ENOENT: - raise - # Now replace the payload of the subpart and twiddle the Content-Type: - del subpart['content-transfer-encoding'] - subpart.set_payload(plaintext) - subpart.set_type('text/plain') - changedp = True - return changedp - - - -def get_file_ext(m): - """ - Get filename extension. Caution: some virus don't put filename - in 'Content-Disposition' header. -""" - fext = '' - filename = m.get_filename('') or m.get_param('name', '') - if filename: - fext = splitext(oneline(filename,'utf-8'))[1] - if len(fext) > 1: - fext = fext[1:] - else: - fext = '' - return fext - - - -class MIMEDelete: - """Filter the MIME content of messages.""" - - implements(IHandler) - - name = 'mime-delete' - description = _('Filter the MIME content of messages.') - - def process(self, mlist, msg, msgdata): - # Short-circuits - if not mlist.filter_content: - return - if msgdata.get('isdigest'): - return - process(mlist, msg, msgdata) diff --git a/src/mailman/pipeline/owner_recipients.py b/src/mailman/pipeline/owner_recipients.py deleted file mode 100644 index 9e4bbf174..000000000 --- a/src/mailman/pipeline/owner_recipients.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (C) 2001-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see . - -"""Calculate the list owner recipients (includes moderators).""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'process', - ] - - - -def process(mlist, msg, msgdata): - """Add owner recipients.""" - # The recipients are the owner and the moderator - msgdata['recipients'] = mlist.owner + mlist.moderator - # Don't decorate these messages with the header/footers - msgdata['nodecorate'] = True - msgdata['personalize'] = False diff --git a/src/mailman/pipeline/replybot.py b/src/mailman/pipeline/replybot.py deleted file mode 100644 index 83aa40214..000000000 --- a/src/mailman/pipeline/replybot.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see . - -"""Handler for automatic responses.""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'Replybot', - ] - - -import logging - -from zope.component import getUtility -from zope.interface import implements - -from mailman.core.i18n import _ -from mailman.email.message import UserNotification -from mailman.interfaces.autorespond import ( - ALWAYS_REPLY, IAutoResponseSet, Response, ResponseAction) -from mailman.interfaces.handler import IHandler -from mailman.interfaces.usermanager import IUserManager -from mailman.utilities.datetime import today -from mailman.utilities.string import expand, wrap - - -log = logging.getLogger('mailman.error') - - - -class Replybot: - """Send automatic responses.""" - - implements(IHandler) - - name = 'replybot' - description = _('Send automatic responses.') - - def process(self, mlist, msg, msgdata): - """See `IHandler`.""" - # There are several cases where the replybot is short-circuited: - # * the original message has an "X-Ack: No" header - # * the message has a Precedence header with values bulk, junk, or - # list, and there's no explicit "X-Ack: yes" header - # * the message metadata has a true 'noack' key - ack = msg.get('x-ack', '').lower() - if ack == 'no' or msgdata.get('noack'): - return - precedence = msg.get('precedence', '').lower() - if ack != 'yes' and precedence in ('bulk', 'junk', 'list'): - return - # Check to see if the list is even configured to autorespond to this - # email message. Note: the incoming message processors should set the - # destination key in the message data. - if msgdata.get('to_owner'): - if mlist.autorespond_owner is ResponseAction.none: - return - response_type = Response.owner - response_text = mlist.autoresponse_owner_text - elif msgdata.get('to_request'): - if mlist.autorespond_requests is ResponseAction.none: - return - response_type = Response.command - response_text = mlist.autoresponse_request_text - elif msgdata.get('to_list'): - if mlist.autorespond_postings is ResponseAction.none: - return - response_type = Response.postings - response_text = mlist.autoresponse_postings_text - else: - # There are no automatic responses for any other destination. - return - # Now see if we're in the grace period for this sender. grace_period - # = 0 means always automatically respond, as does an "X-Ack: yes" - # header (useful for debugging). - response_set = IAutoResponseSet(mlist) - user_manager = getUtility(IUserManager) - address = user_manager.get_address(msg.sender) - if address is None: - address = user_manager.create_address(msg.sender) - grace_period = mlist.autoresponse_grace_period - if grace_period > ALWAYS_REPLY and ack != 'yes': - last = response_set.last_response(address, response_type) - if last is not None and last.date_sent + grace_period > today(): - return - # Okay, we know we're going to respond to this sender, craft the - # message, send it, and update the database. - display_name = mlist.display_name - subject = _( - 'Auto-response for your message to the "$display_name" ' - 'mailing list') - # Do string interpolation into the autoresponse text - d = dict(list_name = mlist.list_name, - display_name = display_name, - listurl = mlist.script_url('listinfo'), - requestemail = mlist.request_address, - owneremail = mlist.owner_address, - ) - # Interpolation and Wrap the response text. - text = wrap(expand(response_text, d)) - outmsg = UserNotification(msg.sender, mlist.bounces_address, - subject, text, mlist.preferred_language) - outmsg['X-Mailer'] = _('The Mailman Replybot') - # prevent recursions and mail loops! - outmsg['X-Ack'] = 'No' - outmsg.send(mlist) - response_set.response_sent(address, response_type) diff --git a/src/mailman/pipeline/rfc_2369.py b/src/mailman/pipeline/rfc_2369.py deleted file mode 100644 index 26bfe094c..000000000 --- a/src/mailman/pipeline/rfc_2369.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright (C) 2011-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see . - -"""RFC 2369 List-* and related headers.""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'RFC2369', - ] - - -from email.utils import formataddr -from zope.interface import implements - -from mailman.config import config -from mailman.core.i18n import _ -from mailman.interfaces.handler import IHandler -from mailman.pipeline.cook_headers import uheader - -CONTINUATION = ',\n\t' - - - -def process(mlist, msg, msgdata): - """Add the RFC 2369 List-* and related headers.""" - # Some people really hate the List-* headers. It seems that the free - # version of Eudora (possibly on for some platforms) does not hide these - # headers by default, pissing off their users. Too bad. Fix the MUAs. - if not mlist.include_rfc2369_headers: - return - list_id = '{0.list_name}.{0.mail_host}'.format(mlist) - if mlist.description: - # Don't wrap the header since here we just want to get it properly RFC - # 2047 encoded. - i18ndesc = uheader(mlist, mlist.description, 'List-Id', maxlinelen=998) - listid_h = formataddr((str(i18ndesc), list_id)) - else: - # Without a description, we need to ensure the MUST brackets. - listid_h = '<{0}>'.format(list_id) - # No other agent should add a List-ID header except Mailman. - del msg['list-id'] - msg['List-Id'] = listid_h - # For internally crafted messages, we also add a (nonstandard), - # "X-List-Administrivia: yes" header. For all others (i.e. those coming - # from list posts), we add a bunch of other RFC 2369 headers. - requestaddr = mlist.request_address - subfieldfmt = '<{0}>, ' - listinfo = mlist.script_url('listinfo') - headers = {} - # XXX reduced_list_headers used to suppress List-Help, List-Subject, and - # List-Unsubscribe from UserNotification. That doesn't seem to make sense - # any more, so always add those three headers (others will still be - # suppressed). - headers.update({ - 'List-Help' : ''.format(requestaddr), - 'List-Unsubscribe': subfieldfmt.format(listinfo, mlist.leave_address), - 'List-Subscribe' : subfieldfmt.format(listinfo, mlist.join_address), - }) - if not msgdata.get('reduced_list_headers'): - # List-Post: is controlled by a separate attribute - if mlist.include_list_post_header: - headers['List-Post'] = ''.format(mlist.posting_address) - # Add RFC 2369 and 5064 archiving headers, if archiving is enabled. - if mlist.archive: - for archiver in config.archivers: - headers['List-Archive'] = '<{0}>'.format( - archiver.list_url(mlist)) - permalink = archiver.permalink(mlist, msg) - if permalink is not None: - headers['Archived-At'] = permalink - # XXX RFC 2369 also defines a List-Owner header which we are not currently - # supporting, but should. - for h, v in headers.items(): - # First we delete any pre-existing headers because the RFC permits - # only one copy of each, and we want to be sure it's ours. - del msg[h] - # Wrap these lines if they are too long. 78 character width probably - # shouldn't be hardcoded, but is at least text-MUA friendly. The - # adding of 2 is for the colon-space separator. - if len(h) + 2 + len(v) > 78: - v = CONTINUATION.join(v.split(', ')) - msg[h] = v - - - -class RFC2369: - """Add the RFC 2369 List-* headers.""" - - implements(IHandler) - - name = 'rfc-2369' - description = _('Add the RFC 2369 List-* headers.') - - def process(self, mlist, msg, msgdata): - """See `IHandler`.""" - process(mlist, msg, msgdata) diff --git a/src/mailman/pipeline/tagger.py b/src/mailman/pipeline/tagger.py deleted file mode 100644 index 49e004a12..000000000 --- a/src/mailman/pipeline/tagger.py +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright (C) 2001-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see . - -"""Extract topics from the original mail message.""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'Tagger', - ] - - -import re -import email.iterators -import email.parser - -from zope.interface import implements - -from mailman.core.i18n import _ -from mailman.interfaces.handler import IHandler - - -OR = '|' -CRNL = '\r\n' -EMPTYBYTES = b'' -NLTAB = '\n\t' - - - -def process(mlist, msg, msgdata): - """Tag the message for topics.""" - if not mlist.topics_enabled: - return - # Extract the Subject:, Keywords:, and possibly body text - matchlines = [] - matchlines.append(msg.get('subject', None)) - matchlines.append(msg.get('keywords', None)) - if mlist.topics_bodylines_limit == 0: - # Don't scan any body lines - pass - elif mlist.topics_bodylines_limit < 0: - # Scan all body lines - matchlines.extend(scanbody(msg)) - else: - # Scan just some of the body lines - matchlines.extend(scanbody(msg, mlist.topics_bodylines_limit)) - # Filter out any 'false' items. - matchlines = [item for item in matchlines if item] - # For each regular expression in the topics list, see if any of the lines - # of interest from the message match the regexp. If so, the message gets - # added to the specific topics bucket. - hits = {} - for name, pattern, desc, emptyflag in mlist.topics: - pattern = OR.join(pattern.splitlines()) - cre = re.compile(pattern, re.IGNORECASE) - for line in matchlines: - if cre.search(line): - hits[name] = 1 - break - if hits: - # Sort the keys and make them available both in the message metadata - # and in a message header. - msgdata['topichits'] = sorted(hits) - msg['X-Topics'] = NLTAB.join(sorted(hits)) - - - -def scanbody(msg, numlines=None): - """Scan the body for keywords.""" - # We only scan the body of the message if it is of MIME type text/plain, - # or if the outer type is multipart/alternative and there is a text/plain - # part. Anything else, and the body is ignored for header-scan purposes. - found = None - if msg.get_content_type() == 'text/plain': - found = msg - elif msg.is_multipart()\ - and msg.get_content_type() == 'multipart/alternative': - for found in msg.get_payload(): - if found.get_content_type() == 'text/plain': - break - else: - found = None - if not found: - return [] - # Now that we have a Message object that meets our criteria, let's extract - # the first numlines of body text. - lines = [] - lineno = 0 - reader = list(email.iterators.body_line_iterator(msg)) - while numlines is None or lineno < numlines: - try: - line = bytes(reader.pop(0)) - except IndexError: - break - # Blank lines don't count - if not line.strip(): - continue - lineno += 1 - lines.append(line) - # Concatenate those body text lines with newlines, and then create a new - # message object from those lines. - p = _ForgivingParser() - msg = p.parsestr(EMPTYBYTES.join(lines)) - return msg.get_all('subject', []) + msg.get_all('keywords', []) - - - -class _ForgivingParser(email.parser.HeaderParser): - """An lax email parser. - - Be a little more forgiving about non-header/continuation lines, since - we'll just read as much as we can from 'header-like' lines in the body. - """ - # BAW: WIBNI we didn't have to cut-n-paste this whole thing just to - # specialize the way it returns? - def _parseheaders(self, container, fp): - """See `email.parser.HeaderParser`.""" - # Parse the headers, returning a list of header/value pairs. None as - # the header means the Unix-From header. - lastheader = '' - lastvalue = [] - lineno = 0 - while 1: - # Don't strip the line before we test for the end condition, - # because whitespace-only header lines are RFC compliant - # continuation lines. - line = fp.readline() - if not line: - break - line = line.splitlines()[0] - if not line: - break - # Ignore the trailing newline - lineno += 1 - # Check for initial Unix From_ line - if line.startswith('From '): - if lineno == 1: - container.set_unixfrom(line) - continue - else: - break - # Header continuation line - if line[0] in ' \t': - if not lastheader: - break - lastvalue.append(line) - continue - # Normal, non-continuation header. BAW: this should check to make - # sure it's a legal header, e.g. doesn't contain spaces. Also, we - # should expose the header matching algorithm in the API, and - # allow for a non-strict parsing mode (that ignores the line - # instead of raising the exception). - i = line.find(':') - if i < 0: - break - if lastheader: - container[lastheader] = NLTAB.join(lastvalue) - lastheader = line[:i] - lastvalue = [line[i+1:].lstrip()] - # Make sure we retain the last header - if lastheader: - container[lastheader] = NLTAB.join(lastvalue) - - - -class Tagger: - """Tag messages with topic matches.""" - - implements(IHandler) - - name = 'tagger' - description = _('Tag messages with topic matches.') - - def process(self, mlist, msg, msgdata): - """See `IHandler`.""" - process(mlist, msg, msgdata) diff --git a/src/mailman/pipeline/tests/__init__.py b/src/mailman/pipeline/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/mailman/pipeline/tests/test_mimedel.py b/src/mailman/pipeline/tests/test_mimedel.py deleted file mode 100644 index 566c1a40c..000000000 --- a/src/mailman/pipeline/tests/test_mimedel.py +++ /dev/null @@ -1,213 +0,0 @@ -# Copyright (C) 2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see . - -"""Test the mime_delete handler.""" - -from __future__ import absolute_import, print_function, unicode_literals - -__metaclass__ = type -__all__ = [ - 'TestDispose', - ] - - -import unittest - -from zope.component import getUtility - -from mailman.app.lifecycle import create_list -from mailman.config import config -from mailman.core import errors -from mailman.interfaces.action import FilterAction -from mailman.interfaces.member import MemberRole -from mailman.interfaces.usermanager import IUserManager -from mailman.pipeline import mime_delete -from mailman.testing.helpers import ( - LogFileMark, - get_queue_messages, - specialized_message_from_string as mfs) -from mailman.testing.layers import ConfigLayer - - - -class TestDispose(unittest.TestCase): - """Test the mime_delete handler.""" - - layer = ConfigLayer - - def setUp(self): - self._mlist = create_list('test@example.com') - self._msg = mfs("""\ -From: anne@example.com -To: test@example.com -Subject: A disposable message -Message-ID: - -""") - # Python 2.7 has assertMultiLineEqual. Let this work without bounds. - self.maxDiff = None - self.eq = getattr(self, 'assertMultiLineEqual', self.assertEqual) - config.push('dispose', """ - [mailman] - site_owner: noreply@example.com - """) - - def tearDown(self): - config.pop('dispose') - - def test_dispose_discard(self): - self._mlist.filter_action = FilterAction.discard - try: - mime_delete.dispose(self._mlist, self._msg, {}, 'discarding') - except errors.DiscardMessage as error: - pass - else: - raise AssertionError('DiscardMessage exception expected') - self.assertEqual(error.message, 'discarding') - # There should be no messages in the 'bad' queue. - self.assertEqual(len(get_queue_messages('bad')), 0) - - def test_dispose_bounce(self): - self._mlist.filter_action = FilterAction.reject - try: - mime_delete.dispose(self._mlist, self._msg, {}, 'rejecting') - except errors.RejectMessage as error: - pass - else: - raise AssertionError('RejectMessage exception expected') - self.assertEqual(error.message, 'rejecting') - # There should be no messages in the 'bad' queue. - self.assertEqual(len(get_queue_messages('bad')), 0) - - def test_dispose_forward(self): - # The disposed message gets forwarded to the list moderators. So - # first add some moderators. - user_manager = getUtility(IUserManager) - anne = user_manager.create_address('anne@example.com') - bart = user_manager.create_address('bart@example.com') - self._mlist.subscribe(anne, MemberRole.moderator) - self._mlist.subscribe(bart, MemberRole.moderator) - # Now set the filter action and dispose the message. - self._mlist.filter_action = FilterAction.forward - try: - mime_delete.dispose(self._mlist, self._msg, {}, 'forwarding') - except errors.DiscardMessage as error: - pass - else: - raise AssertionError('DiscardMessage exception expected') - self.assertEqual(error.message, 'forwarding') - # There should now be a multipart message in the virgin queue destined - # for the mailing list owners. - messages = get_queue_messages('virgin') - self.assertEqual(len(messages), 1) - message = messages[0].msg - self.assertEqual(message.get_content_type(), 'multipart/mixed') - # Anne and Bart should be recipients of the message, but it will look - # like the message is going to the list owners. - self.assertEqual(message['to'], 'test-owner@example.com') - self.assertEqual(message.recipients, - set(['anne@example.com', 'bart@example.com'])) - # The list owner should be the sender. - self.assertEqual(message['from'], 'noreply@example.com') - self.assertEqual(message['subject'], - 'Content filter message notification') - # The body of the first part provides the moderators some details. - part0 = message.get_payload(0) - self.assertEqual(part0.get_content_type(), 'text/plain') - self.eq(part0.get_payload(), """\ -The attached message matched the Test mailing list's content -filtering rules and was prevented from being forwarded on to the list -membership. You are receiving the only remaining copy of the discarded -message. - -""") - # The second part is the container for the original message. - part1 = message.get_payload(1) - self.assertEqual(part1.get_content_type(), 'message/rfc822') - # And the first part of *that* message will be the original message. - original = part1.get_payload(0) - self.assertEqual(original['subject'], 'A disposable message') - self.assertEqual(original['message-id'], '') - - def test_dispose_non_preservable(self): - # Two actions can happen here, depending on a site-wide setting. If - # the site owner has indicated that filtered messages cannot be - # preserved, then this is the same as discarding them. - self._mlist.filter_action = FilterAction.preserve - config.push('non-preservable', """ - [mailman] - filtered_messages_are_preservable: no - """) - try: - mime_delete.dispose(self._mlist, self._msg, {}, 'not preserved') - except errors.DiscardMessage as error: - pass - else: - raise AssertionError('DiscardMessage exception expected') - finally: - config.pop('non-preservable') - self.assertEqual(error.message, 'not preserved') - # There should be no messages in the 'bad' queue. - self.assertEqual(len(get_queue_messages('bad')), 0) - - def test_dispose_preservable(self): - # Two actions can happen here, depending on a site-wide setting. If - # the site owner has indicated that filtered messages can be - # preserved, then this is similar to discarding the message except - # that a copy is preserved in the 'bad' queue. - self._mlist.filter_action = FilterAction.preserve - config.push('preservable', """ - [mailman] - filtered_messages_are_preservable: yes - """) - try: - mime_delete.dispose(self._mlist, self._msg, {}, 'preserved') - except errors.DiscardMessage as error: - pass - else: - raise AssertionError('DiscardMessage exception expected') - finally: - config.pop('preservable') - self.assertEqual(error.message, 'preserved') - # There should be no messages in the 'bad' queue. - messages = get_queue_messages('bad') - self.assertEqual(len(messages), 1) - message = messages[0].msg - self.assertEqual(message['subject'], 'A disposable message') - self.assertEqual(message['message-id'], '') - - def test_bad_action(self): - # This should never happen, but what if it does? - # FilterAction.accept, FilterAction.hold, and FilterAction.defer are - # not valid. They are treated as discard actions, but the problem is - # also logged. - for action in (FilterAction.accept, - FilterAction.hold, - FilterAction.defer): - self._mlist.filter_action = action - mark = LogFileMark('mailman.error') - try: - mime_delete.dispose(self._mlist, self._msg, {}, 'bad action') - except errors.DiscardMessage as error: - pass - else: - raise AssertionError('DiscardMessage exception expected') - self.assertEqual(error.message, 'bad action') - line = mark.readline()[:-1] - self.assertTrue(line.endswith( - '{0} invalid FilterAction: test@example.com. ' - 'Treating as discard'.format(action.name))) diff --git a/src/mailman/pipeline/to_archive.py b/src/mailman/pipeline/to_archive.py deleted file mode 100644 index fd5259a14..000000000 --- a/src/mailman/pipeline/to_archive.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see . - -"""Add the message to the archives.""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'ToArchive', - ] - - -from zope.interface import implements - -from mailman.config import config -from mailman.core.i18n import _ -from mailman.interfaces.handler import IHandler - - - -class ToArchive: - """Add the message to the archives.""" - - implements(IHandler) - - name = 'to-archive' - description = _('Add the message to the archives.') - - def process(self, mlist, msg, msgdata): - """See `IHandler`.""" - # Short circuits. - if msgdata.get('isdigest') or not mlist.archive: - return - # Common practice seems to favor "X-No-Archive: yes". No other value - # for this header seems to make sense, so we'll just test for it's - # presence. I'm keeping "X-Archive: no" for backwards compatibility. - if 'x-no-archive' in msg or msg.get('x-archive', '').lower() == 'no': - return - # Send the message to the archiver queue. - config.switchboards['archive'].enqueue(msg, msgdata) diff --git a/src/mailman/pipeline/to_digest.py b/src/mailman/pipeline/to_digest.py deleted file mode 100644 index 698f16e1e..000000000 --- a/src/mailman/pipeline/to_digest.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see . - -"""Add the message to the list's current digest.""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'ToDigest', - ] - - -import os -import datetime - -from zope.interface import implements - -from mailman.config import config -from mailman.core.i18n import _ -from mailman.email.message import Message -from mailman.interfaces.digests import DigestFrequency -from mailman.interfaces.handler import IHandler -from mailman.utilities.mailbox import Mailbox - - - -class ToDigest: - """Add the message to the digest, possibly sending it.""" - - implements(IHandler) - - name = 'to-digest' - description = _('Add the message to the digest, possibly sending it.') - - def process(self, mlist, msg, msgdata): - """See `IHandler`.""" - # Short circuit for non-digestable messages. - if not mlist.digestable or msgdata.get('isdigest'): - return - # Open the mailbox that will be used to collect the current digest. - mailbox_path = os.path.join(mlist.data_path, 'digest.mmdf') - # Lock the mailbox and append the message. - with Mailbox(mailbox_path, create=True) as mbox: - mbox.add(msg) - # Calculate the current size of the mailbox file. This will not tell - # us exactly how big the resulting MIME and rfc1153 digest will - # actually be, but it's the most easily available metric to decide - # whether the size threshold has been reached. - size = os.path.getsize(mailbox_path) - if size >= mlist.digest_size_threshold * 1024.0: - # The digest is ready to send. Because we don't want to hold up - # this process with crafting the digest, we're going to move the - # digest file to a safe place, then craft a fake message for the - # DigestRunner as a trigger for it to build and send the digest. - mailbox_dest = os.path.join( - mlist.data_path, - 'digest.{0.volume}.{0.next_digest_number}.mmdf'.format(mlist)) - volume = mlist.volume - digest_number = mlist.next_digest_number - bump_digest_number_and_volume(mlist) - os.rename(mailbox_path, mailbox_dest) - config.switchboards['digest'].enqueue( - Message(), - listname=mlist.fqdn_listname, - digest_path=mailbox_dest, - volume=volume, - digest_number=digest_number) - - - -def bump_digest_number_and_volume(mlist): - """Bump the digest number and volume.""" - now = datetime.datetime.now() - if mlist.digest_last_sent_at is None: - # There has been no previous digest. - bump = False - elif mlist.digest_volume_frequency == DigestFrequency.yearly: - bump = (now.year > mlist.digest_last_sent_at.year) - elif mlist.digest_volume_frequency == DigestFrequency.monthly: - # Monthly. - this_month = now.year * 100 + now.month - digest_month = (mlist.digest_last_sent_at.year * 100 + - mlist.digest_last_sent_at.month) - bump = (this_month > digest_month) - elif mlist.digest_volume_frequency == DigestFrequency.quarterly: - # Quarterly. - this_quarter = now.year * 100 + (now.month - 1) // 4 - digest_quarter = (mlist.digest_last_sent_at.year * 100 + - (mlist.digest_last_sent_at.month - 1) // 4) - bump = (this_quarter > digest_quarter) - elif mlist.digest_volume_frequency == DigestFrequency.weekly: - this_week = now.year * 100 + now.isocalendar()[1] - digest_week = (mlist.digest_last_sent_at.year * 100 + - mlist.digest_last_sent_at.isocalendar()[1]) - bump = (this_week > digest_week) - elif mlist.digest_volume_frequency == DigestFrequency.daily: - bump = (now.toordinal() > mlist.digest_last_sent_at.toordinal()) - else: - raise AssertionError( - 'Bad DigestFrequency: {0}'.format( - mlist.digest_volume_frequency)) - if bump: - mlist.volume += 1 - mlist.next_digest_number = 1 - else: - # Just bump the digest number. - mlist.next_digest_number += 1 - mlist.digest_last_sent_at = now diff --git a/src/mailman/pipeline/to_outgoing.py b/src/mailman/pipeline/to_outgoing.py deleted file mode 100644 index 971f87757..000000000 --- a/src/mailman/pipeline/to_outgoing.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see . - -"""Re-queue the message to the outgoing queue. - -This module is only for use by the IncomingRunner for delivering messages -posted to the list membership. Anything else that needs to go out to some -recipient should just be placed in the out queue directly. -""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'ToOutgoing', - ] - - -from zope.interface import implements - -from mailman.config import config -from mailman.core.i18n import _ -from mailman.interfaces.handler import IHandler - - - -class ToOutgoing: - """Send the message to the outgoing queue.""" - - implements(IHandler) - - name = 'to-outgoing' - description = _('Send the message to the outgoing queue.') - - def process(self, mlist, msg, msgdata): - """See `IHandler`.""" - config.switchboards['out'].enqueue( - msg, msgdata, listname=mlist.fqdn_listname) diff --git a/src/mailman/pipeline/to_usenet.py b/src/mailman/pipeline/to_usenet.py deleted file mode 100644 index 26a383c64..000000000 --- a/src/mailman/pipeline/to_usenet.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see . - -"""Move the message to the mail->news queue.""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'ToUsenet', - ] - - -import logging - -from zope.interface import implements - -from mailman.config import config -from mailman.core.i18n import _ -from mailman.interfaces.handler import IHandler - -COMMASPACE = ', ' - -log = logging.getLogger('mailman.error') - - - -class ToUsenet: - """Move the message to the outgoing news queue.""" - - implements(IHandler) - - name = 'to-usenet' - description = _('Move the message to the outgoing news queue.') - - def process(self, mlist, msg, msgdata): - """See `IHandler`.""" - # Short circuits. - if not mlist.gateway_to_news or \ - msgdata.get('isdigest') or \ - msgdata.get('fromusenet'): - return - # sanity checks - error = [] - if not mlist.linked_newsgroup: - error.append('no newsgroup') - if not mlist.nntp_host: - error.append('no NNTP host') - if error: - log.error('NNTP gateway improperly configured: %s', - COMMASPACE.join(error)) - return - # Put the message in the news runner's queue. - config.switchboards['news'].enqueue( - msg, msgdata, listname=mlist.fqdn_listname) diff --git a/src/mailman/runners/digest.py b/src/mailman/runners/digest.py index 513a20322..afc11f732 100644 --- a/src/mailman/runners/digest.py +++ b/src/mailman/runners/digest.py @@ -41,11 +41,10 @@ from email.utils import formatdate, getaddresses, make_msgid from urllib2 import URLError from mailman.config import config -from mailman.core.errors import DiscardMessage from mailman.core.i18n import _ from mailman.core.runner import Runner +from mailman.handlers.decorate import decorate from mailman.interfaces.member import DeliveryMode, DeliveryStatus -from mailman.pipeline.decorate import decorate from mailman.utilities.i18n import make from mailman.utilities.mailbox import Mailbox from mailman.utilities.string import oneline, wrap -- cgit v1.3.1 From 2e7565996c4f63d17028e04e133c851e4c935a2b Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Fri, 23 Mar 2012 13:49:46 -0400 Subject: Rename the member recipients calculating handler to make way for the owner recipients handler. --- src/mailman/core/pipelines.py | 2 +- src/mailman/handlers/calculate_recipients.py | 149 --------------------------- src/mailman/handlers/docs/calc-recips.rst | 2 +- src/mailman/handlers/member_recipients.py | 149 +++++++++++++++++++++++++++ src/mailman/runners/docs/outgoing.rst | 4 +- 5 files changed, 153 insertions(+), 153 deletions(-) delete mode 100644 src/mailman/handlers/calculate_recipients.py create mode 100644 src/mailman/handlers/member_recipients.py diff --git a/src/mailman/core/pipelines.py b/src/mailman/core/pipelines.py index 9d24cc61f..5c0eb7f71 100644 --- a/src/mailman/core/pipelines.py +++ b/src/mailman/core/pipelines.py @@ -98,7 +98,7 @@ class BuiltInPipeline(BasePipeline): _default_handlers = ( 'mime-delete', 'tagger', - 'calculate-recipients', + 'member-recipients', 'avoid-duplicates', 'cleanse', 'cleanse-dkim', diff --git a/src/mailman/handlers/calculate_recipients.py b/src/mailman/handlers/calculate_recipients.py deleted file mode 100644 index 15be07ecd..000000000 --- a/src/mailman/handlers/calculate_recipients.py +++ /dev/null @@ -1,149 +0,0 @@ -# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see . - -"""Calculate the regular (i.e. non-digest) recipients of the message. - -This module calculates the non-digest recipients for the message based on the -list's membership and configuration options. It places the list of recipients -on the `recipients' attribute of the message. This attribute is used by the -SendmailDeliver and BulkDeliver modules. -""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'CalculateRecipients', - ] - -from zope.interface import implements - -from mailman.config import config -from mailman.core import errors -from mailman.core.i18n import _ -from mailman.interfaces.handler import IHandler -from mailman.interfaces.member import DeliveryStatus -from mailman.utilities.string import wrap - - - -class CalculateRecipients: - """Calculate the regular (i.e. non-digest) recipients of the message.""" - - implements(IHandler) - - name = 'calculate-recipients' - description = _('Calculate the regular recipients of the message.') - - def process(self, mlist, msg, msgdata): - """See `IHandler`.""" - # Short circuit if we've already calculated the recipients list, - # regardless of whether the list is empty or not. - if 'recipients' in msgdata: - return - # Should the original sender should be included in the recipients list? - include_sender = True - member = mlist.members.get_member(msg.sender) - if member and not member.receive_own_postings: - include_sender = False - # Support for urgent messages, which bypasses digests and disabled - # delivery and forces an immediate delivery to all members Right Now. - # We are specifically /not/ allowing the site admins password to work - # here because we want to discourage the practice of sending the site - # admin password through email in the clear. (see also Approve.py) - # - # XXX This is broken. - missing = object() - password = msg.get('urgent', missing) - if password is not missing: - if mlist.Authenticate((config.AuthListModerator, - config.AuthListAdmin), - password): - recipients = mlist.getMemberCPAddresses( - mlist.getRegularMemberKeys() + - mlist.getDigestMemberKeys()) - msgdata['recipients'] = recipients - return - else: - # Bad Urgent: password, so reject it instead of passing it on. - # I think it's better that the sender know they screwed up - # than to deliver it normally. - listname = mlist.display_name - text = _("""\ -Your urgent message to the $listname mailing list was not authorized for -delivery. The original message as received by Mailman is attached. -""") - raise errors.RejectMessage(wrap(text)) - # Calculate the regular recipients of the message - recipients = set(member.address.email - for member in mlist.regular_members.members - if member.delivery_status == DeliveryStatus.enabled) - # Remove the sender if they don't want to receive their own posts - if not include_sender and member.address.email in recipients: - recipients.remove(member.address.email) - # Handle topic classifications - do_topic_filters(mlist, msg, msgdata, recipients) - # Bookkeeping - msgdata['recipients'] = recipients - - - -def do_topic_filters(mlist, msg, msgdata, recipients): - """Filter out recipients based on topics.""" - if not mlist.topics_enabled: - # MAS: if topics are currently disabled for the list, send to all - # regardless of ReceiveNonmatchingTopics - return - hits = msgdata.get('topichits') - zap_recipients = [] - if hits: - # The message hit some topics, so only deliver this message to those - # who are interested in one of the hit topics. - for user in recipients: - utopics = mlist.getMemberTopics(user) - if not utopics: - # This user is not interested in any topics, so they get all - # postings. - continue - # BAW: Slow, first-match, set intersection! - for topic in utopics: - if topic in hits: - # The user wants this message - break - else: - # The user was interested in topics, but not any of the ones - # this message matched, so zap him. - zap_recipients.append(user) - else: - # The semantics for a message that did not hit any of the pre-canned - # topics is to troll through the membership list, looking for users - # who selected at least one topic of interest, but turned on - # ReceiveNonmatchingTopics. - for user in recipients: - if not mlist.getMemberTopics(user): - # The user did not select any topics of interest, so he gets - # this message by default. - continue - if not mlist.getMemberOption( - user, config.ReceiveNonmatchingTopics): - # The user has interest in some topics, but elects not to - # receive message that match no topics, so zap him. - zap_recipients.append(user) - # Otherwise, the user wants non-matching messages. - # Prune out the non-receiving users - for user in zap_recipients: - recipients.remove(user) diff --git a/src/mailman/handlers/docs/calc-recips.rst b/src/mailman/handlers/docs/calc-recips.rst index 6dca85816..51a836c82 100644 --- a/src/mailman/handlers/docs/calc-recips.rst +++ b/src/mailman/handlers/docs/calc-recips.rst @@ -57,7 +57,7 @@ but not all of the recipients. >>> recipients = set(('qperson@example.com', 'zperson@example.com')) >>> msgdata = dict(recipients=recipients) - >>> handler = config.handlers['calculate-recipients'] + >>> handler = config.handlers['member-recipients'] >>> handler.process(mlist, msg, msgdata) >>> dump_list(msgdata['recipients']) qperson@example.com diff --git a/src/mailman/handlers/member_recipients.py b/src/mailman/handlers/member_recipients.py new file mode 100644 index 000000000..421d5a1dc --- /dev/null +++ b/src/mailman/handlers/member_recipients.py @@ -0,0 +1,149 @@ +# Copyright (C) 1998-2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Calculate the regular (i.e. non-digest) recipients of the message. + +This module calculates the non-digest recipients for the message based on the +list's membership and configuration options. It places the list of recipients +on the `recipients' attribute of the message. This attribute is used by the +SendmailDeliver and BulkDeliver modules. +""" + +from __future__ import absolute_import, unicode_literals + +__metaclass__ = type +__all__ = [ + 'MemberRecipients', + ] + +from zope.interface import implements + +from mailman.config import config +from mailman.core import errors +from mailman.core.i18n import _ +from mailman.interfaces.handler import IHandler +from mailman.interfaces.member import DeliveryStatus +from mailman.utilities.string import wrap + + + +class MemberRecipients: + """Calculate the regular (i.e. non-digest) recipients of the message.""" + + implements(IHandler) + + name = 'member-recipients' + description = _('Calculate the regular recipients of the message.') + + def process(self, mlist, msg, msgdata): + """See `IHandler`.""" + # Short circuit if we've already calculated the recipients list, + # regardless of whether the list is empty or not. + if 'recipients' in msgdata: + return + # Should the original sender should be included in the recipients list? + include_sender = True + member = mlist.members.get_member(msg.sender) + if member and not member.receive_own_postings: + include_sender = False + # Support for urgent messages, which bypasses digests and disabled + # delivery and forces an immediate delivery to all members Right Now. + # We are specifically /not/ allowing the site admins password to work + # here because we want to discourage the practice of sending the site + # admin password through email in the clear. (see also Approve.py) + # + # XXX This is broken. + missing = object() + password = msg.get('urgent', missing) + if password is not missing: + if mlist.Authenticate((config.AuthListModerator, + config.AuthListAdmin), + password): + recipients = mlist.getMemberCPAddresses( + mlist.getRegularMemberKeys() + + mlist.getDigestMemberKeys()) + msgdata['recipients'] = recipients + return + else: + # Bad Urgent: password, so reject it instead of passing it on. + # I think it's better that the sender know they screwed up + # than to deliver it normally. + listname = mlist.display_name + text = _("""\ +Your urgent message to the $listname mailing list was not authorized for +delivery. The original message as received by Mailman is attached. +""") + raise errors.RejectMessage(wrap(text)) + # Calculate the regular recipients of the message + recipients = set(member.address.email + for member in mlist.regular_members.members + if member.delivery_status == DeliveryStatus.enabled) + # Remove the sender if they don't want to receive their own posts + if not include_sender and member.address.email in recipients: + recipients.remove(member.address.email) + # Handle topic classifications + do_topic_filters(mlist, msg, msgdata, recipients) + # Bookkeeping + msgdata['recipients'] = recipients + + + +def do_topic_filters(mlist, msg, msgdata, recipients): + """Filter out recipients based on topics.""" + if not mlist.topics_enabled: + # MAS: if topics are currently disabled for the list, send to all + # regardless of ReceiveNonmatchingTopics + return + hits = msgdata.get('topichits') + zap_recipients = [] + if hits: + # The message hit some topics, so only deliver this message to those + # who are interested in one of the hit topics. + for user in recipients: + utopics = mlist.getMemberTopics(user) + if not utopics: + # This user is not interested in any topics, so they get all + # postings. + continue + # BAW: Slow, first-match, set intersection! + for topic in utopics: + if topic in hits: + # The user wants this message + break + else: + # The user was interested in topics, but not any of the ones + # this message matched, so zap him. + zap_recipients.append(user) + else: + # The semantics for a message that did not hit any of the pre-canned + # topics is to troll through the membership list, looking for users + # who selected at least one topic of interest, but turned on + # ReceiveNonmatchingTopics. + for user in recipients: + if not mlist.getMemberTopics(user): + # The user did not select any topics of interest, so he gets + # this message by default. + continue + if not mlist.getMemberOption( + user, config.ReceiveNonmatchingTopics): + # The user has interest in some topics, but elects not to + # receive message that match no topics, so zap him. + zap_recipients.append(user) + # Otherwise, the user wants non-matching messages. + # Prune out the non-receiving users + for user in zap_recipients: + recipients.remove(user) diff --git a/src/mailman/runners/docs/outgoing.rst b/src/mailman/runners/docs/outgoing.rst index b1ffb06a0..d8963ae00 100644 --- a/src/mailman/runners/docs/outgoing.rst +++ b/src/mailman/runners/docs/outgoing.rst @@ -33,7 +33,7 @@ move messages to the 'retry queue' for handling delivery failures. Normally, messages would show up in the outgoing queue after the message has been processed by the rule set and pipeline. But we can simulate that here by injecting a message directly into the outgoing queue. First though, we must -call the ``calculate-recipients`` handler so that the message metadata will be +call the ``member-recipients`` handler so that the message metadata will be populated with the list of addresses to deliver the message to. :: @@ -47,7 +47,7 @@ populated with the list of addresses to deliver the message to. ... """) >>> msgdata = {} - >>> handler = config.handlers['calculate-recipients'] + >>> handler = config.handlers['member-recipients'] >>> handler.process(mlist, msg, msgdata) >>> outgoing_queue = config.switchboards['out'] -- cgit v1.3.1 From 3241e87dc0655ba59327d0c3e4b23cee4a0a4572 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Fri, 23 Mar 2012 16:08:38 -0400 Subject: - Move some normal recipient tests to unittests. - Add unittests for owner recipients. There does eventually need to be some documentation about owner recipients. - Implement owner recipients handler. This includes moderators and owners (i.e. all the non-disabled the list administrators). - Changed the [mailman]site_owner setting in testing.cfg to noreply@example.com --- src/mailman/handlers/docs/calc-recips.rst | 35 ++--- src/mailman/handlers/member_recipients.py | 8 +- src/mailman/handlers/owner_recipients.py | 47 ++++-- src/mailman/handlers/tests/test_recipients.py | 200 ++++++++++++++++++++++++++ src/mailman/model/docs/requests.rst | 8 +- src/mailman/runners/docs/lmtp.rst | 2 +- src/mailman/testing/testing.cfg | 3 + 7 files changed, 259 insertions(+), 44 deletions(-) create mode 100644 src/mailman/handlers/tests/test_recipients.py diff --git a/src/mailman/handlers/docs/calc-recips.rst b/src/mailman/handlers/docs/calc-recips.rst index 51a836c82..1439e978f 100644 --- a/src/mailman/handlers/docs/calc-recips.rst +++ b/src/mailman/handlers/docs/calc-recips.rst @@ -6,10 +6,10 @@ Every message that makes it through to the list membership gets sent to a set of recipient addresses. These addresses are calculated by one of the handler modules and depends on a host of factors. - >>> mlist = create_list('_xtest@example.com') + >>> mlist = create_list('test@example.com') -Recipients are calculate from the list members, so add a bunch of members to -start out with. First, create a bunch of addresses... +Recipients are calculate from the list membership, so first some people +subscribe to the mailing list... :: >>> from mailman.interfaces.usermanager import IUserManager @@ -35,42 +35,25 @@ start out with. First, create a bunch of addresses... ...then make some of the members digest members. - >>> from mailman.core.constants import DeliveryMode + >>> from mailman.interfaces.member import DeliveryMode >>> member_d.preferences.delivery_mode = DeliveryMode.plaintext_digests >>> member_e.preferences.delivery_mode = DeliveryMode.mime_digests >>> member_f.preferences.delivery_mode = DeliveryMode.summary_digests -Short-circuiting -================ +Regular delivery recipients +=========================== -Sometimes, the list of recipients already exists in the message metadata. -This can happen for example, when a message was previously delivered to some -but not all of the recipients. -:: +Regular delivery recipients are those people who get messages from the list as +soon as they are posted. In other words, these folks are not digest members. >>> msg = message_from_string("""\ ... From: Xavier Person ... ... Something of great import. ... """) - >>> recipients = set(('qperson@example.com', 'zperson@example.com')) - >>> msgdata = dict(recipients=recipients) - - >>> handler = config.handlers['member-recipients'] - >>> handler.process(mlist, msg, msgdata) - >>> dump_list(msgdata['recipients']) - qperson@example.com - zperson@example.com - - -Regular delivery recipients -=========================== - -Regular delivery recipients are those people who get messages from the list as -soon as they are posted. In other words, these folks are not digest members. - >>> msgdata = {} + >>> handler = config.handlers['member-recipients'] >>> handler.process(mlist, msg, msgdata) >>> dump_list(msgdata['recipients']) aperson@example.com diff --git a/src/mailman/handlers/member_recipients.py b/src/mailman/handlers/member_recipients.py index 421d5a1dc..956ea6adc 100644 --- a/src/mailman/handlers/member_recipients.py +++ b/src/mailman/handlers/member_recipients.py @@ -23,13 +23,14 @@ on the `recipients' attribute of the message. This attribute is used by the SendmailDeliver and BulkDeliver modules. """ -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ 'MemberRecipients', ] + from zope.interface import implements from mailman.config import config @@ -82,10 +83,9 @@ class MemberRecipients: # Bad Urgent: password, so reject it instead of passing it on. # I think it's better that the sender know they screwed up # than to deliver it normally. - listname = mlist.display_name text = _("""\ -Your urgent message to the $listname mailing list was not authorized for -delivery. The original message as received by Mailman is attached. +Your urgent message to the $mlist.display_name mailing list was not authorized +for delivery. The original message as received by Mailman is attached. """) raise errors.RejectMessage(wrap(text)) # Calculate the regular recipients of the message diff --git a/src/mailman/handlers/owner_recipients.py b/src/mailman/handlers/owner_recipients.py index 9e4bbf174..80ed3e598 100644 --- a/src/mailman/handlers/owner_recipients.py +++ b/src/mailman/handlers/owner_recipients.py @@ -17,19 +17,48 @@ """Calculate the list owner recipients (includes moderators).""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ - 'process', + 'OwnerRecipients', ] +from zope.interface import implements + +from mailman.config import config +from mailman.core.i18n import _ +from mailman.interfaces.handler import IHandler +from mailman.interfaces.member import DeliveryStatus + + -def process(mlist, msg, msgdata): - """Add owner recipients.""" - # The recipients are the owner and the moderator - msgdata['recipients'] = mlist.owner + mlist.moderator - # Don't decorate these messages with the header/footers - msgdata['nodecorate'] = True - msgdata['personalize'] = False +class OwnerRecipients: + """Calculate the owner (and moderator) recipients for -owner postings.""" + + implements(IHandler) + + name = 'owner-recipients' + description = _('Calculate the owner and moderator recipients.') + + def process(self, mlist, msg, msgdata): + """See `IHandler`.""" + # Short circuit if we've already calculated the recipients list, + # regardless of whether the list is empty or not. + if 'recipients' in msgdata: + return + # -owner messages go to both the owners and moderators, which is most + # conveniently accessed via the administrators roster. + recipients = set(admin.address.email + for admin in mlist.administrators.members + if admin.delivery_status == DeliveryStatus.enabled) + # To prevent -owner messages from going into a black hole, if there + # are no administrators available, the message goes to the site owner. + if len(recipients) == 0: + msgdata['recipients'] = set((config.mailman.site_owner,)) + else: + msgdata['recipients'] = recipients + # Don't decorate these messages with the header/footers. Eventually + # we should support unique decorations for owner emails. + msgdata['nodecorate'] = True diff --git a/src/mailman/handlers/tests/test_recipients.py b/src/mailman/handlers/tests/test_recipients.py new file mode 100644 index 000000000..29cd5b64a --- /dev/null +++ b/src/mailman/handlers/tests/test_recipients.py @@ -0,0 +1,200 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Testing various recipients stuff.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestMemberRecipients', + 'TestOwnerRecipients', + ] + + +import unittest + +from zope.component import getUtility +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.interfaces.member import DeliveryMode, DeliveryStatus, MemberRole +from mailman.interfaces.usermanager import IUserManager +from mailman.testing.helpers import specialized_message_from_string as mfs +from mailman.testing.layers import ConfigLayer + + + +class TestMemberRecipients(unittest.TestCase): + """Test regular member recipient calculation.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._manager = getUtility(IUserManager) + anne = self._manager.create_address('anne@example.com') + bart = self._manager.create_address('bart@example.com') + cris = self._manager.create_address('cris@example.com') + dave = self._manager.create_address('dave@example.com') + self._anne = self._mlist.subscribe(anne, MemberRole.member) + self._bart = self._mlist.subscribe(bart, MemberRole.member) + self._cris = self._mlist.subscribe(cris, MemberRole.member) + self._dave = self._mlist.subscribe(dave, MemberRole.member) + self._process = config.handlers['member-recipients'].process + self._msg = mfs("""\ +From: Elle Person +To: test@example.com + +""") + + def test_shortcircuit(self): + # When there are already recipients in the message metadata, those are + # used instead of calculating them from the list membership. + recipients = set(('zperson@example.com', 'yperson@example.com')) + msgdata = dict(recipients=recipients) + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], recipients) + + def test_calculate_recipients(self): + # The normal path just adds the list's regular members. + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('anne@example.com', + 'bart@example.com', + 'cris@example.com', + 'dave@example.com'))) + + def test_digest_members_not_included(self): + # Digest members are not included in the recipients calculated by this + # handler. + self._cris.preferences.delivery_mode = DeliveryMode.mime_digests + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('anne@example.com', + 'bart@example.com', + 'dave@example.com'))) + + + +class TestOwnerRecipients(unittest.TestCase): + """Test owner recipient calculation.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + self._manager = getUtility(IUserManager) + anne = self._manager.create_address('anne@example.com') + bart = self._manager.create_address('bart@example.com') + cris = self._manager.create_address('cris@example.com') + dave = self._manager.create_address('dave@example.com') + # Make Cris and Dave owners of the mailing list. + self._anne = self._mlist.subscribe(anne, MemberRole.member) + self._bart = self._mlist.subscribe(bart, MemberRole.member) + self._cris = self._mlist.subscribe(cris, MemberRole.owner) + self._dave = self._mlist.subscribe(dave, MemberRole.owner) + self._process = config.handlers['owner-recipients'].process + self._msg = mfs("""\ +From: Elle Person +To: test-owner@example.com + +""") + + def test_shortcircuit(self): + # When there are already recipients in the message metadata, those are + # used instead of calculating them from the owner membership. + recipients = set(('zperson@example.com', 'yperson@example.com')) + msgdata = dict(recipients=recipients) + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], recipients) + + def test_calculate_recipients(self): + # The normal path just adds the list's owners. + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('cris@example.com', + 'dave@example.com'))) + + def test_with_moderators(self): + # Moderators are included in the owner recipient list. + elle = self._manager.create_address('elle@example.com') + fred = self._manager.create_address('fred@example.com') + gwen = self._manager.create_address('gwen@example.com') + self._mlist.subscribe(elle, MemberRole.moderator) + self._mlist.subscribe(fred, MemberRole.moderator) + self._mlist.subscribe(gwen, MemberRole.owner) + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('cris@example.com', + 'dave@example.com', + 'elle@example.com', + 'fred@example.com', + 'gwen@example.com'))) + + def test_dont_decorate(self): + # Messages to the administrators don't get decorated. + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertTrue(msgdata['nodecorate']) + + def test_omit_disabled_owners(self): + # Owner memberships can be disabled, and these folks will not get the + # messages. + self._dave.preferences.delivery_status = DeliveryStatus.by_user + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('cris@example.com',))) + + def test_include_membership_disabled_owner_enabled(self): + # If an address is subscribed to a mailing list as both an owner and a + # member, and their membership is disabled but their ownership + # subscription is not, they still get owner email. + dave = self._manager.get_address('dave@example.com') + member = self._mlist.subscribe(dave, MemberRole.member) + member.preferences.delivery_status = DeliveryStatus.by_user + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('cris@example.com', + 'dave@example.com'))) + # Dave disables his owner membership but re-enables his list + # membership. He will not get the owner emails now. + member.preferences.delivery_status = DeliveryStatus.enabled + self._dave.preferences.delivery_status = DeliveryStatus.by_user + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('cris@example.com',))) + + def test_all_owners_disabled(self): + # If all the owners are disabled, then the site owner gets the + # message. This prevents a list's -owner address from going into a + # black hole. + self._cris.preferences.delivery_status = DeliveryStatus.by_user + self._dave.preferences.delivery_status = DeliveryStatus.by_user + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('noreply@example.com',))) + + def test_no_owners(self): + # If a list has no owners or moderators, then the site owner gets the + # message. This prevents a list's -owner address from going into a + # black hole. + self._cris.unsubscribe() + self._dave.unsubscribe() + self.assertEqual(self._mlist.administrators.member_count, 0) + msgdata = {} + self._process(self._mlist, self._msg, msgdata) + self.assertEqual(msgdata['recipients'], set(('noreply@example.com',))) diff --git a/src/mailman/model/docs/requests.rst b/src/mailman/model/docs/requests.rst index 1e461e36c..068cc84f6 100644 --- a/src/mailman/model/docs/requests.rst +++ b/src/mailman/model/docs/requests.rst @@ -664,7 +664,7 @@ The admin message is sent to the moderators. Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Subject: A Test List subscription notification - From: changeme@example.com + From: noreply@example.com To: alist-owner@example.com Message-ID: ... Date: ... @@ -676,7 +676,7 @@ The admin message is sent to the moderators. >>> dump_msgdata(messages[0].msgdata) _parsemsg : False - envsender : changeme@example.com + envsender : noreply@example.com listname : alist@example.com nodecorate : True recipients : set([]) @@ -888,7 +888,7 @@ The goodbye message... Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Subject: A Test List unsubscription notification - From: changeme@example.com + From: noreply@example.com To: alist-owner@example.com Message-ID: ... Date: ... @@ -899,7 +899,7 @@ The goodbye message... >>> dump_msgdata(messages[1].msgdata) _parsemsg : False - envsender : changeme@example.com + envsender : noreply@example.com listname : alist@example.com nodecorate : True recipients : set([]) diff --git a/src/mailman/runners/docs/lmtp.rst b/src/mailman/runners/docs/lmtp.rst index 2b6c4b42b..3ce145907 100644 --- a/src/mailman/runners/docs/lmtp.rst +++ b/src/mailman/runners/docs/lmtp.rst @@ -306,7 +306,7 @@ Messages to the `-owner` address also go to the incoming processor. 1 >>> dump_msgdata(messages[0].msgdata) _parsemsg : False - envsender : changeme@example.com + envsender : noreply@example.com listname : mylist@example.com original_size: ... subaddress : owner diff --git a/src/mailman/testing/testing.cfg b/src/mailman/testing/testing.cfg index d503247de..b7e80ff02 100644 --- a/src/mailman/testing/testing.cfg +++ b/src/mailman/testing/testing.cfg @@ -22,6 +22,9 @@ #class: mailman.database.postgresql.PostgreSQLDatabase #url: postgres://barry:barry@localhost/mailman +[mailman] +site_owner: noreply@example.com + [mta] smtp_port: 9025 lmtp_port: 9024 -- cgit v1.3.1 From cc949e304047a65e99c6089fc417883f7c411300 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Fri, 23 Mar 2012 16:34:54 -0400 Subject: Add a doctest for owner recipients. --- src/mailman/handlers/docs/calc-recips.rst | 97 ----------------------------- src/mailman/handlers/docs/member-recips.rst | 97 +++++++++++++++++++++++++++++ src/mailman/handlers/docs/owner-recips.rst | 63 +++++++++++++++++++ 3 files changed, 160 insertions(+), 97 deletions(-) delete mode 100644 src/mailman/handlers/docs/calc-recips.rst create mode 100644 src/mailman/handlers/docs/member-recips.rst create mode 100644 src/mailman/handlers/docs/owner-recips.rst diff --git a/src/mailman/handlers/docs/calc-recips.rst b/src/mailman/handlers/docs/calc-recips.rst deleted file mode 100644 index 1439e978f..000000000 --- a/src/mailman/handlers/docs/calc-recips.rst +++ /dev/null @@ -1,97 +0,0 @@ -====================== -Calculating recipients -====================== - -Every message that makes it through to the list membership gets sent to a set -of recipient addresses. These addresses are calculated by one of the handler -modules and depends on a host of factors. - - >>> mlist = create_list('test@example.com') - -Recipients are calculate from the list membership, so first some people -subscribe to the mailing list... -:: - - >>> from mailman.interfaces.usermanager import IUserManager - >>> from zope.component import getUtility - >>> user_manager = getUtility(IUserManager) - - >>> address_a = user_manager.create_address('aperson@example.com') - >>> address_b = user_manager.create_address('bperson@example.com') - >>> address_c = user_manager.create_address('cperson@example.com') - >>> address_d = user_manager.create_address('dperson@example.com') - >>> address_e = user_manager.create_address('eperson@example.com') - >>> address_f = user_manager.create_address('fperson@example.com') - -...then subscribe these addresses to the mailing list as members... - - >>> from mailman.interfaces.member import MemberRole - >>> member_a = mlist.subscribe(address_a, MemberRole.member) - >>> member_b = mlist.subscribe(address_b, MemberRole.member) - >>> member_c = mlist.subscribe(address_c, MemberRole.member) - >>> member_d = mlist.subscribe(address_d, MemberRole.member) - >>> member_e = mlist.subscribe(address_e, MemberRole.member) - >>> member_f = mlist.subscribe(address_f, MemberRole.member) - -...then make some of the members digest members. - - >>> from mailman.interfaces.member import DeliveryMode - >>> member_d.preferences.delivery_mode = DeliveryMode.plaintext_digests - >>> member_e.preferences.delivery_mode = DeliveryMode.mime_digests - >>> member_f.preferences.delivery_mode = DeliveryMode.summary_digests - - -Regular delivery recipients -=========================== - -Regular delivery recipients are those people who get messages from the list as -soon as they are posted. In other words, these folks are not digest members. - - >>> msg = message_from_string("""\ - ... From: Xavier Person - ... - ... Something of great import. - ... """) - >>> msgdata = {} - >>> handler = config.handlers['member-recipients'] - >>> handler.process(mlist, msg, msgdata) - >>> dump_list(msgdata['recipients']) - aperson@example.com - bperson@example.com - cperson@example.com - -Members can elect not to receive a list copy of their own postings. - - >>> member_c.preferences.receive_own_postings = False - >>> msg = message_from_string("""\ - ... From: Claire Person - ... - ... Something of great import. - ... """) - >>> msgdata = {} - >>> handler.process(mlist, msg, msgdata) - >>> dump_list(msgdata['recipients']) - aperson@example.com - bperson@example.com - -Members can also elect not to receive a list copy of any message on which they -are explicitly named as a recipient. However, see the `avoid duplicates`_ -handler for details. - - -Digest recipients -================= - -XXX Test various digest deliveries. - - -Urgent messages -=============== - -XXX Test various urgent deliveries: - * test_urgent_moderator() - * test_urgent_admin() - * test_urgent_reject() - - -.. _`avoid duplicates`: avoid-duplicates.html diff --git a/src/mailman/handlers/docs/member-recips.rst b/src/mailman/handlers/docs/member-recips.rst new file mode 100644 index 000000000..1439e978f --- /dev/null +++ b/src/mailman/handlers/docs/member-recips.rst @@ -0,0 +1,97 @@ +====================== +Calculating recipients +====================== + +Every message that makes it through to the list membership gets sent to a set +of recipient addresses. These addresses are calculated by one of the handler +modules and depends on a host of factors. + + >>> mlist = create_list('test@example.com') + +Recipients are calculate from the list membership, so first some people +subscribe to the mailing list... +:: + + >>> from mailman.interfaces.usermanager import IUserManager + >>> from zope.component import getUtility + >>> user_manager = getUtility(IUserManager) + + >>> address_a = user_manager.create_address('aperson@example.com') + >>> address_b = user_manager.create_address('bperson@example.com') + >>> address_c = user_manager.create_address('cperson@example.com') + >>> address_d = user_manager.create_address('dperson@example.com') + >>> address_e = user_manager.create_address('eperson@example.com') + >>> address_f = user_manager.create_address('fperson@example.com') + +...then subscribe these addresses to the mailing list as members... + + >>> from mailman.interfaces.member import MemberRole + >>> member_a = mlist.subscribe(address_a, MemberRole.member) + >>> member_b = mlist.subscribe(address_b, MemberRole.member) + >>> member_c = mlist.subscribe(address_c, MemberRole.member) + >>> member_d = mlist.subscribe(address_d, MemberRole.member) + >>> member_e = mlist.subscribe(address_e, MemberRole.member) + >>> member_f = mlist.subscribe(address_f, MemberRole.member) + +...then make some of the members digest members. + + >>> from mailman.interfaces.member import DeliveryMode + >>> member_d.preferences.delivery_mode = DeliveryMode.plaintext_digests + >>> member_e.preferences.delivery_mode = DeliveryMode.mime_digests + >>> member_f.preferences.delivery_mode = DeliveryMode.summary_digests + + +Regular delivery recipients +=========================== + +Regular delivery recipients are those people who get messages from the list as +soon as they are posted. In other words, these folks are not digest members. + + >>> msg = message_from_string("""\ + ... From: Xavier Person + ... + ... Something of great import. + ... """) + >>> msgdata = {} + >>> handler = config.handlers['member-recipients'] + >>> handler.process(mlist, msg, msgdata) + >>> dump_list(msgdata['recipients']) + aperson@example.com + bperson@example.com + cperson@example.com + +Members can elect not to receive a list copy of their own postings. + + >>> member_c.preferences.receive_own_postings = False + >>> msg = message_from_string("""\ + ... From: Claire Person + ... + ... Something of great import. + ... """) + >>> msgdata = {} + >>> handler.process(mlist, msg, msgdata) + >>> dump_list(msgdata['recipients']) + aperson@example.com + bperson@example.com + +Members can also elect not to receive a list copy of any message on which they +are explicitly named as a recipient. However, see the `avoid duplicates`_ +handler for details. + + +Digest recipients +================= + +XXX Test various digest deliveries. + + +Urgent messages +=============== + +XXX Test various urgent deliveries: + * test_urgent_moderator() + * test_urgent_admin() + * test_urgent_reject() + + +.. _`avoid duplicates`: avoid-duplicates.html diff --git a/src/mailman/handlers/docs/owner-recips.rst b/src/mailman/handlers/docs/owner-recips.rst new file mode 100644 index 000000000..e62551ba6 --- /dev/null +++ b/src/mailman/handlers/docs/owner-recips.rst @@ -0,0 +1,63 @@ +===================== +List owner recipients +===================== + +When a message is posted to a mailing list's `-owners` address, all of the +list's administrators will receive a copy. The administrators are defined as +the set of owners and moderators. + + >>> mlist_1 = create_list('alpha@example.com') + +Anne is the owner of the list and Bart is a moderator of the list. + + >>> from mailman.interfaces.usermanager import IUserManager + >>> from zope.component import getUtility + >>> user_manager = getUtility(IUserManager) + >>> anne_addr = user_manager.create_address('anne@example.com') + >>> bart_addr = user_manager.create_address('bart@example.com') + >>> from mailman.interfaces.member import MemberRole + >>> anne = mlist_1.subscribe(anne_addr, MemberRole.owner) + >>> bart = mlist_1.subscribe(bart_addr, MemberRole.moderator) + +The recipients list for the `-owners` address includes both Anne and Bart. + + >>> msg = message_from_string("""\ + ... From: Xavier Person + ... To: alpha@example.com + ... + ... """) + >>> msgdata = {} + >>> handler = config.handlers['owner-recipients'] + >>> handler.process(mlist_1, msg, msgdata) + >>> dump_list(msgdata['recipients']) + anne@example.com + bart@example.com + +Anne disables her owner delivery, so she will not receive `-owner` emails. + + >>> from mailman.interfaces.member import DeliveryStatus + >>> anne.preferences.delivery_status = DeliveryStatus.by_user + >>> msgdata = {} + >>> handler.process(mlist_1, msg, msgdata) + >>> dump_list(msgdata['recipients']) + bart@example.com + +If Bart also disables his owner delivery, then no one could contact the list's +owners. Since this is unacceptable, the site owner is used as a fallback. + + >>> bart.preferences.delivery_status = DeliveryStatus.by_user + >>> msgdata = {} + >>> handler.process(mlist_1, msg, msgdata) + >>> dump_list(msgdata['recipients']) + noreply@example.com + +For mailing lists which have no owners at all, the site owner is also used as +a fallback. + + >>> mlist_2 = create_list('beta@example.com') + >>> mlist_2.administrators.member_count + 0 + >>> msgdata = {} + >>> handler.process(mlist_2, msg, msgdata) + >>> dump_list(msgdata['recipients']) + noreply@example.com -- cgit v1.3.1 From 1d9fdeb0e587352a3bc66a3263152614bb6085dd Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Fri, 23 Mar 2012 17:26:19 -0400 Subject: Add the OwnerPipeline, with tests, and rename some things for consistency. --- src/mailman/core/pipelines.py | 29 +++++++++++++++----- src/mailman/core/tests/test_pipelines.py | 46 +++++++++++++++++++++++++++++++- src/mailman/styles/default.py | 5 ++++ 3 files changed, 73 insertions(+), 7 deletions(-) diff --git a/src/mailman/core/pipelines.py b/src/mailman/core/pipelines.py index 5c0eb7f71..25bb68030 100644 --- a/src/mailman/core/pipelines.py +++ b/src/mailman/core/pipelines.py @@ -15,12 +15,16 @@ # You should have received a copy of the GNU General Public License along with # GNU Mailman. If not, see . -"""Pipeline processor.""" +"""Built-in pipelines.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ + 'BasePipeline', + 'OwnerPipeline', + 'PostingPipeline', + 'VirginPipeline', 'initialize', 'process', ] @@ -89,11 +93,24 @@ class BasePipeline: yield handler -class BuiltInPipeline(BasePipeline): - """The built-in pipeline.""" + +class OwnerPipeline(BasePipeline): + """The built-in owner pipeline.""" + + name = 'default-owner-pipeline' + description = _('The built-in owner pipeline.') + + _default_handlers = ( + 'owner-recipients', + 'to-outgoing', + ) + + +class PostingPipeline(BasePipeline): + """The built-in posting pipeline.""" name = 'default-posting-pipeline' - description = _('The built-in pipeline.') + description = _('The built-in posting pipeline.') _default_handlers = ( 'mime-delete', @@ -139,6 +156,6 @@ def initialize(): handler.name, handler_class)) config.handlers[handler.name] = handler # Set up some pipelines. - for pipeline_class in (BuiltInPipeline, VirginPipeline): + for pipeline_class in (OwnerPipeline, PostingPipeline, VirginPipeline): pipeline = pipeline_class() config.pipelines[pipeline.name] = pipeline diff --git a/src/mailman/core/tests/test_pipelines.py b/src/mailman/core/tests/test_pipelines.py index 0cf3732c9..8f851de95 100644 --- a/src/mailman/core/tests/test_pipelines.py +++ b/src/mailman/core/tests/test_pipelines.py @@ -21,11 +21,14 @@ from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ + 'TestOwnerPipeline', + 'TestPostingPipeline', ] import unittest +from zope.component import getUtility from zope.interface import implements from mailman.app.lifecycle import create_list @@ -33,7 +36,9 @@ from mailman.config import config from mailman.core.errors import DiscardMessage, RejectMessage from mailman.core.pipelines import process from mailman.interfaces.handler import IHandler +from mailman.interfaces.member import MemberRole from mailman.interfaces.pipeline import IPipeline +from mailman.interfaces.usermanager import IUserManager from mailman.testing.helpers import ( LogFileMark, get_queue_messages, @@ -78,7 +83,7 @@ class RejectingPipeline: -class TestBuiltinPipeline(unittest.TestCase): +class TestPostingPipeline(unittest.TestCase): """Test various aspects of the built-in postings pipeline.""" layer = ConfigLayer @@ -133,3 +138,42 @@ testing messages = get_queue_messages('virgin') self.assertEqual(len(messages), 1) self.assertEqual(str(messages[0].msg['subject']), 'a test') + + + +class TestOwnerPipeline(unittest.TestCase): + """Test various aspects of the built-in owner pipeline.""" + + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + user_manager = getUtility(IUserManager) + anne = user_manager.create_address('anne@example.com') + bart = user_manager.create_address('bart@example.com') + self._mlist.subscribe(anne, MemberRole.owner) + self._mlist.subscribe(bart, MemberRole.moderator) + self._msg = mfs("""\ +From: Anne Person +To: test-owner@example.com + +""") + + def test_calculate_recipients(self): + # Recipients are the administrators of the mailing list. + msgdata = dict(listname='test@example.com', + to_owner=True) + process(self._mlist, self._msg, msgdata, + pipeline_name='default-owner-pipeline') + self.assertEqual(msgdata['recipients'], set(('anne@example.com', + 'bart@example.com'))) + + def test_to_outgoing(self): + # The message, with the calculated recipients, gets put in the + # outgoing queue. + process(self._mlist, self._msg, {}, + pipeline_name='default-owner-pipeline') + messages = get_queue_messages('out', sort_on='to') + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0].msgdata['recipients'], + set(('anne@example.com', 'bart@example.com'))) diff --git a/src/mailman/styles/default.py b/src/mailman/styles/default.py index e64bbe40b..18a145d3c 100644 --- a/src/mailman/styles/default.py +++ b/src/mailman/styles/default.py @@ -218,6 +218,11 @@ from: .*@uplinkpro.com # The default pipeline to send accepted messages through to the # mailing list's members. mlist.posting_pipeline = 'default-posting-pipeline' + # The processing chain that messages posted to this mailing list's + # -owner address gets processed by. + mlist.owner_chain = 'default-owner-chain' + # The default pipeline to send -owner email through. + mlist.owner_pipeline = 'default-owner-pipeline' def match(self, mailing_list, styles): """See `IStyle`.""" -- cgit v1.3.1 From e005e1b12fa0bd82d2e126df476b5505b440ce36 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Fri, 23 Mar 2012 19:15:43 -0400 Subject: Verp all messages to the -owners. Add a test to prove that email to -owners now works as expected! --- src/mailman/handlers/owner_recipients.py | 3 + src/mailman/runners/tests/test_owner.py | 142 +++++++++++++++++++++++++++++++ src/mailman/testing/layers.py | 8 ++ 3 files changed, 153 insertions(+) create mode 100644 src/mailman/runners/tests/test_owner.py diff --git a/src/mailman/handlers/owner_recipients.py b/src/mailman/handlers/owner_recipients.py index 80ed3e598..e431d00cf 100644 --- a/src/mailman/handlers/owner_recipients.py +++ b/src/mailman/handlers/owner_recipients.py @@ -62,3 +62,6 @@ class OwnerRecipients: # Don't decorate these messages with the header/footers. Eventually # we should support unique decorations for owner emails. msgdata['nodecorate'] = True + # We should probably always VERP deliveries to the owners. We + # *really* want to know if they are bouncing. + msgdata['verp'] = True diff --git a/src/mailman/runners/tests/test_owner.py b/src/mailman/runners/tests/test_owner.py new file mode 100644 index 000000000..8264ef0d8 --- /dev/null +++ b/src/mailman/runners/tests/test_owner.py @@ -0,0 +1,142 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Test posting to a mailing list's -owner address.""" + +# XXX 2012-03-23 BAW: This is not necessarily the best place for this test. +# We really need a better place to collect these sort of end-to-end posting +# tests. They're not exactly integration tests, but they do touch lots of +# parts of the system. + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestEmailToOwner', + ] + + +import unittest + +from operator import itemgetter +from zope.component import getUtility + +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.interfaces.member import MemberRole +from mailman.interfaces.usermanager import IUserManager +from mailman.testing.helpers import ( + TestableMaster, + get_lmtp_client, + make_testable_runner) +from mailman.runners.incoming import IncomingRunner +from mailman.runners.outgoing import OutgoingRunner +from mailman.runners.pipeline import PipelineRunner +from mailman.testing.layers import SMTPLayer + + + +class TestEmailToOwner(unittest.TestCase): + """Test emailing a mailing list's -owner address.""" + + layer = SMTPLayer + + def setUp(self): + self._mlist = create_list('test@example.com') + # Add some owners, moderators, and members + manager = getUtility(IUserManager) + anne = manager.create_address('anne@example.com') + bart = manager.create_address('bart@example.com') + cris = manager.create_address('cris@example.com') + dave = manager.create_address('dave@example.com') + self._mlist.subscribe(anne, MemberRole.member) + self._mlist.subscribe(anne, MemberRole.owner) + self._mlist.subscribe(bart, MemberRole.moderator) + self._mlist.subscribe(bart, MemberRole.owner) + self._mlist.subscribe(cris, MemberRole.moderator) + self._mlist.subscribe(dave, MemberRole.member) + config.db.commit() + self._inq = make_testable_runner(IncomingRunner, 'in') + self._pipelineq = make_testable_runner(PipelineRunner, 'pipeline') + self._outq = make_testable_runner(OutgoingRunner, 'out') + # Python 2.7 has assertMultiLineEqual. Let this work without bounds. + self.maxDiff = None + self.eq = getattr(self, 'assertMultiLineEqual', self.assertEqual) + + def test_owners_get_email(self): + # XXX 2012-03-23 BAW: We can't use a layer here because we need both + # the SMTPLayer and LMTPLayer and these are incompatible. There's no + # way to make zope.test* happy without causing errors or worse. Live + # with this hack until we can rip all that layer crap out and use + # something like testresources. + def wait(): + get_lmtp_client(quiet=True) + lmtpd = TestableMaster(wait) + lmtpd.start() + # Post a message to the list's -owner address, and all the owners will + # get a copy of the message. + lmtp = get_lmtp_client(quiet=True) + lmtp.lhlo('remote.example.org') + lmtp.sendmail('zuzu@example.org', ['test-owner@example.com'], """\ +From: Zuzu Person +To: test-owner@example.com +Message-ID: + +Can you help me? +""") + lmtpd.stop() + # There should now be one message sitting in the incoming queue. + # Check that, then process it. Don't use get_queue_messages() since + # that will empty the queue. + self.assertEqual(len(config.switchboards['in'].files), 1) + self._inq.run() + # There should now be one message sitting in the pipeline queue. + # Process that one too. + self.assertEqual(len(config.switchboards['pipeline'].files), 1) + self._pipelineq.run() + # The message has made its way to the outgoing queue. Again, check + # and process that one. + self.assertEqual(len(config.switchboards['out'].files), 1) + self._outq.run() + # The SMTP server has now received three messages, one for each of the + # owners and moderators. Of course, Bart is both an owner and a + # moderator, so he'll get only one copy of the message. Dave does not + # get a copy of the message. + messages = sorted(SMTPLayer.smtpd.messages, key=itemgetter('x-rcptto')) + self.assertEqual(len(messages), 3) + self.assertEqual(messages[0]['x-rcptto'], 'anne@example.com') + self.assertEqual(messages[1]['x-rcptto'], 'bart@example.com') + self.assertEqual(messages[2]['x-rcptto'], 'cris@example.com') + # And yet, all three messages are addressed to the -owner address. + for message in messages: + self.assertEqual(message['to'], 'test-owner@example.com') + # All three messages will have two X-MailFrom headers. One is added + # by the LMTP server accepting Zuzu's original message, and will + # contain her posting address, i.e. zuzu@example.com. The second one + # is added by the lazr.smtptest server that accepts Mailman's VERP'd + # message to the individual recipient. By verifying both, we prove + # that Zuzu sent the original message, and that Mailman is VERP'ing + # the copy to all the owners. + self.assertEqual( + messages[0].get_all('x-mailfrom'), + ['zuzu@example.org', 'test-bounces+anne=example.com@example.com']) + self.assertEqual( + messages[1].get_all('x-mailfrom'), + ['zuzu@example.org', 'test-bounces+bart=example.com@example.com']) + self.assertEqual( + messages[2].get_all('x-mailfrom'), + ['zuzu@example.org', 'test-bounces+cris=example.com@example.com']) diff --git a/src/mailman/testing/layers.py b/src/mailman/testing/layers.py index 04ab8f91f..41ef86935 100644 --- a/src/mailman/testing/layers.py +++ b/src/mailman/testing/layers.py @@ -17,6 +17,14 @@ """Mailman test layers.""" +# XXX 2012-03-23 BAW: Layers really really suck. For example, the +# test_owners_get_email() test requires that both the SMTPLayer and LMTPLayer +# be set up, but there's apparently no way to do that and make zope.testing +# happy. This causes no tests failures, but it does cause errors at the end +# of the full test run. For now, I'll ignore that, but I do want to +# eventually get rid of the zope.test* dependencies and use something like +# testresources or some such. + from __future__ import absolute_import, unicode_literals __metaclass__ = type -- cgit v1.3.1