diff options
Diffstat (limited to 'src/mailman/queue/docs')
| -rw-r--r-- | src/mailman/queue/docs/OVERVIEW.txt | 78 | ||||
| -rw-r--r-- | src/mailman/queue/docs/archiver.txt | 34 | ||||
| -rw-r--r-- | src/mailman/queue/docs/command.txt | 170 | ||||
| -rw-r--r-- | src/mailman/queue/docs/incoming.txt | 200 | ||||
| -rw-r--r-- | src/mailman/queue/docs/lmtp.txt | 103 | ||||
| -rw-r--r-- | src/mailman/queue/docs/news.txt | 157 | ||||
| -rw-r--r-- | src/mailman/queue/docs/outgoing.txt | 75 | ||||
| -rw-r--r-- | src/mailman/queue/docs/runner.txt | 72 | ||||
| -rw-r--r-- | src/mailman/queue/docs/switchboard.txt | 182 |
9 files changed, 1071 insertions, 0 deletions
diff --git a/src/mailman/queue/docs/OVERVIEW.txt b/src/mailman/queue/docs/OVERVIEW.txt new file mode 100644 index 000000000..643fa8a5c --- /dev/null +++ b/src/mailman/queue/docs/OVERVIEW.txt @@ -0,0 +1,78 @@ +Alias overview +============== + +A typical Mailman list exposes nine aliases which point to seven different +wrapped scripts. E.g. for a list named `mylist', you'd have: + + mylist-bounces -> bounces + mylist-confirm -> confirm + mylist-join -> join (-subscribe is an alias) + mylist-leave -> leave (-unsubscribe is an alias) + mylist-owner -> owner + mylist -> post + mylist-request -> request + +-request, -join, and -leave are a robot addresses; their sole purpose is to +process emailed commands, although the latter two are hardcoded to +subscription and unsubscription requests. -bounces is the automated bounce +processor, and all messages to list members have their return address set to +-bounces. If the bounce processor fails to extract a bouncing member address, +it can optionally forward the message on to the list owners. + +-owner is for reaching a human operator with minimal list interaction (i.e. no +bounce processing). -confirm is another robot address which processes replies +to VERP-like confirmation notices. + +So delivery flow of messages look like this: + + joerandom ---> mylist ---> list members + | | + | |[bounces] + | mylist-bounces <---+ <-------------------------------+ + | | | + | +--->[internal bounce processing] | + | ^ | | + | | | [bounce found] | + | [bounces *] +--->[register and discard] | + | | | | | + | | | |[*] | + | [list owners] |[no bounce found] | | + | ^ | | | + | | | | | + +-------> mylist-owner <--------+ | | + | | | + | data/owner-bounces.mbox <--[site list] <---+ | + | | + +-------> mylist-join--+ | + | | | + +------> mylist-leave--+ | + | | | + | v | + +-------> mylist-request | + | | | + | +---> [command processor] | + | | | + +-----> mylist-confirm ----> +---> joerandom | + | | + |[bounces] | + +----------------------+ + +A person can send an email to the list address (for posting), the -owner +address (to reach the human operator), or the -confirm, -join, -leave, and +-request mailbots. Message to the list address are then forwarded on to the +list membership, with bounces directed to the -bounces address. + +[*] Messages sent to the -owner address are forwarded on to the list +owner/moderators. All -owner destined messages have their bounces directed to +the site list -bounces address, regardless of whether a human sent the message +or the message was crafted internally. The intention here is that the site +owners want to be notified when one of their list owners' addresses starts +bouncing (yes, the will be automated in a future release). + +Any messages to site owners has their bounces directed to a special +"loop-killer" address, which just dumps the message into +data/owners-bounces.mbox. + +Finally, message to any of the mailbots causes the requested action to be +performed. Results notifications are sent to the author of the message, which +all bounces pointing back to the -bounces address. diff --git a/src/mailman/queue/docs/archiver.txt b/src/mailman/queue/docs/archiver.txt new file mode 100644 index 000000000..601857cd9 --- /dev/null +++ b/src/mailman/queue/docs/archiver.txt @@ -0,0 +1,34 @@ +Archiving +========= + +Mailman can archive to any number of archivers that adhere to the IArchiver +interface. By default, there's a Pipermail archiver. + + >>> from mailman.app.lifecycle import create_list + >>> mlist = create_list(u'test@example.com') + >>> commit() + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: test@example.com + ... Subject: My first post + ... Message-ID: <first> + ... + ... First post! + ... """) + + >>> archiver_queue = config.switchboards['archive'] + >>> ignore = archiver_queue.enqueue(msg, {}, listname=mlist.fqdn_listname) + + >>> from mailman.queue.archive import ArchiveRunner + >>> from mailman.testing.helpers import make_testable_runner + >>> runner = make_testable_runner(ArchiveRunner) + >>> runner.run() + + # The best we can do is verify some landmark exists. Let's use the + # Pipermail pickle file exists. + >>> listname = mlist.fqdn_listname + >>> import os + >>> os.path.exists(os.path.join( + ... config.PUBLIC_ARCHIVE_FILE_DIR, listname, 'pipermail.pck')) + True diff --git a/src/mailman/queue/docs/command.txt b/src/mailman/queue/docs/command.txt new file mode 100644 index 000000000..0b384de01 --- /dev/null +++ b/src/mailman/queue/docs/command.txt @@ -0,0 +1,170 @@ +The command queue runner +======================== + +This queue runner's purpose is to process and respond to email commands. +Commands are extensible using the Mailman plugin system, but Mailman comes +with a number of email commands out of the box. These are processed when a +message is sent to the list's -request address. + + >>> from mailman.app.lifecycle import create_list + >>> mlist = create_list(u'test@example.com') + + +A command in the Subject +------------------------ + +For example, the 'echo' command simply echoes the original command back to the +sender. The command can be in the Subject header. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: test-request@example.com + ... Subject: echo hello + ... Message-ID: <aardvark> + ... + ... """) + + >>> from mailman.inject import inject_message + >>> inject_message(mlist, msg, switchboard='command') + >>> from mailman.queue.command import CommandRunner + >>> from mailman.testing.helpers import make_testable_runner + >>> command = make_testable_runner(CommandRunner) + >>> command.run() + +And now the response is in the virgin queue. + + >>> from mailman.queue import Switchboard + >>> virgin_queue = config.switchboards['virgin'] + >>> len(virgin_queue.files) + 1 + >>> from mailman.testing.helpers import get_queue_messages + >>> item = get_queue_messages('virgin')[0] + >>> print item.msg.as_string() + Subject: The results of your email commands + From: test-bounces@example.com + To: aperson@example.com + ... + <BLANKLINE> + The results of your email command are provided below. + <BLANKLINE> + - Original message details: + From: aperson@example.com + Subject: echo hello + Date: ... + Message-ID: <aardvark> + <BLANKLINE> + - Results: + echo hello + <BLANKLINE> + - Done. + <BLANKLINE> + >>> sorted(item.msgdata.items()) + [..., ('listname', u'test@example.com'), ..., + ('recips', [u'aperson@example.com']), + ...] + + +A command in the body +--------------------- + +The command can also be found in the body of the message, as long as the +message is plain text. + + >>> msg = message_from_string("""\ + ... From: bperson@example.com + ... To: test-request@example.com + ... Message-ID: <bobcat> + ... + ... echo foo bar + ... """) + + >>> inject_message(mlist, msg, switchboard='command') + >>> command.run() + >>> len(virgin_queue.files) + 1 + >>> item = get_queue_messages('virgin')[0] + >>> print item.msg.as_string() + Subject: The results of your email commands + From: test-bounces@example.com + To: bperson@example.com + ... + Precedence: bulk + <BLANKLINE> + The results of your email command are provided below. + <BLANKLINE> + - Original message details: + From: bperson@example.com + Subject: n/a + Date: ... + Message-ID: <bobcat> + <BLANKLINE> + - Results: + echo foo bar + <BLANKLINE> + - Done. + <BLANKLINE> + + +Stopping command processing +--------------------------- + +The 'end' command stops email processing, so that nothing following is looked +at by the command queue. + + >>> msg = message_from_string("""\ + ... From: cperson@example.com + ... To: test-request@example.com + ... Message-ID: <caribou> + ... + ... echo foo bar + ... end ignored + ... echo baz qux + ... """) + + >>> inject_message(mlist, msg, switchboard='command') + >>> command.run() + >>> len(virgin_queue.files) + 1 + >>> item = get_queue_messages('virgin')[0] + >>> print item.msg.as_string() + Subject: The results of your email commands + ... + <BLANKLINE> + - Results: + echo foo bar + <BLANKLINE> + - Unprocessed: + echo baz qux + <BLANKLINE> + - Done. + <BLANKLINE> + +The 'stop' command is an alias for 'end'. + + >>> msg = message_from_string("""\ + ... From: cperson@example.com + ... To: test-request@example.com + ... Message-ID: <caribou> + ... + ... echo foo bar + ... stop ignored + ... echo baz qux + ... """) + + >>> inject_message(mlist, msg, switchboard='command') + >>> command.run() + >>> len(virgin_queue.files) + 1 + >>> item = get_queue_messages('virgin')[0] + >>> print item.msg.as_string() + Subject: The results of your email commands + ... + <BLANKLINE> + - Results: + echo foo bar + <BLANKLINE> + - Unprocessed: + echo baz qux + <BLANKLINE> + - Done. + <BLANKLINE> diff --git a/src/mailman/queue/docs/incoming.txt b/src/mailman/queue/docs/incoming.txt new file mode 100644 index 000000000..deb340e71 --- /dev/null +++ b/src/mailman/queue/docs/incoming.txt @@ -0,0 +1,200 @@ +The incoming queue runner +========================= + +This runner's sole purpose in life is to decide the disposition of the +message. It can either be accepted for delivery, rejected (i.e. bounced), +held for moderator approval, or discarded. + +The runner operates by processing chains on a message/metadata pair in the +context of a mailing list. Each mailing list may have a 'start chain' where +processing begins, with a global default. This chain is processed with the +message eventually ending up in one of the four disposition states described +above. + + >>> from mailman.app.lifecycle import create_list + >>> mlist = create_list(u'_xtest@example.com') + >>> mlist.start_chain + u'built-in' + + +Accepted messages +----------------- + +We have a message that is going to be sent to the mailing list. This message +is so perfectly fine for posting that it will be accepted and forward to the +pipeline queue. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... Subject: My first post + ... Message-ID: <first> + ... + ... First post! + ... """) + +Normally, the upstream mail server would drop the message in the incoming +queue, but this is an effective simulation. + + >>> from mailman.inject import inject_message + >>> inject_message(mlist, msg) + +The incoming queue runner runs until it is empty. + + >>> from mailman.queue.incoming import IncomingRunner + >>> from mailman.testing.helpers import make_testable_runner + >>> incoming = make_testable_runner(IncomingRunner, 'in') + >>> incoming.run() + +And now the message is in the pipeline queue. + + >>> pipeline_queue = config.switchboards['pipeline'] + >>> len(pipeline_queue.files) + 1 + >>> incoming_queue = config.switchboards['in'] + >>> len(incoming_queue.files) + 0 + >>> from mailman.testing.helpers import get_queue_messages + >>> item = get_queue_messages('pipeline')[0] + >>> print item.msg.as_string() + From: aperson@example.com + To: _xtest@example.com + Subject: My first post + Message-ID: <first> + Date: ... + X-Mailman-Rule-Misses: approved; emergency; loop; administrivia; + implicit-dest; + max-recipients; max-size; news-moderation; no-subject; + suspicious-header + <BLANKLINE> + First post! + <BLANKLINE> + >>> sorted(item.msgdata.items()) + [...('envsender', u'noreply@example.com')...('tolist', True)...] + + +Held messages +------------- + +The list moderator sets the emergency flag on the mailing list. The built-in +chain will now hold all posted messages, so nothing will show up in the +pipeline queue. + + # XXX This checks the vette log file because there is no other evidence + # that this chain has done anything. + >>> import os + >>> fp = open(os.path.join(config.LOG_DIR, 'vette')) + >>> fp.seek(0, 2) + + >>> mlist.emergency = True + >>> inject_message(mlist, msg) + >>> file_pos = fp.tell() + >>> incoming.run() + >>> len(pipeline_queue.files) + 0 + >>> len(incoming_queue.files) + 0 + >>> fp.seek(file_pos) + >>> print 'LOG:', fp.read() + LOG: ... HOLD: _xtest@example.com post from aperson@example.com held, + message-id=<first>: n/a + <BLANKLINE> + + >>> mlist.emergency = False + + +Discarded messages +------------------ + +Another possibility is that the message would get immediately discarded. The +built-in chain does not have such a disposition by default, so let's craft a +new chain and set it as the mailing list's start chain. + + >>> from mailman.chains.base import Chain, Link + >>> from mailman.interfaces.chain import LinkAction + >>> truth_rule = config.rules['truth'] + >>> discard_chain = config.chains['discard'] + >>> test_chain = Chain('always-discard', u'Testing discards') + >>> link = Link(truth_rule, LinkAction.jump, discard_chain) + >>> test_chain.append_link(link) + >>> mlist.start_chain = u'always-discard' + + >>> inject_message(mlist, msg) + >>> file_pos = fp.tell() + >>> incoming.run() + >>> len(pipeline_queue.files) + 0 + >>> len(incoming_queue.files) + 0 + >>> fp.seek(file_pos) + >>> print 'LOG:', fp.read() + LOG: ... DISCARD: <first> + <BLANKLINE> + + >>> del config.chains['always-discard'] + + +Rejected messages +----------------- + +Similar to discarded messages, a message can be rejected, or bounced back to +the original sender. Again, the built-in chain doesn't support this so we'll +just create a new chain that does. + + >>> reject_chain = config.chains['reject'] + >>> test_chain = Chain('always-reject', u'Testing rejections') + >>> link = Link(truth_rule, LinkAction.jump, reject_chain) + >>> test_chain.append_link(link) + >>> mlist.start_chain = u'always-reject' + +The virgin queue needs to be cleared out due to artifacts from the previous +tests above. + + >>> virgin_queue = config.switchboards['virgin'] + >>> ignore = get_queue_messages('virgin') + + >>> inject_message(mlist, msg) + >>> file_pos = fp.tell() + >>> incoming.run() + >>> len(pipeline_queue.files) + 0 + >>> len(incoming_queue.files) + 0 + + >>> len(virgin_queue.files) + 1 + >>> item = get_queue_messages('virgin')[0] + >>> print item.msg.as_string() + Subject: My first post + From: _xtest-owner@example.com + To: aperson@example.com + ... + <BLANKLINE> + --===============... + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + <BLANKLINE> + [No bounce details are available] + --===============... + Content-Type: message/rfc822 + MIME-Version: 1.0 + <BLANKLINE> + From: aperson@example.com + To: _xtest@example.com + Subject: My first post + Message-ID: <first> + Date: ... + <BLANKLINE> + First post! + <BLANKLINE> + --===============... + + >>> sorted(item.msgdata.items()) + [...('recips', [u'aperson@example.com'])...] + >>> fp.seek(file_pos) + >>> print 'LOG:', fp.read() + LOG: ... REJECT: <first> + <BLANKLINE> + + >>> del config.chains['always-reject'] diff --git a/src/mailman/queue/docs/lmtp.txt b/src/mailman/queue/docs/lmtp.txt new file mode 100644 index 000000000..75e91fd4e --- /dev/null +++ b/src/mailman/queue/docs/lmtp.txt @@ -0,0 +1,103 @@ +LTMP server +=========== + +Mailman can accept messages via LMTP (RFC 2033). Most modern mail servers +support LMTP local delivery, so this is a very portable way to connect Mailman +with your mail server. + +Our LMTP server is fairly simple though; all it does is make sure that the +message is destined for a valid endpoint, e.g. mylist-join@example.com. + +Let's start a testable LMTP queue runner. + + >>> from mailman.testing import helpers + >>> master = helpers.TestableMaster() + >>> master.start('lmtp') + +It also helps to have a nice LMTP client. + + >>> lmtp = helpers.get_lmtp_client() + (220, '... Python LMTP queue runner 1.0') + >>> lmtp.lhlo('remote.example.org') + (250, ...) + + +Posting address +--------------- + +If the mail server tries to send a message to a nonexistent mailing list, it +will get a 550 error. + + >>> lmtp.sendmail( + ... 'anne.person@example.com', + ... ['mylist@example.com'], """\ + ... From: anne.person@example.com + ... To: mylist@example.com + ... Subject: An interesting message + ... Message-ID: <aardvark> + ... + ... This is an interesting message. + ... """) + Traceback (most recent call last): + ... + SMTPDataError: (550, 'Requested action not taken: mailbox unavailable') + +Once the mailing list is created, the posting address is valid. + + >>> from mailman.app.lifecycle import create_list + >>> create_list(u'mylist@example.com') + <mailing list "mylist@example.com" at ...> + >>> commit() + >>> lmtp.sendmail( + ... 'anne.person@example.com', + ... ['mylist@example.com'], """\ + ... From: anne.person@example.com + ... To: mylist@example.com + ... Subject: An interesting message + ... Message-ID: <badger> + ... + ... This is an interesting message. + ... """) + {} + + +Sub-addresses +------------- + +The LMTP server understands each of the list's sub-addreses, such as -join, +-leave, -request and so on. If the message is posted to an invalid +sub-address though, it is rejected. + + >>> lmtp.sendmail( + ... 'anne.person@example.com', + ... ['mylist-bogus@example.com'], """\ + ... From: anne.person@example.com + ... To: mylist-bogus@example.com + ... Subject: Help + ... Message-ID: <cow> + ... + ... Please help me. + ... """) + Traceback (most recent call last): + ... + SMTPDataError: (550, 'Requested action not taken: mailbox unavailable') + +But the message is accepted if posted to a valid sub-address. + + >>> lmtp.sendmail( + ... 'anne.person@example.com', + ... ['mylist-request@example.com'], """\ + ... From: anne.person@example.com + ... To: mylist-request@example.com + ... Subject: Help + ... Message-ID: <dog> + ... + ... Please help me. + ... """) + {} + + +Clean up +-------- + + >>> master.stop() diff --git a/src/mailman/queue/docs/news.txt b/src/mailman/queue/docs/news.txt new file mode 100644 index 000000000..3375b3d54 --- /dev/null +++ b/src/mailman/queue/docs/news.txt @@ -0,0 +1,157 @@ +The news runner +=============== + +The news runner is the queue runner that gateways mailing list messages to an +NNTP newsgroup. One of the most important things this runner does is prepare +the message for Usenet (yes, I know that NNTP is not Usenet, but this runner +was originally written to gate to Usenet, which has its own rules). + + >>> from mailman.queue.news import prepare_message + >>> mlist = config.db.list_manager.create(u'_xtest@example.com') + >>> mlist.linked_newsgroup = u'comp.lang.python' + +Some NNTP servers such as INN reject messages containing a set of prohibited +headers, so one of the things that the news runner does is remove these +prohibited headers. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... NNTP-Posting-Host: news.example.com + ... NNTP-Posting-Date: today + ... X-Trace: blah blah + ... X-Complaints-To: abuse@dom.ain + ... Xref: blah blah + ... Xref: blah blah + ... Date-Received: yesterday + ... Posted: tomorrow + ... Posting-Version: 99.99 + ... Relay-Version: 88.88 + ... Received: blah blah + ... + ... A message + ... """) + >>> msgdata = {} + >>> prepare_message(mlist, msg, msgdata) + >>> msgdata['prepped'] + True + >>> print msg.as_string() + From: aperson@example.com + To: _xtest@example.com + Newsgroups: comp.lang.python + Message-ID: ... + Lines: 1 + <BLANKLINE> + A message + <BLANKLINE> + +Some NNTP servers will reject messages where certain headers are duplicated, +so the news runner must collapse or move these duplicate headers to an +X-Original-* header that the news server doesn't care about. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... To: two@example.com + ... Cc: three@example.com + ... Cc: four@example.com + ... Cc: five@example.com + ... Content-Transfer-Encoding: yes + ... Content-Transfer-Encoding: no + ... Content-Transfer-Encoding: maybe + ... + ... A message + ... """) + >>> msgdata = {} + >>> prepare_message(mlist, msg, msgdata) + >>> msgdata['prepped'] + True + >>> print msg.as_string() + From: aperson@example.com + Newsgroups: comp.lang.python + Message-ID: ... + Lines: 1 + To: _xtest@example.com + X-Original-To: two@example.com + CC: three@example.com + X-Original-CC: four@example.com + X-Original-CC: five@example.com + Content-Transfer-Encoding: yes + X-Original-Content-Transfer-Encoding: no + X-Original-Content-Transfer-Encoding: maybe + <BLANKLINE> + A message + <BLANKLINE> + +But if no headers are duplicated, then the news runner doesn't need to modify +the message. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... Cc: someother@example.com + ... Content-Transfer-Encoding: yes + ... + ... A message + ... """) + >>> msgdata = {} + >>> prepare_message(mlist, msg, msgdata) + >>> msgdata['prepped'] + True + >>> print msg.as_string() + From: aperson@example.com + To: _xtest@example.com + Cc: someother@example.com + Content-Transfer-Encoding: yes + Newsgroups: comp.lang.python + Message-ID: ... + Lines: 1 + <BLANKLINE> + A message + <BLANKLINE> + + +Newsgroup moderation +-------------------- + +When the newsgroup is moderated, an Approved: header with the list's posting +address is added for the benefit of the Usenet system. + + >>> from mailman.interfaces import NewsModeration + >>> mlist.news_moderation = NewsModeration.open_moderated + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... Approved: this gets deleted + ... + ... """) + >>> prepare_message(mlist, msg, {}) + >>> msg['approved'] + u'_xtest@example.com' + + >>> mlist.news_moderation = NewsModeration.moderated + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... Approved: this gets deleted + ... + ... """) + >>> prepare_message(mlist, msg, {}) + >>> msg['approved'] + u'_xtest@example.com' + +But if the newsgroup is not moderated, the Approved: header is not chnaged. + + >>> mlist.news_moderation = NewsModeration.none + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... Approved: this doesn't get deleted + ... + ... """) + >>> prepare_message(mlist, msg, {}) + >>> msg['approved'] + u"this doesn't get deleted" + + +XXX More of the NewsRunner should be tested. diff --git a/src/mailman/queue/docs/outgoing.txt b/src/mailman/queue/docs/outgoing.txt new file mode 100644 index 000000000..6722dee84 --- /dev/null +++ b/src/mailman/queue/docs/outgoing.txt @@ -0,0 +1,75 @@ +Outgoing queue runner +===================== + +The outgoing queue runner is the process that delivers messages to the +directly upstream SMTP server. It is this external SMTP server that performs +final delivery to the intended recipients. + +Messages that appear in the outgoing queue are processed individually through +a 'delivery module', essentially a pluggable interface for determining how the +recipient set will be batched, whether messages will be personalized and +VERP'd, etc. The outgoing runner doesn't itself support retrying but it can +move messages to the 'retry queue' for handling delivery failures. + + >>> from mailman.app.lifecycle import create_list + >>> mlist = create_list(u'test@example.com') + + >>> from mailman.app.membership import add_member + >>> from mailman.interfaces.member import DeliveryMode + >>> add_member(mlist, u'aperson@example.com', u'Anne Person', + ... u'password', DeliveryMode.regular, u'en') + >>> add_member(mlist, u'bperson@example.com', u'Bart Person', + ... u'password', DeliveryMode.regular, u'en') + >>> add_member(mlist, u'cperson@example.com', u'Cris Person', + ... u'password', DeliveryMode.regular, u'en') + +By setting the mailing list to personalize messages, each recipient will get a +unique copy of the message, with certain headers tailored for that recipient. + + >>> from mailman.interfaces.mailinglist import Personalization + >>> mlist.personalize = Personalization.individual + >>> commit() + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: test@example.com + ... Subject: My first post + ... Message-ID: <first> + ... + ... First post! + ... """) + +Normally, messages would show up in the outgoing queue after the message has +been processed by the rule set and pipeline. But we can simulate that here by +injecting a message directly into the outgoing queue. + + >>> msgdata = {} + >>> handler = config.handlers['calculate-recipients'] + >>> handler.process(mlist, msg, msgdata) + + >>> outgoing_queue = config.switchboards['out'] + >>> ignore = outgoing_queue.enqueue( + ... msg, msgdata, + ... verp=True, listname=mlist.fqdn_listname, tolist=True, + ... _plaintext=True) + +Running the outgoing queue runner processes the message, delivering it to the +upstream SMTP, which happens to be our test server. + + >>> from mailman.queue.outgoing import OutgoingRunner + >>> from mailman.testing.helpers import make_testable_runner + >>> outgoing = make_testable_runner(OutgoingRunner, 'out') + >>> outgoing.run() + +Three messages have been delivered to our SMTP server, one for each recipient. + + >>> from operator import itemgetter + >>> messages = sorted(smtpd.messages, key=itemgetter('sender')) + >>> len(messages) + 3 + + >>> for message in messages: + ... print message['sender'] + test-bounces+aperson=example.com@example.com + test-bounces+bperson=example.com@example.com + test-bounces+cperson=example.com@example.com diff --git a/src/mailman/queue/docs/runner.txt b/src/mailman/queue/docs/runner.txt new file mode 100644 index 000000000..d24a8334c --- /dev/null +++ b/src/mailman/queue/docs/runner.txt @@ -0,0 +1,72 @@ +Queue runners +============= + +The queue runners (qrunner) are the processes that move messages around the +Mailman system. Each qrunner is responsible for a slice of the hash space in +a queue directory. It processes all the files in its slice, sleeps a little +while, then wakes up and runs through its queue files again. + + +Basic architecture +------------------ + +The basic architecture of qrunner is implemented in the base class that all +runners inherit from. This base class implements a .run() method that runs +continuously in a loop until the .stop() method is called. + + >>> mlist = config.db.list_manager.create(u'_xtest@example.com') + >>> mlist.preferred_language = u'en' + +Here is a very simple derived qrunner class. Queue runners use a +configuration section in the configuration files to determine run +characteristics, such as the queue directory to use. Here we push a +configuration section for the test runner. + + >>> config.push('test-runner', """ + ... [qrunner.test] + ... max_restarts: 1 + ... """) + + >>> from mailman.queue import Runner + >>> class TestableRunner(Runner): + ... def _dispose(self, mlist, msg, msgdata): + ... self.msg = msg + ... self.msgdata = msgdata + ... return False + ... + ... def _do_periodic(self): + ... self.stop() + ... + ... def _snooze(self, filecnt): + ... return + + >>> runner = TestableRunner('test') + +This qrunner doesn't do much except run once, storing the message and metadata +on instance variables. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... + ... A test message. + ... """) + >>> switchboard = config.switchboards['test'] + >>> filebase = switchboard.enqueue(msg, listname=mlist.fqdn_listname, + ... foo='yes', bar='no') + >>> runner.run() + >>> print runner.msg.as_string() + From: aperson@example.com + To: _xtest@example.com + <BLANKLINE> + A test message. + <BLANKLINE> + >>> dump_msgdata(runner.msgdata) + _parsemsg: False + bar : no + foo : yes + lang : en + listname : _xtest@example.com + version : 3 + +XXX More of the Runner API should be tested. diff --git a/src/mailman/queue/docs/switchboard.txt b/src/mailman/queue/docs/switchboard.txt new file mode 100644 index 000000000..88ab6ea93 --- /dev/null +++ b/src/mailman/queue/docs/switchboard.txt @@ -0,0 +1,182 @@ +The switchboard +=============== + +The switchboard is subsystem that moves messages between queues. Each +instance of a switchboard is responsible for one queue directory. + + >>> msg = message_from_string("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... + ... A test message. + ... """) + +Create a switchboard by giving its queue directory. + + >>> import os + >>> queue_directory = os.path.join(config.QUEUE_DIR, 'test') + >>> from mailman.queue import Switchboard + >>> switchboard = Switchboard(queue_directory) + >>> switchboard.queue_directory == queue_directory + True + +Here's a helper function for ensuring things work correctly. + + >>> def check_qfiles(directory=None): + ... if directory is None: + ... directory = queue_directory + ... files = {} + ... for qfile in os.listdir(directory): + ... root, ext = os.path.splitext(qfile) + ... files[ext] = files.get(ext, 0) + 1 + ... if len(files) == 0: + ... print 'empty' + ... for ext in sorted(files): + ... print '{0}: {1}'.format(ext, files[ext]) + + +Enqueing and dequeing +--------------------- + +The message can be enqueued with metadata specified in the passed in +dictionary. + + >>> filebase = switchboard.enqueue(msg) + >>> check_qfiles() + .pck: 1 + +To read the contents of a queue file, dequeue it. + + >>> msg, msgdata = switchboard.dequeue(filebase) + >>> print msg.as_string() + From: aperson@example.com + To: _xtest@example.com + <BLANKLINE> + A test message. + <BLANKLINE> + >>> dump_msgdata(msgdata) + _parsemsg: False + version : 3 + >>> check_qfiles() + .bak: 1 + +To complete the dequeing process, removing all traces of the message file, +finish it (without preservation). + + >>> switchboard.finish(filebase) + >>> check_qfiles() + empty + +When enqueing a file, you can provide additional metadata keys by using +keyword arguments. + + >>> filebase = switchboard.enqueue(msg, {'foo': 1}, bar=2) + >>> msg, msgdata = switchboard.dequeue(filebase) + >>> switchboard.finish(filebase) + >>> dump_msgdata(msgdata) + _parsemsg: False + bar : 2 + foo : 1 + version : 3 + +Keyword arguments override keys from the metadata dictionary. + + >>> filebase = switchboard.enqueue(msg, {'foo': 1}, foo=2) + >>> msg, msgdata = switchboard.dequeue(filebase) + >>> switchboard.finish(filebase) + >>> dump_msgdata(msgdata) + _parsemsg: False + foo : 2 + version : 3 + + +Iterating over files +-------------------- + +There are two ways to iterate over all the files in a switchboard's queue. +Normally, queue files end in .pck (for 'pickle') and the easiest way to +iterate over just these files is to use the .files attribute. + + >>> filebase_1 = switchboard.enqueue(msg, foo=1) + >>> filebase_2 = switchboard.enqueue(msg, foo=2) + >>> filebase_3 = switchboard.enqueue(msg, foo=3) + >>> filebases = sorted((filebase_1, filebase_2, filebase_3)) + >>> sorted(switchboard.files) == filebases + True + >>> check_qfiles() + .pck: 3 + +You can also use the .get_files() method if you want to iterate over all the +file bases for some other extension. + + >>> for filebase in switchboard.get_files(): + ... msg, msgdata = switchboard.dequeue(filebase) + >>> bakfiles = sorted(switchboard.get_files('.bak')) + >>> bakfiles == filebases + True + >>> check_qfiles() + .bak: 3 + >>> for filebase in switchboard.get_files('.bak'): + ... switchboard.finish(filebase) + >>> check_qfiles() + empty + + +Recovering files +---------------- + +Calling .dequeue() without calling .finish() leaves .bak backup files in +place. These can be recovered when the switchboard is instantiated. + + >>> filebase_1 = switchboard.enqueue(msg, foo=1) + >>> filebase_2 = switchboard.enqueue(msg, foo=2) + >>> filebase_3 = switchboard.enqueue(msg, foo=3) + >>> for filebase in switchboard.files: + ... msg, msgdata = switchboard.dequeue(filebase) + ... # Don't call .finish() + >>> check_qfiles() + .bak: 3 + >>> switchboard_2 = Switchboard(queue_directory, recover=True) + >>> check_qfiles() + .pck: 3 + +The files can be recovered explicitly. + + >>> for filebase in switchboard.files: + ... msg, msgdata = switchboard.dequeue(filebase) + ... # Don't call .finish() + >>> check_qfiles() + .bak: 3 + >>> switchboard.recover_backup_files() + >>> check_qfiles() + .pck: 3 + +But the files will only be recovered at most three times before they are +considered defective. In order to prevent mail bombs and loops, once this +maximum is reached, the files will be preserved in the 'bad' queue. + + >>> for filebase in switchboard.files: + ... msg, msgdata = switchboard.dequeue(filebase) + ... # Don't call .finish() + >>> check_qfiles() + .bak: 3 + >>> switchboard.recover_backup_files() + >>> check_qfiles() + empty + + >>> bad = config.switchboards['bad'] + >>> check_qfiles(bad.queue_directory) + .psv: 3 + +Clean up + + >>> for file in os.listdir(bad.queue_directory): + ... os.remove(os.path.join(bad.queue_directory, file)) + >>> check_qfiles(bad.queue_directory) + empty + + +Queue slices +------------ + +XXX Add tests for queue slices. |