diff options
Diffstat (limited to 'src/mailman/pipeline/docs/digests.txt')
| -rw-r--r-- | src/mailman/pipeline/docs/digests.txt | 535 |
1 files changed, 535 insertions, 0 deletions
diff --git a/src/mailman/pipeline/docs/digests.txt b/src/mailman/pipeline/docs/digests.txt new file mode 100644 index 000000000..cb939f7ca --- /dev/null +++ b/src/mailman/pipeline/docs/digests.txt @@ -0,0 +1,535 @@ +Digests +======= + +Digests are a way for a user to receive list traffic in collections instead of +as individual messages when immediately posted. There are several forms of +digests, although only two are currently supported: MIME digests and RFC 1153 +(a.k.a. plain text) digests. + + >>> from mailman.pipeline.to_digest import process + >>> mlist = config.db.list_manager.create(u'_xtest@example.com') + >>> mlist.preferred_language = u'en' + >>> mlist.real_name = u'XTest' + >>> mlist.subject_prefix = u'[_XTest] ' + >>> mlist.one_last_digest = set() + >>> switchboard = config.switchboards['virgin'] + +This is a helper function used to iterate through all the accumulated digest +messages, in the order in which they were posted. This makes it easier to +update the tests when we switch to a different mailbox format. + + >>> from mailman.testing.helpers import digest_mbox + >>> from itertools import count + >>> from string import Template + >>> def makemsg(): + ... for i in count(1): + ... text = Template("""\ + ... From: aperson@example.com + ... To: _xtest@example.com + ... Subject: Test message $i + ... + ... Here is message $i + ... """).substitute(i=i) + ... yield message_from_string(text) + + +Short circuiting +---------------- + +When a message is posted to the mailing list, it is generally added to a +running collection of messages. For now, this is a Unix mailbox file, +although in the future this may end up being converted to a maildir style +mailbox. In any event, there are several factors that would bypass the +storing of posted messages to the mailbox. For example, the mailing list may +not allow digests... + + >>> mlist.digestable = False + >>> msg = makemsg().next() + >>> process(mlist, msg, {}) + >>> sum(1 for mboxmsg in digest_mbox(mlist)) + 0 + >>> switchboard.files + [] + +...or they may allow digests but the message is already a digest. + + >>> mlist.digestable = True + >>> process(mlist, msg, dict(isdigest=True)) + >>> sum(1 for mboxmsg in digest_mbox(mlist)) + 0 + >>> switchboard.files + [] + + +Sending a digest +---------------- + +For messages which are not digests, but which are posted to a digestable +mailing list, the messages will be stored until they reach a criteria +triggering the sending of the digest. If none of those criteria are met, then +the message will just sit in the mailbox for a while. + + >>> mlist.digest_size_threshold = 10000 + >>> process(mlist, msg, {}) + >>> switchboard.files + [] + >>> digest = digest_mbox(mlist) + >>> sum(1 for mboxmsg in digest) + 1 + >>> import os + >>> os.remove(digest._path) + +When the size of the digest mbox reaches the maximum size threshold, a digest +is crafted and sent out. This puts two messages in the virgin queue, an HTML +digest and an RFC 1153 plain text digest. The size threshold is in KB. + + >>> mlist.digest_size_threshold = 1 + >>> mlist.volume = 2 + >>> mlist.next_digest_number = 10 + >>> size = 0 + >>> for msg in makemsg(): + ... process(mlist, msg, {}) + ... size += len(str(msg)) + ... if size > mlist.digest_size_threshold * 1024: + ... break + >>> sum(1 for mboxmsg in digest_mbox(mlist)) + 0 + >>> len(switchboard.files) + 2 + >>> for filebase in switchboard.files: + ... qmsg, qdata = switchboard.dequeue(filebase) + ... switchboard.finish(filebase) + ... if qmsg.is_multipart(): + ... mimemsg = qmsg + ... mimedata = qdata + ... else: + ... rfc1153msg = qmsg + ... rfc1153data = qdata + >>> print mimemsg.as_string() + Content-Type: multipart/mixed; boundary="..." + MIME-Version: 1.0 + From: _xtest-request@example.com + Subject: XTest Digest, Vol 2, Issue 10 + To: _xtest@example.com + Reply-To: _xtest@example.com + Date: ... + Message-ID: ... + <BLANKLINE> + --... + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Description: XTest Digest, Vol 2, Issue 10 + <BLANKLINE> + Send XTest mailing list submissions to + _xtest@example.com + <BLANKLINE> + To subscribe or unsubscribe via the World Wide Web, visit + http://lists.example.com/listinfo/_xtest@example.com + or, via email, send a message with subject or body 'help' to + _xtest-request@example.com + <BLANKLINE> + You can reach the person managing the list at + _xtest-owner@example.com + <BLANKLINE> + When replying, please edit your Subject line so it is more specific + than "Re: Contents of XTest digest..." + <BLANKLINE> + --... + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Description: Today's Topics (8 messages) + <BLANKLINE> + Today's Topics: + <BLANKLINE> + 1. Test message 1 (aperson@example.com) + 2. Test message 2 (aperson@example.com) + 3. Test message 3 (aperson@example.com) + 4. Test message 4 (aperson@example.com) + 5. Test message 5 (aperson@example.com) + 6. Test message 6 (aperson@example.com) + 7. Test message 7 (aperson@example.com) + 8. Test message 8 (aperson@example.com) + <BLANKLINE> + --... + Content-Type: multipart/digest; boundary="..." + MIME-Version: 1.0 + <BLANKLINE> + --... + Content-Type: message/rfc822 + MIME-Version: 1.0 + <BLANKLINE> + From: aperson@example.com + To: _xtest@example.com + Subject: Test message 1 + Message: 1 + <BLANKLINE> + Here is message 1 + <BLANKLINE> + <BLANKLINE> + --... + Content-Type: message/rfc822 + MIME-Version: 1.0 + <BLANKLINE> + From: aperson@example.com + To: _xtest@example.com + Subject: Test message 2 + Message: 2 + <BLANKLINE> + Here is message 2 + <BLANKLINE> + <BLANKLINE> + --... + Content-Type: message/rfc822 + MIME-Version: 1.0 + <BLANKLINE> + From: aperson@example.com + To: _xtest@example.com + Subject: Test message 3 + Message: 3 + <BLANKLINE> + Here is message 3 + <BLANKLINE> + <BLANKLINE> + --... + Content-Type: message/rfc822 + MIME-Version: 1.0 + <BLANKLINE> + From: aperson@example.com + To: _xtest@example.com + Subject: Test message 4 + Message: 4 + <BLANKLINE> + Here is message 4 + <BLANKLINE> + <BLANKLINE> + --... + Content-Type: message/rfc822 + MIME-Version: 1.0 + <BLANKLINE> + From: aperson@example.com + To: _xtest@example.com + Subject: Test message 5 + Message: 5 + <BLANKLINE> + Here is message 5 + <BLANKLINE> + <BLANKLINE> + --... + Content-Type: message/rfc822 + MIME-Version: 1.0 + <BLANKLINE> + From: aperson@example.com + To: _xtest@example.com + Subject: Test message 6 + Message: 6 + <BLANKLINE> + Here is message 6 + <BLANKLINE> + <BLANKLINE> + --... + Content-Type: message/rfc822 + MIME-Version: 1.0 + <BLANKLINE> + From: aperson@example.com + To: _xtest@example.com + Subject: Test message 7 + Message: 7 + <BLANKLINE> + Here is message 7 + <BLANKLINE> + <BLANKLINE> + --... + Content-Type: message/rfc822 + MIME-Version: 1.0 + <BLANKLINE> + From: aperson@example.com + To: _xtest@example.com + Subject: Test message 8 + Message: 8 + <BLANKLINE> + Here is message 8 + <BLANKLINE> + <BLANKLINE> + --... + --... + >>> dump_msgdata(mimedata) + _parsemsg: False + isdigest : True + listname : _xtest@example.com + recips : set([]) + version : 3 + + + >>> print rfc1153msg.as_string() + From: _xtest-request@example.com + Subject: XTest Digest, Vol 2, Issue 10 + To: _xtest@example.com + Reply-To: _xtest@example.com + Date: ... + Message-ID: ... + MIME-Version: 1.0 + Content-Type: text/plain; charset="us-ascii" + Content-Transfer-Encoding: 7bit + <BLANKLINE> + Send XTest mailing list submissions to + _xtest@example.com + <BLANKLINE> + To subscribe or unsubscribe via the World Wide Web, visit + http://lists.example.com/listinfo/_xtest@example.com + or, via email, send a message with subject or body 'help' to + _xtest-request@example.com + <BLANKLINE> + You can reach the person managing the list at + _xtest-owner@example.com + <BLANKLINE> + When replying, please edit your Subject line so it is more specific + than "Re: Contents of XTest digest..." + <BLANKLINE> + <BLANKLINE> + Today's Topics: + <BLANKLINE> + 1. Test message 1 (aperson@example.com) + 2. Test message 2 (aperson@example.com) + 3. Test message 3 (aperson@example.com) + 4. Test message 4 (aperson@example.com) + 5. Test message 5 (aperson@example.com) + 6. Test message 6 (aperson@example.com) + 7. Test message 7 (aperson@example.com) + 8. Test message 8 (aperson@example.com) + <BLANKLINE> + <BLANKLINE> + ---------------------------------------------------------------------- + <BLANKLINE> + Message: 1 + From: aperson@example.com + Subject: Test message 1 + To: _xtest@example.com + Message-ID: ... + <BLANKLINE> + Here is message 1 + <BLANKLINE> + <BLANKLINE> + ------------------------------ + <BLANKLINE> + Message: 2 + From: aperson@example.com + Subject: Test message 2 + To: _xtest@example.com + Message-ID: ... + <BLANKLINE> + Here is message 2 + <BLANKLINE> + <BLANKLINE> + ------------------------------ + <BLANKLINE> + Message: 3 + From: aperson@example.com + Subject: Test message 3 + To: _xtest@example.com + Message-ID: ... + <BLANKLINE> + Here is message 3 + <BLANKLINE> + <BLANKLINE> + ------------------------------ + <BLANKLINE> + Message: 4 + From: aperson@example.com + Subject: Test message 4 + To: _xtest@example.com + Message-ID: ... + <BLANKLINE> + Here is message 4 + <BLANKLINE> + <BLANKLINE> + ------------------------------ + <BLANKLINE> + Message: 5 + From: aperson@example.com + Subject: Test message 5 + To: _xtest@example.com + Message-ID: ... + <BLANKLINE> + Here is message 5 + <BLANKLINE> + <BLANKLINE> + ------------------------------ + <BLANKLINE> + Message: 6 + From: aperson@example.com + Subject: Test message 6 + To: _xtest@example.com + Message-ID: ... + <BLANKLINE> + Here is message 6 + <BLANKLINE> + <BLANKLINE> + ------------------------------ + <BLANKLINE> + Message: 7 + From: aperson@example.com + Subject: Test message 7 + To: _xtest@example.com + Message-ID: ... + <BLANKLINE> + Here is message 7 + <BLANKLINE> + <BLANKLINE> + ------------------------------ + <BLANKLINE> + Message: 8 + From: aperson@example.com + Subject: Test message 8 + To: _xtest@example.com + Message-ID: ... + <BLANKLINE> + Here is message 8 + <BLANKLINE> + <BLANKLINE> + End of XTest Digest, Vol 2, Issue 10 + ************************************ + <BLANKLINE> + >>> dump_msgdata(rfc1153data) + _parsemsg: False + isdigest : True + listname : _xtest@example.com + recips : set([]) + version : 3 + + +Internationalized digests +------------------------- + +When messages come in with a content-type character set different than that of +the list's preferred language, recipients will get an internationalized +digest. French is not enabled by default site-wide, so enable that now. + + >>> config.languages.enable_language('fr') + + # Simulate the site administrator setting the default server language to + # French in the configuration file. Without this, the English template + # will be found and the masthead won't be translated. + >>> config.push('french', """ + ... [mailman] + ... default_language: fr + ... """) + + >>> mlist.preferred_language = u'fr' + >>> msg = message_from_string("""\ + ... From: aperson@example.org + ... To: _xtest@example.com + ... Subject: =?iso-2022-jp?b?GyRCMGxIVhsoQg==?= + ... MIME-Version: 1.0 + ... Content-Type: text/plain; charset=iso-2022-jp + ... Content-Transfer-Encoding: 7bit + ... + ... \x1b$B0lHV\x1b(B + ... """) + +Set the digest threshold to zero so that the digests will be sent immediately. + + >>> mlist.digest_size_threshold = 0 + >>> process(mlist, msg, {}) + >>> sum(1 for mboxmsg in digest_mbox(mlist)) + 0 + >>> len(switchboard.files) + 2 + >>> for filebase in switchboard.files: + ... qmsg, qdata = switchboard.dequeue(filebase) + ... switchboard.finish(filebase) + ... if qmsg.is_multipart(): + ... mimemsg = qmsg + ... mimedata = qdata + ... else: + ... rfc1153msg = qmsg + ... rfc1153data = qdata + >>> print mimemsg.as_string() + Content-Type: multipart/mixed; boundary="..." + MIME-Version: 1.0 + From: _xtest-request@example.com + Subject: Groupe XTest, Vol. 2, Parution 11 + To: _xtest@example.com + Reply-To: _xtest@example.com + Date: ... + Message-ID: ... + <BLANKLINE> + --... + Content-Type: text/plain; charset="iso-8859-1" + MIME-Version: 1.0 + Content-Transfer-Encoding: quoted-printable + Content-Description: Groupe XTest, Vol. 2, Parution 11 + <BLANKLINE> + Envoyez vos messages pour la liste XTest =E0 + _xtest@example.com + <BLANKLINE> + Pour vous (d=E9s)abonner par le web, consultez + http://lists.example.com/listinfo/_xtest@example.com + <BLANKLINE> + ou, par courriel, envoyez un message avec =AB=A0help=A0=BB dans le corps ou + dans le sujet =E0 + _xtest-request@example.com + <BLANKLINE> + Vous pouvez contacter l'administrateur de la liste =E0 l'adresse + _xtest-owner@example.com + <BLANKLINE> + Si vous r=E9pondez, n'oubliez pas de changer l'objet du message afin + qu'il soit plus sp=E9cifique que =AB=A0Re: Contenu du groupe de XTest...=A0= + =BB + <BLANKLINE> + --... + Content-Type: text/plain; charset="utf-8" + MIME-Version: 1.0 + Content-Transfer-Encoding: base64 + Content-Description: Today's Topics (1 messages) + <BLANKLINE> + VGjDqG1lcyBkdSBqb3VyIDoKCiAgIDEuIOS4gOeVqiAoYXBlcnNvbkBleGFtcGxlLm9yZykK + <BLANKLINE> + --... + Content-Type: multipart/digest; boundary="..." + MIME-Version: 1.0 + <BLANKLINE> + --... + Content-Type: message/rfc822 + MIME-Version: 1.0 + <BLANKLINE> + Content-Transfer-Encoding: 7bit + From: aperson@example.org + MIME-Version: 1.0 + To: _xtest@example.com + Content-Type: text/plain; charset=iso-2022-jp + Subject: =?iso-2022-jp?b?GyRCMGxIVhsoQg==?= + Message: 1 + <BLANKLINE> + 一番 + <BLANKLINE> + <BLANKLINE> + --... + --... + >>> dump_msgdata(mimedata) + _parsemsg: False + isdigest : True + listname : _xtest@example.com + recips : set([]) + version : 3 + + >>> print rfc1153msg.as_string() + From: _xtest-request@example.com + Subject: Groupe XTest, Vol. 2, Parution 11 + To: _xtest@example.com + Reply-To: _xtest@example.com + Date: ... + Message-ID: ... + MIME-Version: 1.0 + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: base64 + <BLANKLINE> + ... + <BLANKLINE> + >>> dump_msgdata(rfc1153data) + _parsemsg: False + isdigest : True + listname : _xtest@example.com + recips : set([]) + version : 3 |