diff options
Diffstat (limited to 'src/mailman/queue/docs/digester.txt')
| -rw-r--r-- | src/mailman/queue/docs/digester.txt | 484 |
1 files changed, 484 insertions, 0 deletions
diff --git a/src/mailman/queue/docs/digester.txt b/src/mailman/queue/docs/digester.txt new file mode 100644 index 000000000..487549e90 --- /dev/null +++ b/src/mailman/queue/docs/digester.txt @@ -0,0 +1,484 @@ +Digesting +========= + +Mailman crafts and sends digests by a separate digest queue runner process. +This starts by a number of messages being posted to the mailing list. + + >>> mlist = create_list('test@example.com') + >>> mlist.digest_size_threshold = 0.5 + >>> mlist.volume = 1 + >>> mlist.next_digest_number = 1 + >>> size = 0 + + >>> from string import Template + >>> process = config.handlers['to-digest'].process + >>> for i in range(1, 5): + ... text = Template("""\ + ... From: aperson@example.com + ... To: xtest@example.com + ... Subject: Test message $i + ... + ... Here is message $i + ... """).substitute(i=i) + ... msg = message_from_string(text) + ... process(mlist, msg, {}) + ... size += len(text) + ... if size >= mlist.digest_size_threshold * 1024: + ... break + +The queue runner gets kicked off when a marker message gets dropped into the +digest queue. The message metadata points to the mailbox file containing the +messages to put in the digest. + + >>> digestq = config.switchboards['digest'] + >>> len(digestq.files) + 1 + + >>> from mailman.testing.helpers import get_queue_messages + >>> entry = get_queue_messages('digest')[0] + +The marker message is empty. + + >>> print entry.msg.as_string() + +But the message metadata has a reference to the digest file. + + >>> dump_msgdata(entry.msgdata) + _parsemsg : False + digest_number: 1 + digest_path : .../lists/test@example.com/digest.1.1.mmdf + listname : test@example.com + version : 3 + volume : 1 + + # Put the messages back in the queue for the runner to handle. + >>> filebase = digestq.enqueue(entry.msg, entry.msgdata) + +There are 4 messages in the digest. + + >>> from mailman.utilities.mailbox import Mailbox + >>> sum(1 for item in Mailbox(entry.msgdata['digest_path'])) + 4 + +When the queue runner runs, it processes the digest mailbox, crafting both the +plain text (RFC 1153) digest and the MIME digest. + + >>> from mailman.queue.digest import DigestRunner + >>> from mailman.testing.helpers import make_testable_runner + >>> runner = make_testable_runner(DigestRunner) + >>> runner.run() + +The digest runner places both digests into the virgin queue for final +delivery. + + >>> messages = get_queue_messages('virgin') + >>> len(messages) + 2 + +The MIME digest is a multipart, and the RFC 1153 digest is the other one. + + >>> if messages[0].msg.is_multipart(): + ... mime = messages[0].msg + ... rfc1153 = messages[1].msg + ... else: + ... mime = messages[1].msg + ... rfc1153 = messages[0].msg + +The MIME digest has lots of good stuff, all contained in the multipart. + + >>> print mime.as_string() + Content-Type: multipart/mixed; boundary="===============...==" + MIME-Version: 1.0 + From: test-request@example.com + Subject: Test Digest, Vol 1, Issue 1 + To: test@example.com + Reply-To: test@example.com + Date: ... + Message-ID: ... + <BLANKLINE> + --===============...== + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Description: Test Digest, Vol 1, Issue 1 + <BLANKLINE> + Send Test mailing list submissions to + test@example.com + <BLANKLINE> + To subscribe or unsubscribe via the World Wide Web, visit + http://lists.example.com/listinfo/test@example.com + or, via email, send a message with subject or body 'help' to + test-request@example.com + <BLANKLINE> + You can reach the person managing the list at + test-owner@example.com + <BLANKLINE> + When replying, please edit your Subject line so it is more specific + than "Re: Contents of Test digest..." + <BLANKLINE> + --===============...== + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Description: Today's Topics (4 messages) + <BLANKLINE> + Today's Topics: + <BLANKLINE> + 1. Test message 1 (aperson@example.com) + 2. Test message 2 (aperson@example.com) + 3. Test message 3 (aperson@example.com) + 4. Test message 4 (aperson@example.com) + <BLANKLINE> + --===============...== + Content-Type: message/rfc822 + MIME-Version: 1.0 + <BLANKLINE> + From: aperson@example.com + To: xtest@example.com + Subject: Test message 1 + <BLANKLINE> + Here is message 1 + <BLANKLINE> + --===============...== + Content-Type: message/rfc822 + MIME-Version: 1.0 + <BLANKLINE> + From: aperson@example.com + To: xtest@example.com + Subject: Test message 2 + <BLANKLINE> + Here is message 2 + <BLANKLINE> + --===============...== + Content-Type: message/rfc822 + MIME-Version: 1.0 + <BLANKLINE> + From: aperson@example.com + To: xtest@example.com + Subject: Test message 3 + <BLANKLINE> + Here is message 3 + <BLANKLINE> + --===============...== + Content-Type: message/rfc822 + MIME-Version: 1.0 + <BLANKLINE> + From: aperson@example.com + To: xtest@example.com + Subject: Test message 4 + <BLANKLINE> + Here is message 4 + <BLANKLINE> + --===============...== + Content-Type: text/plain; charset="us-ascii" + MIME-Version: 1.0 + Content-Transfer-Encoding: 7bit + Content-Description: Digest Footer + <BLANKLINE> + _______________________________________________ + Test mailing list + test@example.com + http://lists.example.com/listinfo/test@example.com + <BLANKLINE> + --===============...==-- + +The RFC 1153 contains the digest in a single plain text message. + + >>> print rfc1153.as_string() + From: test-request@example.com + Subject: Test Digest, Vol 1, Issue 1 + To: test@example.com + Reply-To: test@example.com + Date: ... + Message-ID: ... + MIME-Version: 1.0 + Content-Type: text/plain; charset="us-ascii" + Content-Transfer-Encoding: 7bit + <BLANKLINE> + Send Test mailing list submissions to + test@example.com + <BLANKLINE> + To subscribe or unsubscribe via the World Wide Web, visit + http://lists.example.com/listinfo/test@example.com + or, via email, send a message with subject or body 'help' to + test-request@example.com + <BLANKLINE> + You can reach the person managing the list at + test-owner@example.com + <BLANKLINE> + When replying, please edit your Subject line so it is more specific + than "Re: Contents of Test digest..." + <BLANKLINE> + <BLANKLINE> + Today's Topics: + <BLANKLINE> + 1. Test message 1 (aperson@example.com) + 2. Test message 2 (aperson@example.com) + 3. Test message 3 (aperson@example.com) + 4. Test message 4 (aperson@example.com) + <BLANKLINE> + <BLANKLINE> + ---------------------------------------------------------------------- + <BLANKLINE> + From: aperson@example.com + Subject: Test message 1 + To: xtest@example.com + Message-ID: ... + <BLANKLINE> + Here is message 1 + <BLANKLINE> + ------------------------------ + <BLANKLINE> + From: aperson@example.com + Subject: Test message 2 + To: xtest@example.com + Message-ID: ... + <BLANKLINE> + Here is message 2 + <BLANKLINE> + ------------------------------ + <BLANKLINE> + From: aperson@example.com + Subject: Test message 3 + To: xtest@example.com + Message-ID: ... + <BLANKLINE> + Here is message 3 + <BLANKLINE> + ------------------------------ + <BLANKLINE> + From: aperson@example.com + Subject: Test message 4 + To: xtest@example.com + Message-ID: ... + <BLANKLINE> + Here is message 4 + <BLANKLINE> + ------------------------------ + <BLANKLINE> + _______________________________________________ + Test mailing list + test@example.com + http://lists.example.com/listinfo/test@example.com + <BLANKLINE> + <BLANKLINE> + End of Test Digest, Vol 1, Issue 1 + ********************************** + <BLANKLINE> + + +Internationalized digests +------------------------- + +When messages come in with a content-type character set different than that of +the list's preferred language, recipients will get an internationalized +digest. French is not enabled by default site-wide, so enable that now. + + >>> config.languages.enable_language('fr') + + # Simulate the site administrator setting the default server language to + # French in the configuration file. Without this, the English template + # will be found and the masthead won't be translated. + >>> config.push('french', """ + ... [mailman] + ... default_language: fr + ... """) + + >>> mlist.preferred_language = u'fr' + >>> msg = message_from_string("""\ + ... From: aperson@example.org + ... To: test@example.com + ... Subject: =?iso-2022-jp?b?GyRCMGxIVhsoQg==?= + ... MIME-Version: 1.0 + ... Content-Type: text/plain; charset=iso-2022-jp + ... Content-Transfer-Encoding: 7bit + ... + ... \x1b$B0lHV\x1b(B + ... """) + +Set the digest threshold to zero so that the digests will be sent immediately. + + >>> mlist.digest_size_threshold = 0 + >>> process(mlist, msg, {}) + +The marker message is sitting in the digest queue. + + >>> len(digestq.files) + 1 + >>> entry = get_queue_messages('digest')[0] + >>> dump_msgdata(entry.msgdata) + _parsemsg : False + digest_number: 2 + digest_path : .../lists/test@example.com/digest.1.2.mmdf + listname : test@example.com + version : 3 + volume : 1 + +The digest queue runner runs a loop, placing the two digests into the virgin +queue. + + # Put the messages back in the queue for the runner to handle. + >>> filebase = digestq.enqueue(entry.msg, entry.msgdata) + >>> runner.run() + >>> messages = get_queue_messages('virgin') + >>> len(messages) + 2 + +One of which is the MIME digest and the other of which is the RFC 1153 digest. + + >>> if messages[0].msg.is_multipart(): + ... mime = messages[0].msg + ... rfc1153 = messages[1].msg + ... else: + ... mime = messages[1].msg + ... rfc1153 = messages[0].msg + +You can see that the digests contain a mix of French and Japanese. + + >>> print mime.as_string() + Content-Type: multipart/mixed; boundary="===============...==" + MIME-Version: 1.0 + From: test-request@example.com + Subject: Groupe Test, Vol. 1, Parution 2 + To: test@example.com + Reply-To: test@example.com + Date: ... + Message-ID: ... + <BLANKLINE> + --===============...== + Content-Type: text/plain; charset="iso-8859-1" + MIME-Version: 1.0 + Content-Transfer-Encoding: quoted-printable + Content-Description: Groupe Test, Vol. 1, Parution 2 + <BLANKLINE> + Envoyez vos messages pour la liste Test =E0 + test@example.com + <BLANKLINE> + Pour vous (d=E9s)abonner par le web, consultez + http://lists.example.com/listinfo/test@example.com + <BLANKLINE> + ou, par courriel, envoyez un message avec =AB=A0help=A0=BB dans le corps ou + dans le sujet =E0 + test-request@example.com + <BLANKLINE> + Vous pouvez contacter l'administrateur de la liste =E0 l'adresse + test-owner@example.com + <BLANKLINE> + Si vous r=E9pondez, n'oubliez pas de changer l'objet du message afin + qu'il soit plus sp=E9cifique que =AB=A0Re: Contenu du groupe de Test...=A0= + =BB + --===============...== + Content-Type: text/plain; charset="utf-8" + MIME-Version: 1.0 + Content-Transfer-Encoding: base64 + Content-Description: Today's Topics (1 messages) + <BLANKLINE> + VGjDqG1lcyBkdSBqb3VyIDoKCiAgIDEuIOS4gOeVqiAoYXBlcnNvbkBleGFtcGxlLm9yZykK + <BLANKLINE> + --===============...== + Content-Type: message/rfc822 + MIME-Version: 1.0 + <BLANKLINE> + From: aperson@example.org + To: test@example.com + Subject: =?iso-2022-jp?b?GyRCMGxIVhsoQg==?= + MIME-Version: 1.0 + Content-Type: text/plain; charset=iso-2022-jp + Content-Transfer-Encoding: 7bit + <BLANKLINE> + 一番 + <BLANKLINE> + --===============...== + Content-Type: text/plain; charset="iso-8859-1" + MIME-Version: 1.0 + Content-Transfer-Encoding: quoted-printable + Content-Description: =?utf-8?q?Pied_de_page_des_remises_group=C3=A9es?= + <BLANKLINE> + _______________________________________________ + Test mailing list + test@example.com + http://lists.example.com/listinfo/test@example.com + <BLANKLINE> + --===============...==-- + +The RFC 1153 digest will be encoded in UTF-8 since it contains a mixture of +French and Japanese characters. + + >>> print rfc1153.as_string() + From: test-request@example.com + Subject: Groupe Test, Vol. 1, Parution 2 + To: test@example.com + Reply-To: test@example.com + Date: ... + Message-ID: ... + MIME-Version: 1.0 + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: base64 + <BLANKLINE> + RW52b... + Y29tC... + Ly9sa... + dXJya... + cyBvd... + dmV6I... + dGVzd... + ZGUgY... + Zmlxd... + ZXMgZ... + LS0tL... + LS0tL... + c3RAZ... + d2RvZ... + LWpwC... + X19fX... + dEBle... + cGxlL... + KioqK... + <BLANKLINE> + +The content can be decoded to see the actual digest text. + + # We must display the repr of the decoded value because doctests cannot + # handle the non-ascii characters. + >>> [repr(line) for line in rfc1153.get_payload(decode=True).splitlines()] + ["'Envoyez vos messages pour la liste Test \\xc3\\xa0'", + "'\\ttest@example.com'", + "''", + "'Pour vous (d\\xc3\\xa9s)abonner par le web, consultez'", + "'\\thttp://lists.example.com/listinfo/test@example.com'", + "''", + "'ou, par courriel, envoyez un message avec \\xc2\\xab\\xc2\\xa0... + "'dans le sujet \\xc3\\xa0'", + "'\\ttest-request@example.com'", + "''", + '"Vous pouvez contacter l\'administrateur de la liste \\xc3\\xa0 ... + "'\\ttest-owner@example.com'", + "''", + '"Si vous r\\xc3\\xa9pondez, n\'oubliez pas de changer l\'objet du ... + '"qu\'il soit plus sp\\xc3\\xa9cifique que \\xc2\\xab\\xc2\\xa0Re: ... + "''", + "'Th\\xc3\\xa8mes du jour :'", + "''", + "' 1. \\xe4\\xb8\\x80\\xe7\\x95\\xaa (aperson@example.org)'", + "''", + "''", + "'---------------------------------------------------------------------... + "''", + "'From: aperson@example.org'", + "'Subject: \\xe4\\xb8\\x80\\xe7\\x95\\xaa'", + "'To: test@example.com'", + "'Message-ID: ... + "'Content-Type: text/plain; charset=iso-2022-jp'", + "''", + "'\\xe4\\xb8\\x80\\xe7\\x95\\xaa'", + "''", + "'------------------------------'", + "''", + "'_______________________________________________'", + "'Test mailing list'", + "'test@example.com'", + "'http://lists.example.com/listinfo/test@example.com'", + "''", + "''", + "'Fin de Groupe Test, Vol. 1, Parution 2'", + "'**************************************'"] |