Get rid of one_last_digest. Move this into a separate OneLastDigest table.

Make it explicit to get the last digest (I could imagine an user selecting not
to).  Actually add tests for this.

Start to get rid of 'from storm.locals import *' in favor of more specific
imports.  Start to use Store.of() instead of config.db.store where we can.

Rework (delivery)MemberRosters to use the Member attributes.  This ensures
that the layered lookup happens at the expense of query optimization.

Sundry and various cleanups.

2 files changed, 169 insertions, 44 deletions

diff --git a/src/mailman/queue/digest.py b/src/mailman/queue/digest.py
index a590a9997..30705b8be 100644
--- a/src/mailman/queue/digest.py
+++ b/src/mailman/queue/digest.py
@@ -43,6 +43,7 @@ from mailman.Utils import maketext, oneline, wrap
 from mailman.config import config
 from mailman.core.errors import DiscardMessage
 from mailman.i18n import _
+from mailman.interfaces.member import DeliveryMode, DeliveryStatus
 from mailman.pipeline.decorate import decorate
 from mailman.pipeline.scrubber import process as scrubber
 from mailman.queue import Runner
@@ -332,10 +333,6 @@ class DigestRunner(Runner):
         # digest to ensure that there will be no gaps in the messages they
         # receive.
         digest_members = set(mlist.digest_members.members)
-        for address in mlist.one_last_digest:
-            member = mlist.digest_members.get_member(address)
-            if member:
-                digest_members.add(member)
         for member in digest_members:
             if member.delivery_status <> DeliveryStatus.enabled:
                 continue
@@ -350,6 +347,16 @@ class DigestRunner(Runner):
                 raise AssertionError(
                     'Digest member "{0}" unexpected delivery mode: {1}'.format(
                         email_address, member.delivery_mode))
+        # Add also the folks who are receiving one last digest.
+        for address, delivery_mode in mlist.last_digest_recipients:
+            if delivery_mode == DeliveryMode.plaintext_digests:
+                rfc1153_recipients.add(address.original_address)
+            elif delivery_mode == DeliveryMode.mime_digests:
+                mime_recipients.add(address.original_address)
+            else:
+                raise AssertionError(
+                    'OLD recipient "{0}" unexpected delivery mode: {1}'.format(
+                        address, delivery_mode))
         # Send the digests to the virgin queue for final delivery.
         queue = config.switchboards['virgin']
         queue.enqueue(mime,
@@ -360,6 +367,3 @@ class DigestRunner(Runner):
                       recips=rfc1153_recipients,
                       listname=mlist.fqdn_listname,
                       isdigest=True)
-        # Now that we've delivered the last digest to folks who were waiting
-        # for it, clear that recipient set.
-        mlist.one_last_digest.clear()
diff --git a/src/mailman/queue/docs/digester.txt b/src/mailman/queue/docs/digester.txt
index 57e80317c..2c9c813b2 100644
--- a/src/mailman/queue/docs/digester.txt
+++ b/src/mailman/queue/docs/digester.txt
@@ -8,23 +8,27 @@ This starts by a number of messages being posted to the mailing list.
     >>> mlist.digest_size_threshold = 0.5
     >>> mlist.volume = 1
     >>> mlist.next_digest_number = 1
-    >>> size = 0
 
     >>> from string import Template
     >>> process = config.handlers['to-digest'].process
-    >>> for i in range(1, 5):
-    ...     text = Template("""\
+
+    >>> def fill_digest():
+    ...     size = 0
+    ...     for i in range(1, 5):
+    ...         text = Template("""\
     ... From: aperson@example.com
     ... To: xtest@example.com
     ... Subject: Test message $i
     ...
     ... Here is message $i
     ... """).substitute(i=i)
-    ...     msg = message_from_string(text)
-    ...     process(mlist, msg, {})
-    ...     size += len(text)
-    ...     if size >= mlist.digest_size_threshold * 1024:
-    ...         break
+    ...         msg = message_from_string(text)
+    ...         process(mlist, msg, {})
+    ...         size += len(text)
+    ...         if size >= mlist.digest_size_threshold * 1024:
+    ...             break
+
+    >>> fill_digest()
 
 The queue runner gets kicked off when a marker message gets dropped into the
 digest queue.  The message metadata points to the mailbox file containing the
@@ -77,16 +81,16 @@ delivery.
 
 The MIME digest is a multipart, and the RFC 1153 digest is the other one.
 
-    >>> if messages[0].msg.is_multipart():
-    ...     mime = messages[0].msg
-    ...     rfc1153 = messages[1].msg
-    ... else:
-    ...     mime = messages[1].msg
-    ...     rfc1153 = messages[0].msg
+    >>> def mime_rfc1153(messages):
+    ...     if messages[0].msg.is_multipart():
+    ...         return messages[0], messages[1]
+    ...     return messages[1], messages[0]
+
+    >>> mime, rfc1153 = mime_rfc1153(messages)
 
 The MIME digest has lots of good stuff, all contained in the multipart.
 
-    >>> print mime.as_string()
+    >>> print mime.msg.as_string()
     Content-Type: multipart/mixed; boundary="===============...=="
     MIME-Version: 1.0
     From: test-request@example.com
@@ -103,15 +107,15 @@ The MIME digest has lots of good stuff, all contained in the multipart.
     Content-Description: Test Digest, Vol 1, Issue 1
     <BLANKLINE>
     Send Test mailing list submissions to
-    	test@example.com
+        test@example.com
     <BLANKLINE>
     To subscribe or unsubscribe via the World Wide Web, visit
-    	http://lists.example.com/listinfo/test@example.com
+        http://lists.example.com/listinfo/test@example.com
     or, via email, send a message with subject or body 'help' to
-    	test-request@example.com
+        test-request@example.com
     <BLANKLINE>
     You can reach the person managing the list at
-    	test-owner@example.com
+        test-owner@example.com
     <BLANKLINE>
     When replying, please edit your Subject line so it is more specific
     than "Re: Contents of Test digest..."
@@ -184,7 +188,7 @@ The MIME digest has lots of good stuff, all contained in the multipart.
 
 The RFC 1153 contains the digest in a single plain text message.
 
-    >>> print rfc1153.as_string()
+    >>> print rfc1153.msg.as_string()
     From: test-request@example.com
     Subject: Test Digest, Vol 1, Issue 1
     To: test@example.com
@@ -196,15 +200,15 @@ The RFC 1153 contains the digest in a single plain text message.
     Content-Transfer-Encoding: 7bit
     <BLANKLINE>
     Send Test mailing list submissions to
-    	test@example.com
+        test@example.com
     <BLANKLINE>
     To subscribe or unsubscribe via the World Wide Web, visit
-    	http://lists.example.com/listinfo/test@example.com
+        http://lists.example.com/listinfo/test@example.com
     or, via email, send a message with subject or body 'help' to
-    	test-request@example.com
+        test-request@example.com
     <BLANKLINE>
     You can reach the person managing the list at
-    	test-owner@example.com
+        test-owner@example.com
     <BLANKLINE>
     When replying, please edit your Subject line so it is more specific
     than "Re: Contents of Test digest..."
@@ -324,16 +328,11 @@ queue.
 
 One of which is the MIME digest and the other of which is the RFC 1153 digest.
 
-    >>> if messages[0].msg.is_multipart():
-    ...     mime = messages[0].msg
-    ...     rfc1153 = messages[1].msg
-    ... else:
-    ...     mime = messages[1].msg
-    ...     rfc1153 = messages[0].msg
+    >>> mime, rfc1153 = mime_rfc1153(messages)
 
 You can see that the digests contain a mix of French and Japanese.
 
-    >>> print mime.as_string()
+    >>> print mime.msg.as_string()
     Content-Type: multipart/mixed; boundary="===============...=="
     MIME-Version: 1.0
     From: test-request@example.com
@@ -350,17 +349,17 @@ You can see that the digests contain a mix of French and Japanese.
     Content-Description: Groupe Test, Vol. 1, Parution 2
     <BLANKLINE>
     Envoyez vos messages pour la liste Test =E0
-    	test@example.com
+        test@example.com
     <BLANKLINE>
     Pour vous (d=E9s)abonner par le web, consultez
-    	http://lists.example.com/listinfo/test@example.com
+        http://lists.example.com/listinfo/test@example.com
     <BLANKLINE>
     ou, par courriel, envoyez un message avec =AB=A0help=A0=BB dans le corps ou
     dans le sujet =E0
-    	test-request@example.com
+        test-request@example.com
     <BLANKLINE>
     Vous pouvez contacter l'administrateur de la liste =E0 l'adresse
-    	test-owner@example.com
+        test-owner@example.com
     <BLANKLINE>
     Si vous r=E9pondez, n'oubliez pas de changer l'objet du message afin
     qu'il soit plus sp=E9cifique que =AB=A0Re: Contenu du groupe de Test...=A0=
@@ -402,7 +401,7 @@ You can see that the digests contain a mix of French and Japanese.
 The RFC 1153 digest will be encoded in UTF-8 since it contains a mixture of
 French and Japanese characters.
 
-    >>> print rfc1153.as_string()
+    >>> print rfc1153.msg.as_string()
     From: test-request@example.com
     Subject: Groupe Test, Vol. 1, Parution 2
     To: test@example.com
@@ -420,7 +419,8 @@ The content can be decoded to see the actual digest text.
 
     # We must display the repr of the decoded value because doctests cannot
     # handle the non-ascii characters.
-    >>> [repr(line) for line in rfc1153.get_payload(decode=True).splitlines()]
+    >>> [repr(line)
+    ...  for line in rfc1153.msg.get_payload(decode=True).splitlines()]
     ["'Envoyez vos messages pour la liste Test \\xc3\\xa0'",
      "'\\ttest@example.com'",
      "''",
@@ -462,3 +462,124 @@ The content can be decoded to see the actual digest text.
      "''",
      "'Fin de Groupe Test, Vol. 1, Parution 2'",
      "'**************************************'"]
+
+     >>> config.pop('french')
+
+
+Digest delivery
+---------------
+
+A mailing list's members can choose to receive normal delivery, plain text
+digests, or MIME digests.
+
+    >>> len(get_queue_messages('virgin'))
+    0
+
+    >>> from mailman.interfaces.member import DeliveryMode, MemberRole
+    >>> def subscribe(email, mode):
+    ...     address = config.db.user_manager.create_address(email)
+    ...     member = address.subscribe(mlist, MemberRole.member)
+    ...     member.preferences.delivery_mode = mode
+    ...     return member
+
+Two regular delivery members subscribe to the mailing list.
+
+    >>> member_1 = subscribe(u'uperson@example.com', DeliveryMode.regular)
+    >>> member_2 = subscribe(u'vperson@example.com', DeliveryMode.regular)
+
+Two MIME digest members subscribe to the mailing list.
+
+    >>> member_3 = subscribe(u'wperson@example.com', DeliveryMode.mime_digests)
+    >>> member_4 = subscribe(u'xperson@example.com', DeliveryMode.mime_digests)
+
+One RFC 1153 digest member subscribes to the mailing list.
+
+    >>> member_5 = subscribe(
+    ...     u'yperson@example.com', DeliveryMode.plaintext_digests)
+    >>> member_6 = subscribe(
+    ...     u'zperson@example.com', DeliveryMode.plaintext_digests)
+
+When a digest gets sent, the appropriate recipient list is chosen.
+
+    >>> mlist.preferred_language = u'en'
+    >>> mlist.digest_size_threshold = 0.5
+    >>> fill_digest()
+    >>> runner.run()
+
+The digests are sitting in the virgin queue.  One of them is the MIME digest
+and the other is the RFC 1153 digest.
+
+    >>> messages = get_queue_messages('virgin')
+    >>> len(messages)
+    2
+
+    >>> mime, rfc1153 = mime_rfc1153(messages)
+
+Only wperson and xperson get the MIME digests.
+
+    >>> sorted(mime.msgdata['recips'])
+    [u'wperson@example.com', u'xperson@example.com']
+
+Only yperson and zperson get the RFC 1153 digests.
+
+    >>> sorted(rfc1153.msgdata['recips'])
+    [u'yperson@example.com', u'zperson@example.com']
+
+Now uperson decides that they would like to start receiving digests too.
+
+    >>> member_1.preferences.delivery_mode = DeliveryMode.mime_digests
+    >>> fill_digest()
+    >>> runner.run()
+
+    >>> messages = get_queue_messages('virgin')
+    >>> len(messages)
+    2
+
+    >>> mime, rfc1153 = mime_rfc1153(messages)
+    >>> sorted(mime.msgdata['recips'])
+    [u'uperson@example.com', u'wperson@example.com', u'xperson@example.com']
+
+    >>> sorted(rfc1153.msgdata['recips'])
+    [u'yperson@example.com', u'zperson@example.com']
+
+At this point, both uperson and wperson decide that they'd rather receive
+regular deliveries instead of digests.  uperson would like to get any last
+digest that may be sent so that she doesn't miss anything.  wperson does care
+as much and does not want to receive one last digest.
+
+    >>> mlist.send_one_last_digest_to(
+    ...     member_1.address, member_1.preferences.delivery_mode)
+
+    >>> member_1.preferences.delivery_mode = DeliveryMode.regular
+    >>> member_3.preferences.delivery_mode = DeliveryMode.regular
+
+    >>> fill_digest()
+    >>> runner.run()
+
+    >>> messages = get_queue_messages('virgin')
+    >>> len(messages)
+    2
+
+    >>> mime, rfc1153 = mime_rfc1153(messages)
+    >>> sorted(mime.msgdata['recips'])
+    [u'uperson@example.com', u'xperson@example.com']
+
+    >>> sorted(rfc1153.msgdata['recips'])
+    [u'yperson@example.com', u'zperson@example.com']
+
+Since uperson has received their last digest, they will not get any more of
+them.
+
+    >>> fill_digest()
+    >>> runner.run()
+
+    >>> messages = get_queue_messages('virgin')
+    >>> len(messages)
+    2
+
+    >>> mime, rfc1153 = mime_rfc1153(messages)
+    >>> sorted(mime.msgdata['recips'])
+    [u'xperson@example.com']
+
+    >>> sorted(rfc1153.msgdata['recips'])
+    [u'yperson@example.com', u'zperson@example.com']