Convert the tests for the CalcRecips handler to doc tests. There are

some XXX's in the doc test because digest recipients aren't tested
(though those may go in a different doctest), and neither are urgent
messages.  This latter is for the same reason that the Approved
handler is not yet tested; which password do you use in that header?

The CalcRecips tests would also seem the natural place to test the
receive_list_copy preference, but that actually gets processed in the
AvoidDuplicates handler, so it isn't tested here.

Add delivery_status (of type enum DeliveryStatus) to preferences.  I'm
not entirely sure that's the right place for it, but it lets me finish
converting the test for now.

Expose the rest of the preferences through the IMember interface.

10 files changed, 245 insertions, 130 deletions

diff --git a/Mailman/Handlers/CalcRecips.py b/Mailman/Handlers/CalcRecips.py
index 5db860a79..29135999c 100644
--- a/Mailman/Handlers/CalcRecips.py
+++ b/Mailman/Handlers/CalcRecips.py
@@ -26,8 +26,8 @@ SendmailDeliver and BulkDeliver modules.
 from Mailman import Errors
 from Mailman import Message
 from Mailman import Utils
-from Mailman.MemberAdaptor import ENABLED
 from Mailman.configuration import config
+from Mailman.constants import DeliveryStatus
 from Mailman.i18n import _
 
 
@@ -35,16 +35,14 @@ from Mailman.i18n import _
 def process(mlist, msg, msgdata):
     # Short circuit if we've already calculated the recipients list,
     # regardless of whether the list is empty or not.
-    if msgdata.has_key('recips'):
+    if 'recips' in msgdata:
         return
     # Should the original sender should be included in the recipients list?
-    include_sender = 1
+    include_sender = True
     sender = msg.get_sender()
-    try:
-        if mlist.getMemberOption(sender, config.DontReceiveOwnPosts):
-            include_sender = 0
-    except Errors.NotAMemberError:
-        pass
+    member = mlist.members.get_member(sender)
+    if member and not member.receive_own_postings:
+        include_sender = False
     # Support for urgent messages, which bypasses digests and disabled
     # delivery and forces an immediate delivery to all members Right Now.  We
     # are specifically /not/ allowing the site admins password to work here
@@ -71,18 +69,12 @@ delivery.  The original message as received by Mailman is attached.
 """)
             raise Errors.RejectMessage, Utils.wrap(text)
     # Calculate the regular recipients of the message
-    recips = [mlist.getMemberCPAddress(m)
-              for m in mlist.getRegularMemberKeys()
-              if mlist.getDeliveryStatus(m) == ENABLED]
+    recips = set(member.address.address
+                 for member in mlist.regular_members.members
+                 if member.delivery_status == DeliveryStatus.enabled)
     # Remove the sender if they don't want to receive their own posts
-    if not include_sender:
-        try:
-            recips.remove(mlist.getMemberCPAddress(sender))
-        except (Errors.NotAMemberError, ValueError):
-            # Sender does not want to get copies of their own messages (not
-            # metoo), but delivery to their address is disabled (nomail).  Or
-            # the sender is not a member of the mailing list.
-            pass
+    if not include_sender and member.address.address in recips:
+        recips.remove(member.address.address)
     # Handle topic classifications
     do_topic_filters(mlist, msg, msgdata, recips)
     # Bookkeeping
diff --git a/Mailman/constants.py b/Mailman/constants.py
index 7b3876a2a..0933d7713 100644
--- a/Mailman/constants.py
+++ b/Mailman/constants.py
@@ -64,3 +64,4 @@ class SystemDefaultPreferences(object):
     receive_list_copy = True
     receive_own_postings = True
     delivery_mode = DeliveryMode.regular
+    delivery_status = DeliveryStatus.enabled
diff --git a/Mailman/database/model/member.py b/Mailman/database/model/member.py
index efb72ee11..1dc942323 100644
--- a/Mailman/database/model/member.py
+++ b/Mailman/database/model/member.py
@@ -58,10 +58,6 @@ class Member(Entity):
         return getattr(SystemDefaultPreferences, preference)
 
     @property
-    def delivery_mode(self):
-        return self._lookup('delivery_mode')
-
-    @property
     def acknowledge_posts(self):
         return self._lookup('acknowledge_posts')
 
@@ -69,11 +65,27 @@ class Member(Entity):
     def preferred_language(self):
         return self._lookup('preferred_language')
 
-    def unsubscribe(self):
-        self.preferences.delete()
-        self.delete()
+    @property
+    def receive_list_copy(self):
+        return self._lookup('receive_list_copy')
+
+    @property
+    def receive_own_postings(self):
+        return self._lookup('receive_own_postings')
+
+    @property
+    def delivery_mode(self):
+        return self._lookup('delivery_mode')
+
+    @property
+    def delivery_status(self):
+        return self._lookup('delivery_status')
 
     @property
     def options_url(self):
         # XXX Um, this is definitely wrong
         return 'http://example.com/' + self.address.address
+
+    def unsubscribe(self):
+        self.preferences.delete()
+        self.delete()
diff --git a/Mailman/database/model/preferences.py b/Mailman/database/model/preferences.py
index 33511f54b..07d4d84e2 100644
--- a/Mailman/database/model/preferences.py
+++ b/Mailman/database/model/preferences.py
@@ -37,6 +37,7 @@ class Preferences(Entity):
     has_field('receive_list_copy',      Boolean)
     has_field('receive_own_postings',   Boolean)
     has_field('delivery_mode',          EnumType)
+    has_field('delivery_status',        EnumType)
     # Options
     using_options(shortnames=True)
 
diff --git a/Mailman/docs/calc-recips.txt b/Mailman/docs/calc-recips.txt
new file mode 100644
index 000000000..9646778d6
--- /dev/null
+++ b/Mailman/docs/calc-recips.txt
@@ -0,0 +1,130 @@
+Calculating recipients
+======================
+
+Every message that makes it through to the list membership gets sent to a set
+of recipient addresses.  These addresses are calculated by one of the handler
+modules and depends on a host of factors.
+
+    >>> from email import message_from_string
+    >>> from Mailman.Message import Message
+    >>> from Mailman.Handlers.CalcRecips import process
+    >>> from Mailman.configuration import config
+    >>> from Mailman.database import flush
+    >>> mlist = config.list_manager.create('_xtest@example.com')
+    >>> flush()
+
+
+Recipients are calculate from the list members, so add a bunch of members to
+start out with.  First, create a bunch of addresses...
+
+    >>> address_a = config.user_manager.create_address('aperson@example.com')
+    >>> address_b = config.user_manager.create_address('bperson@example.com')
+    >>> address_c = config.user_manager.create_address('cperson@example.com')
+    >>> address_d = config.user_manager.create_address('dperson@example.com')
+    >>> address_e = config.user_manager.create_address('eperson@example.com')
+    >>> address_f = config.user_manager.create_address('fperson@example.com')
+
+...then subscribe these addresses to the mailing list as members...
+
+    >>> from Mailman.constants import MemberRole
+    >>> member_a = address_a.subscribe(mlist, MemberRole.member)
+    >>> member_b = address_b.subscribe(mlist, MemberRole.member)
+    >>> member_c = address_c.subscribe(mlist, MemberRole.member)
+    >>> member_d = address_d.subscribe(mlist, MemberRole.member)
+    >>> member_e = address_e.subscribe(mlist, MemberRole.member)
+    >>> member_f = address_f.subscribe(mlist, MemberRole.member)
+
+...then make some of the members digest members.
+
+    >>> from Mailman.constants import DeliveryMode
+    >>> member_d.preferences.delivery_mode = DeliveryMode.plaintext_digests
+    >>> member_e.preferences.delivery_mode = DeliveryMode.mime_digests
+    >>> member_f.preferences.delivery_mode = DeliveryMode.summary_digests
+    >>> flush()
+
+
+Short-circuiting
+----------------
+
+Sometimes, the list of recipients already exists in the message metadata.
+This can happen for example, when a message was previously delivered to some
+but not all of the recipients.
+
+    >>> msg = message_from_string("""\
+    ... From: Xavier Person <xperson@example.com>
+    ...
+    ... Something of great import.
+    ... """, Message)
+    >>> recips = set(('qperson@example.com', 'zperson@example.com'))
+    >>> msgdata = dict(recips=recips)
+    >>> process(mlist, msg, msgdata)
+    >>> sorted(msgdata['recips'])
+    ['qperson@example.com', 'zperson@example.com']
+
+
+Regular delivery recipients
+---------------------------
+
+Regular delivery recipients are those people who get messages from the list as
+soon as they are posted.  In other words, these folks are not digest members.
+
+    >>> msgdata = {}
+    >>> process(mlist, msg, msgdata)
+    >>> sorted(msgdata['recips'])
+    ['aperson@example.com', 'bperson@example.com', 'cperson@example.com']
+
+Members can elect not to receive a list copy of their own postings.
+
+    >>> member_c.preferences.receive_own_postings = False
+    >>> flush()
+    >>> msg = message_from_string("""\
+    ... From: Claire Person <cperson@example.com>
+    ...
+    ... Something of great import.
+    ... """, Message)
+    >>> msgdata = {}
+    >>> process(mlist, msg, msgdata)
+    >>> sorted(msgdata['recips'])
+    ['aperson@example.com', 'bperson@example.com']
+
+Members can also elect not to receive a list copy of any message on which they
+are explicitly named as a recipient.  However, see the AvoidDuplicates handler
+for details.
+
+
+Digest recipients
+-----------------
+
+XXX Test various digest deliveries.
+
+
+Urgent messages
+---------------
+
+XXX Test various urgent deliveries:
+    * test_urgent_moderator()
+    * test_urgent_admin()
+    * test_urgent_reject()
+
+
+Clean up
+--------
+
+    >>> for member in mlist.members.members:
+    ...     member.unsubscribe()
+    >>> flush()
+    >>> list(mlist.members.members)
+    []
+    >>> for user in config.user_manager.users:
+    ...     config.user_manager.delete_user(user)
+    >>> for address in config.user_manager.addresses:
+    ...     config.user_manager.delete_address(address)
+    >>> for mlist in config.list_manager.mailing_lists:
+    ...     config.list_manager.delete(mlist)
+    >>> flush()
+    >>> list(config.user_manager.users)
+    []
+    >>> list(config.user_manager.addresses)
+    []
+    >>> list(config.list_manager.mailing_lists)
+    []
diff --git a/Mailman/interfaces/member.py b/Mailman/interfaces/member.py
index 4bb03f41d..9921f7dab 100644
--- a/Mailman/interfaces/member.py
+++ b/Mailman/interfaces/member.py
@@ -41,7 +41,7 @@ class IMember(Interface):
         """Unsubscribe (and delete) this member from the mailing list."""
 
     acknowledge_posts = Attribute(
-        """This is the actual acknowledgment setting for this member.
+        """Send an acknowledgment for every posting?
 
         Unlike going through the preferences, this attribute return the
         preference value based on the following lookup order:
@@ -52,8 +52,8 @@ class IMember(Interface):
         4. System default
         """)
 
-    delivery_mode = Attribute(
-        """This is the actual delivery mode for this member.
+    preferred_language = Attribute(
+        """The preferred language for interacting with a mailing list.
 
         Unlike going through the preferences, this attribute return the
         preference value based on the following lookup order:
@@ -64,8 +64,32 @@ class IMember(Interface):
         4. System default
         """)
 
-    preferred_language = Attribute(
-        """This is the actual preferred language for this member.
+    receive_list_copy = Attribute(
+        """Should an explicit recipient receive a list copy?
+
+        Unlike going through the preferences, this attribute return the
+        preference value based on the following lookup order:
+
+        1. The member
+        2. The address
+        3. The user
+        4. System default
+        """)
+
+    receive_own_postings = Attribute(
+        """Should the poster get a list copy of their own messages?
+
+        Unlike going through the preferences, this attribute return the
+        preference value based on the following lookup order:
+
+        1. The member
+        2. The address
+        3. The user
+        4. System default
+        """)
+
+    delivery_mode = Attribute(
+        """The preferred delivery mode.
 
         Unlike going through the preferences, this attribute return the
         preference value based on the following lookup order:
@@ -76,6 +100,19 @@ class IMember(Interface):
         4. System default
         """)
 
+    delivery_status = Attribute(
+        """The delivery status.
+
+        Unlike going through the preferences, this attribute return the
+        preference value based on the following lookup order:
+
+        1. The member
+        2. The address
+        3. The user
+        4. System default
+
+        XXX I'm not sure this is the right place to put this.""")
+
     options_url = Attribute(
         """Return the url for the given member's option page.
 
diff --git a/Mailman/interfaces/preferences.py b/Mailman/interfaces/preferences.py
index 17cfebae6..0809874e2 100644
--- a/Mailman/interfaces/preferences.py
+++ b/Mailman/interfaces/preferences.py
@@ -59,3 +59,11 @@ class IPreferences(Interface):
 
         This is an enum constant of the type DeliveryMode.  It may also be
         None which means that no preference is specified.""")
+
+    delivery_status = Attribute(
+        """The delivery status.
+
+        This is an enum constant of type DeliveryStatus.  It may also be None
+        which means that no preference is specified.
+
+        XXX I'm not sure this is the right place to put this.""")
diff --git a/Mailman/testing/test_after_delivery.py b/Mailman/testing/test_after_delivery.py
index 0abad5d6e..ea4801b39 100644
--- a/Mailman/testing/test_after_delivery.py
+++ b/Mailman/testing/test_after_delivery.py
@@ -15,7 +15,7 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
 # USA.
 
-"""Doctest harness for testing booking done after message delivery."""
+"""Doctest harness for testing bookkeeping done after message delivery."""
 
 import doctest
 import unittest
diff --git a/Mailman/testing/test_calc_recips.py b/Mailman/testing/test_calc_recips.py
new file mode 100644
index 000000000..7e876d428
--- /dev/null
+++ b/Mailman/testing/test_calc_recips.py
@@ -0,0 +1,32 @@
+# Copyright (C) 2007 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+
+"""Doctest harness for testing the recipient calculation handler."""
+
+import doctest
+import unittest
+
+options = (doctest.ELLIPSIS
+           | doctest.NORMALIZE_WHITESPACE
+           | doctest.REPORT_NDIFF)
+
+
+def test_suite():
+    suite = unittest.TestSuite()
+    suite.addTest(doctest.DocFileSuite('../docs/calc-recips.txt',
+                                       optionflags=options))
+    return suite
diff --git a/Mailman/testing/test_handlers.py b/Mailman/testing/test_handlers.py
index 9c236584f..bbc7f5ba8 100644
--- a/Mailman/testing/test_handlers.py
+++ b/Mailman/testing/test_handlers.py
@@ -39,7 +39,6 @@ from Mailman.testing.base import TestBase
 from Mailman.Handlers import Acknowledge
 from Mailman.Handlers import AfterDelivery
 from Mailman.Handlers import Approve
-from Mailman.Handlers import CalcRecips
 from Mailman.Handlers import Cleanse
 from Mailman.Handlers import CookHeaders
 from Mailman.Handlers import FileRecips
@@ -138,102 +137,6 @@ X-BeenThere: %s
 
 
 
-class TestCalcRecips(TestBase):
-    def setUp(self):
-        TestBase.setUp(self)
-        # Add a bunch of regular members
-        mlist = self._mlist
-        mlist.addNewMember('aperson@example.org')
-        mlist.addNewMember('bperson@example.com')
-        mlist.addNewMember('cperson@example.com')
-        # And a bunch of digest members
-        mlist.addNewMember('dperson@example.com', digest=1)
-        mlist.addNewMember('eperson@example.com', digest=1)
-        mlist.addNewMember('fperson@example.com', digest=1)
-
-    def test_short_circuit(self):
-        msgdata = {'recips': 1}
-        rtn = CalcRecips.process(self._mlist, None, msgdata)
-        # Not really a great test, but there's little else to assert
-        self.assertEqual(rtn, None)
-
-    def test_simple_path(self):
-        msgdata = {}
-        msg = email.message_from_string("""\
-From: dperson@example.com
-
-""", Message.Message)
-        CalcRecips.process(self._mlist, msg, msgdata)
-        self.failUnless(msgdata.has_key('recips'))
-        recips = msgdata['recips']
-        recips.sort()
-        self.assertEqual(recips, ['aperson@example.org', 'bperson@example.com',
-                                  'cperson@example.com'])
-
-    def test_exclude_sender(self):
-        msgdata = {}
-        msg = email.message_from_string("""\
-From: cperson@example.com
-
-""", Message.Message)
-        self._mlist.setMemberOption('cperson@example.com',
-                                    config.DontReceiveOwnPosts, 1)
-        CalcRecips.process(self._mlist, msg, msgdata)
-        self.failUnless(msgdata.has_key('recips'))
-        recips = msgdata['recips']
-        recips.sort()
-        self.assertEqual(recips, ['aperson@example.org', 'bperson@example.com'])
-
-    def test_urgent_moderator(self):
-        self._mlist.mod_password = password('xxXXxx')
-        msgdata = {}
-        msg = email.message_from_string("""\
-From: dperson@example.com
-Urgent: xxXXxx
-
-""", Message.Message)
-        CalcRecips.process(self._mlist, msg, msgdata)
-        self.failUnless(msgdata.has_key('recips'))
-        recips = msgdata['recips']
-        recips.sort()
-        self.assertEqual(recips, ['aperson@example.org', 'bperson@example.com',
-                                  'cperson@example.com', 'dperson@example.com',
-                                  'eperson@example.com', 'fperson@example.com'])
-
-    def test_urgent_admin(self):
-        self._mlist.mod_password = password('yyYYyy')
-        self._mlist.password = password('xxXXxx')
-        msgdata = {}
-        msg = email.message_from_string("""\
-From: dperson@example.com
-Urgent: xxXXxx
-
-""", Message.Message)
-        CalcRecips.process(self._mlist, msg, msgdata)
-        self.failUnless(msgdata.has_key('recips'))
-        recips = msgdata['recips']
-        recips.sort()
-        self.assertEqual(recips, ['aperson@example.org', 'bperson@example.com',
-                                  'cperson@example.com', 'dperson@example.com',
-                                  'eperson@example.com', 'fperson@example.com'])
-
-    def test_urgent_reject(self):
-        self._mlist.mod_password = password('yyYYyy')
-        self._mlist.password = password('xxXXxx')
-        msgdata = {}
-        msg = email.message_from_string("""\
-From: dperson@example.com
-Urgent: zzZZzz
-
-""", Message.Message)
-        self.assertRaises(Errors.RejectMessage,
-                          CalcRecips.process,
-                          self._mlist, msg, msgdata)
-
-    # BAW: must test the do_topic_filters() path...
-
-
-
 class TestCleanse(TestBase):
     def setUp(self):
         TestBase.setUp(self)
@@ -1560,7 +1463,6 @@ Mailman rocks!
 def test_suite():
     suite = unittest.TestSuite()
     suite.addTest(unittest.makeSuite(TestApprove))
-    suite.addTest(unittest.makeSuite(TestCalcRecips))
     suite.addTest(unittest.makeSuite(TestCleanse))
     suite.addTest(unittest.makeSuite(TestCookHeaders))
     suite.addTest(unittest.makeSuite(TestFileRecips))