3 files changed, 250 insertions, 32 deletions

diff --git a/Mailman/Handlers/AvoidDuplicates.py b/Mailman/Handlers/AvoidDuplicates.py
index 1ef05628f..a652906a9 100644
--- a/Mailman/Handlers/AvoidDuplicates.py
+++ b/Mailman/Handlers/AvoidDuplicates.py
@@ -31,61 +31,63 @@ COMMASPACE = ', '
 
 
 def process(mlist, msg, msgdata):
-    recips = msgdata['recips']
+    recips = msgdata.get('recips')
     # Short circuit
     if not recips:
         return
-    # Seed this set with addresses we don't care about dup avoiding
-    explicit_recips = {}
-    listaddrs = [mlist.GetListEmail(), mlist.GetBouncesEmail(),
-                 mlist.GetOwnerEmail(), mlist.GetRequestEmail()]
-    for addr in listaddrs:
-        explicit_recips[addr] = True
+    # Seed this set with addresses we don't care about dup avoiding.
+    listaddrs = set((mlist.posting_address,
+                     mlist.bounces_address,
+                     mlist.owner_address,
+                     mlist.request_address))
+    explicit_recips = listaddrs.copy()
     # Figure out the set of explicit recipients
-    ccaddrs = {}
+    cc_addresses = {}
     for header in ('to', 'cc', 'resent-to', 'resent-cc'):
         addrs = getaddresses(msg.get_all(header, []))
+        header_addresses = dict((addr, formataddr((name, addr)))
+                                for name, addr in addrs
+                                if addr)
         if header == 'cc':
-            for name, addr in addrs:
-                ccaddrs[addr] = name, addr
-        for name, addr in addrs:
-            if not addr:
-                continue
-            # Ignore the list addresses for purposes of dup avoidance
-            explicit_recips[addr] = True
+            # Yes, it's possible that an address is mentioned in multiple CC
+            # headers using different names.  In that case, the last real name
+            # will win, but that doesn't seem like such a big deal.  Besides,
+            # how else would you chose?
+            cc_addresses.update(header_addresses)
+        # Ignore the list addresses for purposes of dup avoidance.
+        explicit_recips |= set(header_addresses)
     # Now strip out the list addresses
-    for addr in listaddrs:
-        del explicit_recips[addr]
+    explicit_recips -= listaddrs
     if not explicit_recips:
         # No one was explicitly addressed, so we can't do any dup collapsing
         return
-    newrecips = []
+    newrecips = set()
     for r in recips:
         # If this recipient is explicitly addressed...
-        if explicit_recips.has_key(r):
+        if r in explicit_recips:
             send_duplicate = True
             # If the member wants to receive duplicates, or if the recipient
-            # is not a member at all, just flag the X-Mailman-Duplicate: yes
+            # is not a member at all, they will get a copy.
             # header.
-            if mlist.isMember(r) and \
-                   mlist.getMemberOption(r, config.DontReceiveDuplicates):
+            member = mlist.members.get_member(r)
+            if member and not member.receive_list_copy:
                 send_duplicate = False
             # We'll send a duplicate unless the user doesn't wish it.  If
             # personalization is enabled, the add-dupe-header flag will add a
             # X-Mailman-Duplicate: yes header for this user's message.
             if send_duplicate:
-                msgdata.setdefault('add-dup-header', {})[r] = True
-                newrecips.append(r)
-            elif ccaddrs.has_key(r):
-                del ccaddrs[r]
+                msgdata.setdefault('add-dup-header', set()).add(r)
+                newrecips.add(r)
+            elif r in cc_addresses:
+                del cc_addresses[r]
         else:
             # Otherwise, this is the first time they've been in the recips
             # list.  Add them to the newrecips list and flag them as having
             # received this message.
-            newrecips.append(r)
-    # Set the new list of recipients
-    msgdata['recips'] = newrecips
+            newrecips.add(r)
+    # Set the new list of recipients.  XXX recips should always be a set.
+    msgdata['recips'] = list(newrecips)
     # RFC 2822 specifies zero or one CC header
-    del msg['cc']
-    if ccaddrs:
-        msg['Cc'] = COMMASPACE.join([formataddr(i) for i in ccaddrs.values()])
+    if cc_addresses:
+        del msg['cc']
+        msg['CC'] = COMMASPACE.join(cc_addresses.values())
diff --git a/Mailman/docs/avoid-duplicates.txt b/Mailman/docs/avoid-duplicates.txt
new file mode 100644
index 000000000..6a58382fd
--- /dev/null
+++ b/Mailman/docs/avoid-duplicates.txt
@@ -0,0 +1,183 @@
+Avoid duplicates
+================
+
+The AvoidDuplicates handler module implements several strategies to try to
+reduce the reception of duplicate messages.  It does this by removing certain
+recipients from the list of recipients that earlier handler modules
+(e.g. CalcRecips) calculates.
+
+    >>> from email import message_from_string
+    >>> from Mailman.Message import Message
+    >>> from Mailman.Handlers.AvoidDuplicates import process
+    >>> from Mailman.configuration import config
+    >>> from Mailman.database import flush
+    >>> mlist = config.list_manager.create('_xtest@example.com')
+    >>> flush()
+
+Create some members we're going to use.
+
+    >>> from Mailman.constants import MemberRole
+    >>> address_a = config.user_manager.create_address('aperson@example.com')
+    >>> address_b = config.user_manager.create_address('bperson@example.com')
+    >>> member_a = address_a.subscribe(mlist, MemberRole.member)
+    >>> member_b = address_b.subscribe(mlist, MemberRole.member)
+    >>> flush()
+    >>> # This is the message metadata dictionary as it would be produced by
+    >>> # the CalcRecips handler.
+    >>> recips = dict(recips=['aperson@example.com', 'bperson@example.com'])
+
+
+Short circuiting
+----------------
+
+The module short-circuits if there are no recipients.
+
+    >>> msg = message_from_string("""\
+    ... From: aperson@example.com
+    ... Subject: A message of great import
+    ...
+    ... Something
+    ... """, Message)
+    >>> msgdata = {}
+    >>> process(mlist, msg, msgdata)
+    >>> msgdata
+    {}
+    >>> print msg.as_string()
+    From: aperson@example.com
+    Subject: A message of great import
+    <BLANKLINE>
+    Something
+    <BLANKLINE>
+
+
+Suppressing the list copy
+-------------------------
+
+Members can elect not to receive a list copy of any message on which they are
+explicitly named as a recipient.  This is done by setting their
+receive_list_copy preference to False.  However, if they aren't mentioned in
+one of the recipient headers (i.e. To, CC, Resent-To, or Resent-CC), then they
+will get a list copy.
+
+    >>> member_a.preferences.receive_list_copy = False
+    >>> flush()
+    >>> msg = message_from_string("""\
+    ... From: Claire Person <cperson@example.com>
+    ...
+    ... Something of great import.
+    ... """, Message)
+    >>> msgdata = recips.copy()
+    >>> process(mlist, msg, msgdata)
+    >>> sorted(msgdata['recips'])
+    ['aperson@example.com', 'bperson@example.com']
+    >>> print msg.as_string()
+    From: Claire Person <cperson@example.com>
+    <BLANKLINE>
+    Something of great import.
+    <BLANKLINE>
+
+If they're mentioned on the CC line, they won't get a list copy.
+
+    >>> msg = message_from_string("""\
+    ... From: Claire Person <cperson@example.com>
+    ... CC: aperson@example.com
+    ...
+    ... Something of great import.
+    ... """, Message)
+    >>> msgdata = recips.copy()
+    >>> process(mlist, msg, msgdata)
+    >>> sorted(msgdata['recips'])
+    ['bperson@example.com']
+    >>> print msg.as_string()
+    From: Claire Person <cperson@example.com>
+    CC: aperson@example.com
+    <BLANKLINE>
+    Something of great import.
+    <BLANKLINE>
+
+But if they're mentioned on the CC line and have receive_list_copy set to True
+(the default), then they still get a list copy.
+
+    >>> msg = message_from_string("""\
+    ... From: Claire Person <cperson@example.com>
+    ... CC: bperson@example.com
+    ...
+    ... Something of great import.
+    ... """, Message)
+    >>> msgdata = recips.copy()
+    >>> process(mlist, msg, msgdata)
+    >>> sorted(msgdata['recips'])
+    ['aperson@example.com', 'bperson@example.com']
+    >>> print msg.as_string()
+    From: Claire Person <cperson@example.com>
+    CC: bperson@example.com
+    <BLANKLINE>
+    Something of great import.
+    <BLANKLINE>
+
+Other headers checked for recipients include the To...
+
+    >>> msg = message_from_string("""\
+    ... From: Claire Person <cperson@example.com>
+    ... To: aperson@example.com
+    ...
+    ... Something of great import.
+    ... """, Message)
+    >>> msgdata = recips.copy()
+    >>> process(mlist, msg, msgdata)
+    >>> sorted(msgdata['recips'])
+    ['bperson@example.com']
+    >>> print msg.as_string()
+    From: Claire Person <cperson@example.com>
+    To: aperson@example.com
+    <BLANKLINE>
+    Something of great import.
+    <BLANKLINE>
+
+...Resent-To...
+
+    >>> msg = message_from_string("""\
+    ... From: Claire Person <cperson@example.com>
+    ... Resent-To: aperson@example.com
+    ...
+    ... Something of great import.
+    ... """, Message)
+    >>> msgdata = recips.copy()
+    >>> process(mlist, msg, msgdata)
+    >>> sorted(msgdata['recips'])
+    ['bperson@example.com']
+    >>> print msg.as_string()
+    From: Claire Person <cperson@example.com>
+    Resent-To: aperson@example.com
+    <BLANKLINE>
+    Something of great import.
+    <BLANKLINE>
+
+...and Resent-CC headers.
+
+    >>> msg = message_from_string("""\
+    ... From: Claire Person <cperson@example.com>
+    ... Resent-Cc: aperson@example.com
+    ...
+    ... Something of great import.
+    ... """, Message)
+    >>> msgdata = recips.copy()
+    >>> process(mlist, msg, msgdata)
+    >>> sorted(msgdata['recips'])
+    ['bperson@example.com']
+    >>> print msg.as_string()
+    From: Claire Person <cperson@example.com>
+    Resent-Cc: aperson@example.com
+    <BLANKLINE>
+    Something of great import.
+    <BLANKLINE>
+
+
+Clean up
+--------
+
+    >>> for mlist in config.list_manager.mailing_lists:
+    ...     config.list_manager.delete(mlist)
+    >>> flush()
+    >>> list(config.list_manager.mailing_lists)
+    []
diff --git a/Mailman/testing/test_avoid_duplicates.py b/Mailman/testing/test_avoid_duplicates.py
new file mode 100644
index 000000000..96599f1ed
--- /dev/null
+++ b/Mailman/testing/test_avoid_duplicates.py
@@ -0,0 +1,33 @@
+# Copyright (C) 2007 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+
+"""Doctest harness for the AvoidDuplicates handler."""
+
+import os
+import doctest
+import unittest
+
+options = (doctest.ELLIPSIS
+           | doctest.NORMALIZE_WHITESPACE
+           | doctest.REPORT_NDIFF)
+
+
+def test_suite():
+    suite = unittest.TestSuite()
+    suite.addTest(doctest.DocFileSuite('../docs/avoid-duplicates.txt',
+                                       optionflags=options))
+    return suite