This change implements storing list members and digest members as

dicts instead of lists, which optimizes Utils.FindMatchingAddresses
and general membership management, especially for large lists.
MailList.py now supplies .GetMembers() and .GetDigestMembers() to
supply the data in list form to anything that needs it that way.
An new install showed this worked fine with some cursory testing of
the cgi's and interactive poking around.
A detailed listing of the changes follows:
Mailman/Defaults.py.in: change data version to 11
Mailman/Digester.py: initvars now instantiates digest_members as {}
                     instead of []
                     lines 113-114 and 121-122 now use del
This change implements storing list members and digest members as
dicts instead of lists, which optimizes Utils.FindMatchingAddresses
and general membership management, especially for large lists.
MailList.py now supplies .GetMembers() and .GetDigestMembers() to
supply the data in list form to anything that needs it that way.
Though INSTALL shows up on the changed files section, a diff a few
seconds ago didn't show any differences in that file, so I hope nobody
changed it in the interim.
An new install showed this worked fine with some cursory testing of
the cgi's and interactive poking around.
A detailed listing of the changes follows:
Mailman/Defaults.py.in: change data version to 11
Mailman/Digester.py: initvars now instantiates digest_members as {}
                     instead of []
                     lines 113-114 and 121-122 now use del
                     list.[digest_]member instead of
                     list.[digest_]members.remove
                     when figuring who to actually send digests to,
                     use list.GetDigestMembers() instead of
                     list.digest_members.
Mailman/HTMLFormatter: now uses list.Get[Digest]Members to get
                       subscribers, and length of digested subscribers
                       and regular members
MailCommandHandler, SecurityManager,Cgi/handle_opts, Cgi/options: all
               simple replacements of list.[digest_]members with
                       list.Get[Digest]Members().
Mailman/Cgi/admin.py: mostly simple replacements of
list.[digest_]members with the Get..() methods, however, the
membership management section now works much quicker and changes
digest->nodigest subscriptions via dictionary manipulations.
Mailman/versions.py: updates lists to use dicts and changed
list.[digest_]members to use the list.Get[Digest]Members() methods.
Mailman/Utils.py: added a function "GetPossibleMatchingAddresses"
which when fed an address, returns the list of addresses that "smart"
address matching would match.
changed FindMatchingAddresses(name, list) to use a new signature:
FindMatchingAddresses(name, *dicts), where dicts is a list of
dictionaries keyed by addresses.  Just realized that this would better
be FindMatchingAddresses(name, dict, *dicts) so that it enforces
atleast 2 args... I'll make that change in a sec.
All uses of FindMatchingAddresses have been changed to fit the new
arguments.
scott
----:**-F1  cvs30458aaa    1:12PM 0.98 Mail   (Text Fill)--L59--32%-------------------------------------------
?

11 files changed, 143 insertions, 74 deletions

diff --git a/Mailman/Cgi/admin.py b/Mailman/Cgi/admin.py
index 410edb0e9..0ebf2bc93 100644
--- a/Mailman/Cgi/admin.py
+++ b/Mailman/Cgi/admin.py
@@ -144,12 +144,12 @@ def main():
                                     ' (does it have the colon?)<ul> %s </ul>',
                                     line)
 
-	if not lst.digestable and len(lst.digest_members):
+	if not lst.digestable and len(lst.GetDigestMembers()):
 	    AddErrorMessage(doc,
                             'Warning:  you have digest members,'
                             ' but digests are turned off.'
                             '  Those people will not receive mail.')
-	if not lst.nondigestable and len(lst.members):
+	if not lst.nondigestable and len(lst.GetMembers()):
 	    AddErrorMessage(doc,
                             'Warning:  you have lst members,'
                             ' but non-digestified mail is turned'
@@ -487,13 +487,7 @@ def FormatMembershipOptions(lst):
                            user_table.GetCurrentCellIndex(),
                            bgcolor="#cccccc", colspan=8)
 
-    members = {}
-    digests = {}
-    for member in lst.members:
-        members[member] = 1
-    for member in lst.digest_members:
-        digests[member] = 1
-    all = lst.members + lst.digest_members
+    all = lst.GetMembers() + lst.GetDigestMembers()
     if len(all) > lst.admin_member_chunksize:
         chunks = Utils.chunkify(all, lst.admin_member_chunksize)
         if not cgi_data.has_key("chunk"):
@@ -520,7 +514,7 @@ def FormatMembershipOptions(lst):
         cells = [member + "<input type=hidden name=user value=%s>" % (member),
                  "subscribed " +CheckBox(member + "_subscribed", "on", 1).Format(),
                  ]
-        if members.get(member):
+        if lst.members.get(member):
             cells.append("digest " + CheckBox(member + "_digest", "off", 0).Format())
         else:
             cells.append("digest " + CheckBox(member + "_digest", "on", 1).Format())
@@ -772,18 +766,18 @@ def ChangeOptions(lst, category, cgi_info, document):
                 dirty = 1
                 continue
             if not cgi_info.has_key("%s_digest" % (user)):
-                if user in lst.digest_members:
-                    lst.digest_members.remove(user)
+                if lst.digest_members.has_key(user):
+                    del lst.digest_members[user]
                     dirty = 1
-                if user not in lst.members:
-                    lst.members.append(user)
+                if not lst.members.has_key(user):
+                    lst.members[user] = 1
                     dirty = 1
             else:
-                if user not in lst.digest_members:
-                    lst.digest_members.append(user)
+                if not lst.digest_members.has_key(user):
+                    lst.digest_members[user] = 1
                     dirty = 1
-                if user in lst.members:
-                    lst.members.remove(user)
+                if lst.members.has_key(user):
+                    del lst.members[user]
                     dirty = 1
                 
             for opt in ("hide", "nomail", "ack", "norcv", "plain"):
diff --git a/Mailman/Cgi/handle_opts.py b/Mailman/Cgi/handle_opts.py
index 0c64716f5..a96c4bbbe 100644
--- a/Mailman/Cgi/handle_opts.py
+++ b/Mailman/Cgi/handle_opts.py
@@ -88,8 +88,12 @@ def main():
 
     error = 0
     operation = ""
-
-    if string.lower(user) not in list.members + list.digest_members:
+    user = Utils.LCDomain(user)
+    #
+    # XXX shouldn't this check use Utils.FindMatchingAddresses?
+    #                                               -scott
+    if not list.members.has_key(user) \
+       and not list.digest_members.has_key(user):
         PrintResults("%s not a member!<p>" % user)
 
     if form.has_key("unsub"):
diff --git a/Mailman/Cgi/options.py b/Mailman/Cgi/options.py
index 97f2c1631..768678385 100644
--- a/Mailman/Cgi/options.py
+++ b/Mailman/Cgi/options.py
@@ -63,8 +63,9 @@ def main():
         doc.AddItem(htmlformat.Bold("%s: No such list." % list_name ))
         print doc.Format()
         sys.exit(0)
-
-    if Utils.LCDomain(user) not in list.members + list.digest_members:
+    user = Utils.LCDomain(user)
+    if not list.members.has_key(user) \
+       and not list.digest_members.has_key(user):
         doc.AddItem(htmlformat.Header(2, "Error"))
         doc.AddItem(htmlformat.Bold("%s: No such member %s."
                                     % (list_name, `user`)))
diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in
index 2a3f6dcef..12e7b04a8 100644
--- a/Mailman/Defaults.py.in
+++ b/Mailman/Defaults.py.in
@@ -269,4 +269,4 @@ PRIVATE_ARCHIVE_FILE_DIR	= os.path.join(PREFIX, 'archives/private')
 VERSION           = '@VERSION@'
 
 # Data file version number
-DATA_FILE_VERSION = 10
+DATA_FILE_VERSION = 11
diff --git a/Mailman/Digester.py b/Mailman/Digester.py
index 6b6a3ef6b..a794bd865 100644
--- a/Mailman/Digester.py
+++ b/Mailman/Digester.py
@@ -60,7 +60,7 @@ class Digester:
 	self.digest_footer = mm_cfg.DEFAULT_DIGEST_FOOTER
 	
 	# Non-configurable.
-	self.digest_members = []
+	self.digest_members = {}
 	self.next_digest_number = 1
 
     def GetConfigInfo(self):
@@ -104,25 +104,25 @@ class Digester:
 	addr = self.FindUser(sender)
 	if not addr:
 	    raise Errors.MMNotAMemberError
-	if addr in self.members:
+	if self.members.has_key(addr):
 	    if value == 0:
 		raise Errors.MMAlreadyUndigested
 	    else:
 		if not self.digestable:
 		    raise Errors.MMCantDigestError
-		self.members.remove(addr)
-		self.digest_members.append(addr)
+		del self.members[addr]
+		self.digest_members[addr] = 1
 	else:
 	    if value == 1:
 		raise Errors.MMAlreadyDigested
 	    else:
 		if not self.nondigestable:
 		    raise Errors.MMMustDigestError
-		self.digest_members.remove(addr)
-		self.members.append(addr)
+		del self.digest_members[addr]
+		self.members[addr] = 1
 	self.Save()
 
-# Internal function, don't call this.
+    # Internal function, don't call this.
     def SaveForDigest(self, post):
 	"""Add message to index, and to the digest.  If the digest is large
 	enough when we're done writing, send it out."""
@@ -228,7 +228,8 @@ class Digester:
 	def HatesMime(x, s=self, v=mm_cfg.DisableMime):
 	    return s.GetUserOption(x, v)
 
-	recipients = filter(DeliveryEnabled, self.digest_members)
+        digestmembers = self.GetDigestMembers()
+	recipients = filter(DeliveryEnabled, digestmembers)
 	mime_recipients = filter(LikesMime, recipients)
 	text_recipients = filter(HatesMime, recipients)
 	self.LogMsg("digest",
@@ -237,8 +238,8 @@ class Digester:
 	self.LogMsg("digest",
 		    ('Fake %d digesters, %d disabled.  '
 		     'Active: %d MIMEers, %d non.'),
-		    len(self.digest_members),
-		    len(self.digest_members) - len(recipients),
+		    len(digestmembers),
+		    len(digestmembers) - len(recipients),
 		    len(mime_recipients), len(text_recipients))
 
     def SendDigest(self):
@@ -255,7 +256,8 @@ class Digester:
 	    return not s.GetUserOption(x, v)
 	def HatesMime(x, s=self, v=mm_cfg.DisableMime):
 	    return s.GetUserOption(x, v)
-	recipients = filter(DeliveryEnabled, self.digest_members)
+        digestmembers = self.GetDigestMembers()
+	recipients = filter(DeliveryEnabled, digestmembers)
 	mime_recipients = filter(LikesMime, recipients)
 	text_recipients = filter(HatesMime, recipients)
 
@@ -265,10 +267,10 @@ class Digester:
                     self.real_name,
                     self.next_digest_number,
                     topics_number,
-                    len(self.digest_members),
+                    len(digestmembers),
                     len(mime_recipients),
                     len(text_recipients),
-                    len(self.digest_members) - len(recipients))
+                    len(digestmembers) - len(recipients))
 
         if mime_recipients or text_recipients:
             d = Digest(self, topics_text, digest_file.read())
diff --git a/Mailman/HTMLFormatter.py b/Mailman/HTMLFormatter.py
index fcd63ad39..2318653d6 100644
--- a/Mailman/HTMLFormatter.py
+++ b/Mailman/HTMLFormatter.py
@@ -69,12 +69,15 @@ class HTMLFormatter:
 	def NotHidden(x, s=self, v=mm_cfg.ConcealSubscription):
 	    return not s.GetUserOption(x, v)
 
+        
 	if digest:
-	    people = filter(NotHidden, self.digest_members)
-	    num_concealed = len(self.digest_members) - len(people)
+            digestmembers = self.GetDigestMembers()
+	    people = filter(NotHidden, digestmembers)
+	    num_concealed = len(digestmembers) - len(people)
 	else:
-	    people = filter(NotHidden, self.members)
-	    num_concealed = len(self.members) - len(people)
+            members = self.GetMembers()
+	    people = filter(NotHidden, members)
+	    num_concealed = len(members) - len(people)
         people.sort()
 	if (num_concealed > 0):
 	    plurality = (((num_concealed > 1) and "s") or "")
@@ -339,6 +342,8 @@ class HTMLFormatter:
     # This needs to wait until after the list is inited, so let's build it
     # when it's needed only.
     def GetStandardReplacements(self):
+        dmember_len = len(self.GetDigestMembers())
+        member_len = len(self.GetMembers())
 	return { 
 	    '<mm-mailman-footer>' : self.GetMailmanFooter(),
 	    '<mm-list-name>' : self.real_name,
@@ -355,10 +360,9 @@ class HTMLFormatter:
             	self.RestrictedListMessage('current archive',
                                            self.archive_private),
 	    '<mm-digest-users>' : self.FormatUsers(1),
-	    '<mm-num-reg-users>' : `len(self.members)`,
-	    '<mm-num-digesters>' : `len(self.digest_members)`,
-	    '<mm-num-members>' : (`len(self.members)`
-				  + `len(self.digest_members)`),
+	    '<mm-num-reg-users>' : `member_len`,
+	    '<mm-num-digesters>' : `dmember_len`,
+	    '<mm-num-members>' : (`member_len + dmember_len`),
 	    '<mm-posting-addr>' : '%s' % self.GetListEmail(),
 	    '<mm-request-addr>' : '%s' % self.GetRequestEmail(),
 	    '<mm-owner>' : self.GetAdminEmail(),
diff --git a/Mailman/MailCommandHandler.py b/Mailman/MailCommandHandler.py
index 7702eae82..f78eda456 100644
--- a/Mailman/MailCommandHandler.py
+++ b/Mailman/MailCommandHandler.py
@@ -358,25 +358,26 @@ class MailCommandHandler:
 	    self.AddError("Private list: only members may see list "
 			  "of subscribers.")
 	    return
-	if not len(self.digest_members) and not len(self.members):
+        digestmembers = self.GetDigestMembers()
+        members = self.GetMembers()
+	if not len(digestmembers) and not len(members):
 	    self.AddToResponse("NO MEMBERS.")
 	    return
 	
 	def NotHidden(x, s=self, v=mm_cfg.ConcealSubscription):
 	    return not s.GetUserOption(x, v)
 
-	if len(self.digest_members):
+
+	if len(digestmembers):
 	    self.AddToResponse("")
 	    self.AddToResponse("Digest Members:")
-            digestmembers = self.digest_members[:]
             digestmembers.sort()
 	    self.AddToResponse(string.join(map(AddTab,
                                                filter(NotHidden,
                                                       digestmembers)),
                                            "\n"))
-	if len(self.members):
+	if len(members):
 	    self.AddToResponse("Non-Digest Members:")
-            members = self.members[:]
             members.sort()
 	    self.AddToResponse(string.join(map(AddTab,
                                                filter(NotHidden, members)),
diff --git a/Mailman/MailList.py b/Mailman/MailList.py
index 1fdebd3de..68d57a807 100644
--- a/Mailman/MailList.py
+++ b/Mailman/MailList.py
@@ -63,8 +63,17 @@ class MailList(MailCommandHandler, HTMLFormatter, Deliverer, ListAdmin,
 	for f in self._log_files.values():
 	    f.close()
 
+    def GetMembers(self):
+        """returns a list of the members."""
+        return self.members.keys()
+
+    def GetDigestMembers(self):
+        """returns a list of digest members."""
+        return self.digest_members.keys()
+
     def GetAdminEmail(self):
         return '%s-admin@%s' % (self._internal_name, self.host_name)
+
     def GetMemberAdminEmail(self, member):
         """Usually the member addr, but modified for umbrella lists.
 
@@ -114,7 +123,7 @@ class MailList(MailCommandHandler, HTMLFormatter, Deliverer, ListAdmin,
 
     def GetUserOption(self, user, option):
 	if option == mm_cfg.Digests:
-	    return user in self.digest_members
+	    return self.digest_members.has_key(user)
 	if not self.user_options.has_key(user):
 	    return 0
 	return not not self.user_options[user] & option
@@ -132,8 +141,8 @@ class MailList(MailCommandHandler, HTMLFormatter, Deliverer, ListAdmin,
 
     def FindUser(self, email):
 	matches = Utils.FindMatchingAddresses(email,
-						 (self.members
-						  + self.digest_members))
+                                              (self.members,
+                                               self.digest_members))
 	if not matches or not len(matches):
 	    return None
 	return matches[0]
@@ -157,7 +166,7 @@ class MailList(MailCommandHandler, HTMLFormatter, Deliverer, ListAdmin,
 
 	# Must save this state, even though it isn't configurable
 	self.volume = 1
-	self.members = [] # self.digest_members is initted in mm_digest
+	self.members = {} # self.digest_members is initted in mm_digest
 	self.data_version = mm_cfg.VERSION
 	self.last_post_time = 0
 	
@@ -767,10 +776,10 @@ class MailList(MailCommandHandler, HTMLFormatter, Deliverer, ListAdmin,
 	if self.IsMember(name):
 	    raise Errors.MMAlreadyAMember
 	if digest:
-	    self.digest_members.append(name)
+	    self.digest_members[name] = 1
             kind = " (D)"
 	else:
-	    self.members.append(name)
+	    self.members[name] = 1
             kind = ""
         self.SetUserOption(name, mm_cfg.DisableMime,
                            1 - self.mime_is_default_digest)
@@ -800,8 +809,8 @@ class MailList(MailCommandHandler, HTMLFormatter, Deliverer, ListAdmin,
 	self.IsListInitialized()
         # FindMatchingAddresses *should* never return more than 1 address.
         # However, should log this, just to make sure.
-	aliases = Utils.FindMatchingAddresses(name, self.members + 
-						 self.digest_members)
+	aliases = Utils.FindMatchingAddresses(name, self.members, 
+                                              self.digest_members)
 	if not len(aliases):
 	    raise Errors.MMNoSuchUserError
 
@@ -814,14 +823,14 @@ class MailList(MailCommandHandler, HTMLFormatter, Deliverer, ListAdmin,
 	    if me.user_options.has_key(alias):
 		del me.user_options[alias]
 	    try:
-		me.members.remove(alias)
+		del me.members[alias]
 		kind = "regular"
-	    except ValueError:
+	    except KeyError:
 		pass
 	    try:
-		me.digest_members.remove(alias)
+		del me.digest_members[alias]
 		kind = "digest"
-	    except ValueError:
+	    except KeyError:
 		pass
 
 	map(DoActualRemoval, aliases)
@@ -835,8 +844,8 @@ class MailList(MailCommandHandler, HTMLFormatter, Deliverer, ListAdmin,
                     self._internal_name, name, whence)
 
     def IsMember(self, address):
-	return len(Utils.FindMatchingAddresses(address, self.members +
-						    self.digest_members))
+	return len(Utils.FindMatchingAddresses(address, self.members,
+                                               self.digest_members))
 
     def HasExplicitDest(self, msg):
 	"""True if list name or any acceptable_alias is included among the
@@ -1038,7 +1047,7 @@ class MailList(MailCommandHandler, HTMLFormatter, Deliverer, ListAdmin,
 	    if self.GetUserOption(sender, mm_cfg.AcknowlegePosts):
 		ack_post = 1
 	# Deliver the mail.
-	recipients = self.members[:] 
+	recipients = self.GetMembers()
 	if dont_send_to_sender:
             try:
                 recipients.remove(sender)
diff --git a/Mailman/SecurityManager.py b/Mailman/SecurityManager.py
index c25b59900..c3fd72adf 100644
--- a/Mailman/SecurityManager.py
+++ b/Mailman/SecurityManager.py
@@ -70,7 +70,8 @@ class SecurityManager:
     def ConfirmUserPassword(self, user, pw):
 	if self.ValidAdminPassword(pw):
 	    return 1
-	if not user in self.members and not user in self.digest_members:
+	if not self.members.has_key(user) \
+           and not self.digest_members.has_key(user):
 	    user = self.FindUser(user)
         try:
             if string.lower(pw) <> string.lower(self.passwords[user]):
diff --git a/Mailman/Utils.py b/Mailman/Utils.py
index 3dc248856..8009cb2d9 100644
--- a/Mailman/Utils.py
+++ b/Mailman/Utils.py
@@ -319,16 +319,48 @@ def AddressesMatch(addr1, addr2):
     return 1
 
 
-def FindMatchingAddresses(name, array):
-    """Given an email address, and a list of email addresses, returns the
-    subset of the list that matches the given address.  Should sort based
-    on exactness of match, just in case."""
 
-    def CallAddressesMatch (x, y=name):
-	return AddressesMatch(x,y)
+def GetPossibleMatchingAddrs(name):
+    """returns a sorted list of addresses that could possibly match
+    a given name.
+
+    For Example, given scott@pobox.com, return ['scott@pobox.com'],
+    given scott@blackbox.pobox.com return ['scott@blackbox.pobox.com',
+                                           'scott@pobox.com']"""
+
+    name = LCDomain(name)
+    user, domain = ParseEmail(name)
+    res = [name]
+    domain = domain[1:]
+    while len(domain) >= 2:
+        res.append("%s@%s" % (user, string.join(domain, ".")))
+        domain = domain[1:]
+    return res
+
+
+
+def FindMatchingAddresses(name, *dicts):
+    """Given an email address, and any number of dictionaries keyed by
+    email addresses, returns the subset of the list that matches the
+    given address.  Should sort based on exactness of match,
+    just in case."""
+
+    if not mm_cfg.SMART_ADDRESS_MATCH:
+        for d in dicts:
+            if d.has_key(LCDomain(name)):
+                return [name]
+        return []
+    #
+    # GetPossibleMatchingAddrs return LCDomain'd values
+    #
+    p_matches = GetPossibleMatchingAddrs(name) 
+    res = []
+    for pm in p_matches:
+        for d in dicts:
+            if d.has_key(pm):
+                res.append(pm)
+    return res
 
-    matches = filter(CallAddressesMatch, array)
-    return matches
   
 def GetRandomSeed():
     chr1 = int(random.random() * 57) + 65
diff --git a/Mailman/versions.py b/Mailman/versions.py
index 80fee9260..fa6c6c28f 100644
--- a/Mailman/versions.py
+++ b/Mailman/versions.py
@@ -100,13 +100,29 @@ def UpdateOldVars(l, stored_state):
     else: # make sure everyone gets the behavior the list used to have
         if l.posters:
             l.member_posting_only = 0
+    #
+    # transfer the list data type for holding members and digest members
+    # to the dict data type starting file format version 11
+    #
+    if type(l.members) is type([]):
+        members = {}
+        for m in l.members:
+            members[m] = 1
+        l.members = members
+    if type(l.digest_members) is type([]):
+        dmembers = {}
+        for dm in l.digest_members:
+            dmembers[dm] = 1
+        l.digest_members = dmembers
+
+        
     
 
 def UpdateOldUsers(l):
     """Transform sense of changed user options."""
     if older(l.data_version, "1.0b1.2"):
         # Mime-digest bitfield changed from Enable to Disable after 1.0b1.1.
-        for m in l.members + l.digest_members:
+        for m in l.GetMembers() + l.GetDigestMembers():
             was = l.GetUserOption(m, mm_cfg.DisableMime)
             l.SetUserOption(m, mm_cfg.DisableMime, not was)
 
@@ -165,3 +181,8 @@ def older(version, reference):
 #            section = int(section)
 #        got.append(section)
 #    return got
+
+
+
+
+