Implement the new, simplified membership model. Rosters and RosterSets as

they were previously known are now gone.  Rosters, rather than being a
database entity that collects users, is now just a filter on the member
database.  This way, we can use generic rosters to search for regular members,
digest members, owners, or moderators.  More advanced rosters can do all kinds
of other membership queries.  But rosters no longer need to be a database
entity.

Users have a name, password, optional preferences, and a set of addresses, but
users are not subscribed to mailing lists.

Addresses have the email address, some verification information, and optional
preferences.

Members tie an address to a mailing list, through a role, with optional
preferences.

Other changes here include:

MailList.fqdn_listname() moved to the MailingList model entity.

Added MemberRole enum and SystemDefaultPreferences to Mailman.constants.

Profiles are renamed to Preferences (same with the interface), but the files
are not yet moved.  This happens later.

We mostly don't need has_*() relationships on the entity classes, because we
generally don't need the reverse relationship.  Use belongs_to() because that
creates the foreign key, even though the wording seems counter intuitive.

IAddress.subscribe() added.

Tell Elixir to use shortnames for all tables.

Remove the OldStyleMembership fields from MailingList.

Remove all the interface elements and database fields that talk about rosters
and rostersets.

Convert Version entity to has_field().

10 files changed, 288 insertions, 204 deletions

diff --git a/Mailman/database/model/__init__.py b/Mailman/database/model/__init__.py
index 11ca11f89..612510632 100644
--- a/Mailman/database/model/__init__.py
+++ b/Mailman/database/model/__init__.py
@@ -19,9 +19,7 @@ __all__ = [
     'Address',
     'Language',
     'MailingList',
-    'Profile',
-    'Roster',
-    'RosterSet',
+    'Preferences',
     'User',
     'Version',
     ]
@@ -44,9 +42,8 @@ from Mailman.configuration import config
 from Mailman.database.model.address import Address
 from Mailman.database.model.language import Language
 from Mailman.database.model.mailinglist import MailingList
-from Mailman.database.model.profile import Profile
-from Mailman.database.model.roster import Roster
-from Mailman.database.model.rosterset import RosterSet
+from Mailman.database.model.member import Member
+from Mailman.database.model.profile import Preferences
 from Mailman.database.model.user import User
 from Mailman.database.model.version import Version
 
diff --git a/Mailman/database/model/address.py b/Mailman/database/model/address.py
index 53d5016e5..450c40235 100644
--- a/Mailman/database/model/address.py
+++ b/Mailman/database/model/address.py
@@ -21,11 +21,12 @@ from zope.interface import implements
 
 from Mailman.interfaces import IAddress
 
-
-ROSTER_KIND = 'Mailman.database.model.roster.Roster'
-USER_KIND   = 'Mailman.database.model.user.User'
+MEMBER_KIND     = 'Mailman.database.model.member.Member'
+PREFERENCE_KIND = 'Mailman.database.model.profile.Preferences'
+USER_KIND       = 'Mailman.database.model.user.User'
 
 
+
 class Address(Entity):
     implements(IAddress)
 
@@ -35,8 +36,10 @@ class Address(Entity):
     has_field('registered_on',  DateTime)
     has_field('validated_on',   DateTime)
     # Relationships
-    has_and_belongs_to_many('rosters', of_kind=ROSTER_KIND)
-    belongs_to('user', of_kind=USER_KIND)
+    belongs_to('user',          of_kind=USER_KIND)
+    belongs_to('preferences',   of_kind=PREFERENCE_KIND)
+    # Options
+    using_options(shortnames=True)
 
     def __str__(self):
         return formataddr((self.real_name, self.address))
@@ -44,3 +47,11 @@ class Address(Entity):
     def __repr__(self):
         return '<Address: %s [%s]>' % (
             str(self), ('verified' if self.verified else 'not verified'))
+
+    def subscribe(self, mlist, role):
+        from Mailman.database.model import Member
+        # This member has no preferences by default.
+        member = Member(role=role,
+                        mailing_list=mlist.fqdn_listname,
+                        address=self)
+        return member
diff --git a/Mailman/database/model/language.py b/Mailman/database/model/language.py
index 3597a128d..e065d5bad 100644
--- a/Mailman/database/model/language.py
+++ b/Mailman/database/model/language.py
@@ -20,3 +20,5 @@ from elixir import *
 
 class Language(Entity):
     has_field('code',   Unicode)
+    # Options
+    using_options(shortnames=True)
diff --git a/Mailman/database/model/mailinglist.py b/Mailman/database/model/mailinglist.py
index 28e2c11dc..edd2eab0d 100644
--- a/Mailman/database/model/mailinglist.py
+++ b/Mailman/database/model/mailinglist.py
@@ -50,16 +50,6 @@ class MailingList(Entity):
     has_field('one_last_digest',                            PickleType),
     has_field('volume',                                     Integer),
     has_field('last_post_time',                             Float),
-    # OldStyleMemberships attributes, temporarily stored as pickles.
-    has_field('bounce_info',                                PickleType),
-    has_field('delivery_status',                            PickleType),
-    has_field('digest_members',                             PickleType),
-    has_field('language',                                   PickleType),
-    has_field('members',                                    PickleType),
-    has_field('passwords',                                  PickleType),
-    has_field('topics_userinterest',                        PickleType),
-    has_field('user_options',                               PickleType),
-    has_field('usernames',                                  PickleType),
     # Attributes which are directly modifiable via the web u/i.  The more
     # complicated attributes are currently stored as pickles, though that
     # will change as the schema and implementation is developed.
@@ -163,113 +153,29 @@ class MailingList(Entity):
     has_field('umbrella_member_suffix',                     Unicode),
     has_field('unsubscribe_policy',                         Integer),
     has_field('welcome_msg',                                Unicode),
-    # Indirect relationships
-    has_field('owner_rosterset',                            Unicode),
-    has_field('moderator_rosterset',                        Unicode),
     # Relationships
 ##     has_and_belongs_to_many(
 ##         'available_languages',
 ##         of_kind='Mailman.database.model.languages.Language')
+    # Options
+    using_options(shortnames=True)
 
     def __init__(self, fqdn_listname):
         super(MailingList, self).__init__()
         listname, hostname = split_listname(fqdn_listname)
         self.list_name = listname
         self.host_name = hostname
-        # Create two roster sets, one for the owners and one for the
-        # moderators.  MailingLists are connected to RosterSets indirectly, in
-        # order to preserve the ability to store user data and list data in
-        # different databases.
-        name = fqdn_listname + ' owners'
-        self.owner_rosterset = name
-        roster = config.user_manager.create_roster(name)
-        config.user_manager.create_rosterset(name).add(roster)
-        name = fqdn_listname + ' moderators'
-        self.moderator_rosterset = name
-        roster = config.user_manager.create_roster(name)
-        config.user_manager.create_rosterset(name).add(roster)
-
-    def delete_rosters(self):
-        listname = fqdn_listname(self.list_name, self.host_name)
-        # Delete the list owner roster and roster set.
-        name = listname + ' owners'
-        roster = config.user_manager.get_roster(name)
-        assert roster, 'Missing roster: %s' % name
-        config.user_manager.delete_roster(roster)
-        rosterset = config.user_manager.get_rosterset(name)
-        assert rosterset, 'Missing roster set: %s' % name
-        config.user_manager.delete_rosterset(rosterset)
-        name = listname + ' moderators'
-        roster = config.user_manager.get_roster(name)
-        assert roster, 'Missing roster: %s' % name
-        config.user_manager.delete_roster(roster)
-        rosterset = config.user_manager.get_rosterset(name)
-        assert rosterset, 'Missing roster set: %s' % name
-        config.user_manager.delete_rosterset(rosterset)
-
-    # IMailingListRosters
+        # Create several rosters for filtering out or querying the membership
+        # table.
+        from Mailman.database.model import roster
+        self.owners = roster.OwnerRoster(self)
+        self.moderators = roster.ModeratorRoster(self)
+        self.administrators = roster.AdministratorRoster(self)
+        self.members = roster.MemberRoster(self)
+        self.regular_members = roster.RegularMemberRoster(self)
+        self.digest_members = roster.DigestMemberRoster(self)
 
     @property
-    def owners(self):
-        for user in _collect_users(self.owner_rosterset):
-            yield user
-
-    @property
-    def moderators(self):
-        for user in _collect_users(self.moderator_rosterset):
-            yield user
-
-    @property
-    def administrators(self):
-        for user in _collect_users(self.owner_rosterset,
-                                   self.moderator_rosterset):
-            yield user
-
-    @property
-    def owner_rosters(self):
-        rosterset = config.user_manager.get_rosterset(self.owner_rosterset)
-        for roster in rosterset.rosters:
-            yield roster
-
-    @property
-    def moderator_rosters(self):
-        rosterset = config.user_manager.get_rosterset(self.moderator_rosterset)
-        for roster in rosterset.rosters:
-            yield roster
-
-    def add_owner_roster(self, roster):
-        rosterset = config.user_manager.get_rosterset(self.owner_rosterset)
-        rosterset.add(roster)
-
-    def delete_owner_roster(self, roster):
-        rosterset = config.user_manager.get_rosterset(self.owner_rosterset)
-        rosterset.delete(roster)
-
-    def add_moderator_roster(self, roster):
-        rosterset = config.user_manager.get_rosterset(self.moderator_rosterset)
-        rosterset.add(roster)
-
-    def delete_moderator_roster(self, roster):
-        rosterset = config.user_manager.get_rosterset(self.moderator_rosterset)
-        rosterset.delete(roster)
-
-
-
-def _collect_users(*rosterset_names):
-    users = set()
-    for name in rosterset_names:
-        # We have to indirectly look up the roster set's name in the user
-        # manager.  This is how we enforce separation between the list manager
-        # and the user manager storages.
-        rosterset = config.user_manager.get_rosterset(name)
-        assert rosterset is not None, 'No RosterSet named: %s' % name
-        for roster in rosterset.rosters:
-            # Rosters collect addresses.  It's not required that an address is
-            # linked to a user, but it must be the case that all addresses on
-            # the owner roster are linked to a user.  Get the user that's
-            # linked to each address and add it to the set.
-            for address in roster.addresses:
-                user = config.user_manager.get_user(address.address)
-                assert user is not None, 'Unlinked address: ' + address.address
-                users.add(user)
-    return users
+    def fqdn_listname(self):
+        """See IMailingListIdentity."""
+        return fqdn_listname(self.list_name, self.host_name)
diff --git a/Mailman/database/model/member.py b/Mailman/database/model/member.py
new file mode 100644
index 000000000..db37ebb49
--- /dev/null
+++ b/Mailman/database/model/member.py
@@ -0,0 +1,61 @@
+# Copyright (C) 2007 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+
+from elixir import *
+from zope.interface import implements
+
+from Mailman.Utils import split_listname
+from Mailman.constants import SystemDefaultPreferences
+from Mailman.database.types import EnumType
+from Mailman.interfaces import IMember, IPreferences
+
+
+ADDRESS_KIND    = 'Mailman.database.model.address.Address'
+PREFERENCE_KIND = 'Mailman.database.model.profile.Preferences'
+
+
+
+class Member(Entity):
+    implements(IMember)
+
+    has_field('role',           EnumType)
+    has_field('mailing_list',   Unicode)
+    # Relationships
+    belongs_to('address',       of_kind=ADDRESS_KIND)
+    belongs_to('_preferences',  of_kind=PREFERENCE_KIND)
+    # Options
+    using_options(shortnames=True)
+
+    def __repr__(self):
+        return '<Member: %s on %s as %s>' % (
+            self.address, self.mailing_list, self.role)
+
+    @property
+    def preferences(self):
+        from Mailman.database.model import MailingList
+        if self._preferences:
+            return self._preferences
+        if self.address.preferences:
+            return self.address.preferences
+        if self.address.user.preferences:
+            return self.address.user.preferences
+        list_name, host_name = split_listname(self.mailing_list)
+        mlist = MailingList.get_by(list_name=list_name,
+                                   host_name=host_name)
+        if mlist.preferences:
+            return mlist.preferences
+        return SystemDefaultPreferences
diff --git a/Mailman/database/model/profile.py b/Mailman/database/model/profile.py
index 49e108728..935ae08fb 100644
--- a/Mailman/database/model/profile.py
+++ b/Mailman/database/model/profile.py
@@ -19,13 +19,18 @@ from elixir import *
 from email.utils import formataddr
 from zope.interface import implements
 
-from Mailman.constants import DeliveryMode
+from Mailman.constants import SystemDefaultPreferences as Prefs
 from Mailman.database.types import EnumType
-from Mailman.interfaces import IProfile
+from Mailman.interfaces import IPreferences
 
+ADDRESS_KIND    = 'Mailman.database.model.address.Address'
+MEMBER_KIND     = 'Mailman.database.model.member.Member'
+USER_KIND       = 'Mailman.database.model.user.User'
 
-class Profile(Entity):
-    implements(IProfile)
+
+
+class Preferences(Entity):
+    implements(IPreferences)
 
     has_field('acknowledge_posts',      Boolean)
     has_field('hide_address',           Boolean)
@@ -33,14 +38,14 @@ class Profile(Entity):
     has_field('receive_list_copy',      Boolean)
     has_field('receive_own_postings',   Boolean)
     has_field('delivery_mode',          EnumType)
-    # Relationships
-    belongs_to('user', of_kind='Mailman.database.model.user.User')
+    # Options
+    using_options(shortnames=True)
 
     def __init__(self):
-        super(Profile, self).__init__()
-        self.acknowledge_posts      = False
-        self.hide_address           = True
-        self.preferred_language     = 'en'
-        self.receive_list_copy      = True
-        self.receive_own_postings   = True
-        self.delivery_mode          = DeliveryMode.regular
+        super(Preferences, self).__init__()
+        self.acknowledge_posts      = Prefs.acknowledge_posts
+        self.hide_address           = Prefs.hide_address
+        self.preferred_language     = Prefs.preferred_language
+        self.receive_list_copy      = Prefs.receive_list_copy
+        self.receive_own_postings   = Prefs.receive_own_postings
+        self.delivery_mode          = Prefs.delivery_mode
diff --git a/Mailman/database/model/roster.py b/Mailman/database/model/roster.py
index bf8447433..03aa9efc3 100644
--- a/Mailman/database/model/roster.py
+++ b/Mailman/database/model/roster.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2006-2007 by the Free Software Foundation, Inc.
+# Copyright (C) 2007 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -15,37 +15,164 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
 # USA.
 
-from elixir import *
+"""An implementation of an IRoster.
+
+These are hard-coded rosters which know how to filter a set of members to find
+the ones that fit a particular role.  These are used as the member, owner,
+moderator, and administrator roster filters.
+"""
+
 from zope.interface import implements
 
-from Mailman.Errors import ExistingAddressError
+from Mailman.constants import DeliveryMode, MemberRole
+from Mailman.database.model import Member
 from Mailman.interfaces import IRoster
 
 
-ADDRESS_KIND    = 'Mailman.database.model.address.Address'
-ROSTERSET_KIND  = 'Mailman.database.model.rosterset.RosterSet'
+
+class AbstractRoster(object):
+    """An abstract IRoster class.
 
+    This class takes the simple approach of implemented the 'users' and
+    'addresses' properties in terms of the 'members' property.  This may not
+    be the most efficient way, but it works.
 
-class Roster(Entity):
+    This requires that subclasses implement the 'members' property.
+    """
     implements(IRoster)
 
-    has_field('name', Unicode)
-    # Relationships
-    has_and_belongs_to_many('addresses', of_kind=ADDRESS_KIND)
-    has_and_belongs_to_many('roster_set', of_kind=ROSTERSET_KIND)
+    def __init__(self, mlist):
+        self._mlist = mlist
+
+    @property
+    def members(self):
+        raise NotImplementedError
+
+    @property
+    def users(self):
+        # Members are linked to addresses, which in turn are linked to users.
+        # So while the 'members' attribute does most of the work, we have to
+        # keep a set of unique users.  It's possible for the same user to be
+        # subscribed to a mailing list multiple times with different
+        # addresses.
+        users = set(member.address.user for member in self.members)
+        for user in users:
+            yield user
+
+    @property
+    def addresses(self):
+        # Every Member is linked to exactly one address so the 'members'
+        # attribute does most of the work.
+        for member in self.members:
+            yield member.address
+
+
+
+class MemberRoster(AbstractRoster):
+    """Return all the members of a list."""
+
+    name = 'member'
+
+    @property
+    def members(self):
+        # Query for all the Members which have a role of MemberRole.member and
+        # are subscribed to this mailing list.  XXX we have to use a private
+        # data attribute of MailList for now.
+        for member in Member.select_by(mailing_list=self._mlist.fqdn_listname,
+                                       role=MemberRole.member):
+            yield member
+
+
+
+class OwnerRoster(AbstractRoster):
+    """Return all the owners of a list."""
+
+    name = 'owner'
+
+    @property
+    def members(self):
+        # Query for all the Members which have a role of MemberRole.member and
+        # are subscribed to this mailing list.  XXX we have to use a private
+        # data attribute of MailList for now.
+        for member in Member.select_by(mailing_list=self._mlist.fqdn_listname,
+                                       role=MemberRole.owner):
+            yield member
+
+
+
+class ModeratorRoster(AbstractRoster):
+    """Return all the owners of a list."""
+
+    name = 'moderator'
+
+    @property
+    def members(self):
+        # Query for all the Members which have a role of MemberRole.member and
+        # are subscribed to this mailing list.  XXX we have to use a private
+        # data attribute of MailList for now.
+        for member in Member.select_by(mailing_list=self._mlist.fqdn_listname,
+                                       role=MemberRole.moderator):
+            yield member
+
+
+
+class AdministratorRoster(AbstractRoster):
+    """Return all the administrators of a list."""
+
+    name = 'administrator'
+
+    @property
+    def members(self):
+        # Administrators are defined as the union of the owners and the
+        # moderators.  Until I figure out a more efficient way of doing this,
+        # this will have to do.
+        owners = Member.select_by(mailing_list=self._mlist.fqdn_listname,
+                                  role=MemberRole.owner)
+        moderators = Member.select_by(mailing_list=self._mlist.fqdn_listname,
+                                      role=MemberRole.moderator)
+        members = set(owners)
+        members.update(set(moderators))
+        for member in members:
+            yield member
+
+
+
+class RegularMemberRoster(AbstractRoster):
+    """Return all the regular delivery members of a list."""
+
+    name = 'regular_members'
+
+    @property
+    def members(self):
+        # Query for all the Members which have a role of MemberRole.member and
+        # are subscribed to this mailing list.  Then return only those members
+        # that have a regular delivery mode.
+        for member in Member.select_by(mailing_list=self._mlist.fqdn_listname,
+                                       role=MemberRole.member):
+            if member.preferences.delivery_mode == DeliveryMode.regular:
+                yield member
+
+
+
+_digest_modes = (
+    DeliveryMode.mime_digests,
+    DeliveryMode.plaintext_digests,
+    DeliveryMode.summary_digests,
+    )
+
+
+
+class DigestMemberRoster(AbstractRoster):
+    """Return all the regular delivery members of a list."""
+
+    name = 'regular_members'
 
-    def create(self, email_address, real_name=None):
-        """See IRoster"""
-        from Mailman.database.model.address import Address
-        addr = Address.get_by(address=email_address)
-        if addr:
-            raise ExistingAddressError(email_address)
-        addr = Address(address=email_address, real_name=real_name)
-        # Make sure all the expected links are made, including to the null
-        # (i.e. everyone) roster.
-        self.addresses.append(addr)
-        addr.rosters.append(self)
-        null_roster = Roster.get_by(name='')
-        null_roster.addresses.append(addr)
-        addr.rosters.append(null_roster)
-        return addr
+    @property
+    def members(self):
+        # Query for all the Members which have a role of MemberRole.member and
+        # are subscribed to this mailing list.  Then return only those members
+        # that have one of the digest delivery modes.
+        for member in Member.select_by(mailing_list=self._mlist.fqdn_listname,
+                                       role=MemberRole.member):
+            if member.preferences.delivery_mode in _digest_modes:
+                yield member
diff --git a/Mailman/database/model/user.py b/Mailman/database/model/user.py
index be634b9df..d646606a9 100644
--- a/Mailman/database/model/user.py
+++ b/Mailman/database/model/user.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2006-2007 by the Free Software Foundation, Inc.
+# Copyright (C) 2007 by the Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
@@ -23,15 +23,19 @@ from Mailman import Errors
 from Mailman.database.model import Address
 from Mailman.interfaces import IUser
 
+ADDRESS_KIND    = 'Mailman.database.model.address.Address'
 
+
+
 class User(Entity):
     implements(IUser)
 
     has_field('real_name',  Unicode)
     has_field('password',   Unicode)
     # Relationships
-    has_one('profile', of_kind='Mailman.database.model.profile.Profile')
-    has_many('addresses', of_kind='Mailman.database.model.address.Address')
+    has_many('addresses', of_kind=ADDRESS_KIND)
+    # Options
+    using_options(shortnames=True)
 
     def link(self, address):
         if address.user is not None:
diff --git a/Mailman/database/model/version.py b/Mailman/database/model/version.py
index e22e8ae11..7b12778ce 100644
--- a/Mailman/database/model/version.py
+++ b/Mailman/database/model/version.py
@@ -19,7 +19,7 @@ from elixir import *
 
 
 class Version(Entity):
-    with_fields(
-        component   = Field(String),
-        version     = Field(Integer),
-        )
+    has_field('component',  Unicode)
+    has_field('version',    Integer)
+    # Options
+    using_options(shortnames=True)
diff --git a/Mailman/database/usermanager.py b/Mailman/database/usermanager.py
index 97a740803..ed0b552a8 100644
--- a/Mailman/database/usermanager.py
+++ b/Mailman/database/usermanager.py
@@ -35,47 +35,18 @@ from Mailman.interfaces import IUserManager
 class UserManager(object):
     implements(IUserManager)
 
-    def __init__(self):
-        # Create the null roster if it does not already exist.  It's more
-        # likely to exist than not so try to get it before creating it.
-        lockfile = os.path.join(config.LOCK_DIR, '<umgrcreatelock>')
-        with LockFile(lockfile):
-            roster = self.get_roster('')
-            if roster is None:
-                self.create_roster('')
-        objectstore.flush()
-
-    def create_roster(self, name):
-        roster = Roster.get_by(name=name)
-        if roster:
-            raise Errors.RosterExistsError(name)
-        return Roster(name=name)
-
-    def get_roster(self, name):
-        return Roster.get_by(name=name)
-
-    def delete_roster(self, roster):
-        roster.delete()
-
-    @property
-    def rosters(self):
-        for roster in Roster.select():
-            yield roster
-
-    def create_rosterset(self, name):
-        return RosterSet(name=name)
-
-    def delete_rosterset(self, rosterset):
-        rosterset.delete()
-
-    def get_rosterset(self, name):
-        return RosterSet.get_by(name=name)
-
-    def create_user(self):
+    def create_user(self, address=None, real_name=None):
         user = User()
-        # Users always have a profile
-        user.profile = Profile()
-        user.profile.user = user
+        # Users always have preferences
+        user.preferences = Preferences()
+        user.preferences.user = user
+        if real_name:
+            user.real_name = real_name
+        if address:
+            kws = dict(address=address)
+            if real_name:
+                kws['real_name'] = real_name
+            user.link(Address(**kws))
         return user
 
     def delete_user(self, user):