6 files changed, 176 insertions, 156 deletions

diff --git a/src/mailman/interfaces/address.py b/src/mailman/interfaces/address.py
index d54ea64c3..89a4c5b3b 100644
--- a/src/mailman/interfaces/address.py
+++ b/src/mailman/interfaces/address.py
@@ -84,7 +84,7 @@ class IAddress(Interface):
 
         Registeration is really the date at which this address became known to
         us.  It may have been explicitly registered by a user, or it may have
-        been implicitly registered, e.g. by showing up in a non-member
+        been implicitly registered, e.g. by showing up in a nonmember
         posting.""")
 
     verified_on = Attribute(
diff --git a/src/mailman/model/docs/membership.txt b/src/mailman/model/docs/membership.txt
index 28d14f149..ebf1b6ebf 100644
--- a/src/mailman/model/docs/membership.txt
+++ b/src/mailman/model/docs/membership.txt
@@ -15,168 +15,105 @@ roster set's name.  Roster also have names, but are related to roster sets
 by a more direct containment relationship.  This is because it is possible to
 store mailing list data in a different database than user data.
 
-When we create a mailing list, it starts out with no members.
-::
+When we create a mailing list, it starts out with no members, owners,
+moderators, administrators, or nonmembers.
 
     >>> mlist = create_list('test@example.com')
-
-    >>> def print_addresses(roster_addresses):
-    ...     sorted_addresses = sorted(
-    ...         address.address for address in roster_addresses)
-    ...     if len(sorted_addresses) == 0:
-    ...         print 'No addresses'
-    ...     else:
-    ...         for address in sorted_addresses:
-    ...             print address
-
-    >>> print_addresses(mlist.members.members)
-    No addresses
-    >>> sorted(user.real_name for user in mlist.members.users)
-    []
-    >>> sorted(address.address for member in mlist.members.addresses)
-    []
-
-...no owners...
-
-    >>> print_addresses(mlist.owners.members)
-    No addresses
-    >>> sorted(user.real_name for user in mlist.owners.users)
-    []
-    >>> sorted(address.address for member in mlist.owners.addresses)
-    []
-
-...no moderators...
-
-    >>> print_addresses(mlist.moderators.members)
-    No addresses
-    >>> sorted(user.real_name for user in mlist.moderators.users)
-    []
-    >>> sorted(address.address for member in mlist.moderators.addresses)
-    []
-
-...and no administrators...
-
-    >>> print_addresses(mlist.administrators.members)
-    No addresses
-    >>> sorted(user.real_name for user in mlist.administrators.users)
-    []
-    >>> sorted(address.address for member in mlist.administrators.addresses)
-    []
-
-...and no nonmembers.
-
-    >>> print_addresses(mlist.nonmembers.members)
-    No addresses
-    >>> sorted(user.real_name for user in mlist.nonmembers.users)
-    []
-    >>> sorted(address.address for member in mlist.nonmembers.addresses)
-    []
+    >>> dump_list(mlist.members.members)
+    *Empty*
+    >>> dump_list(mlist.owners.members)
+    *Empty*
+    >>> dump_list(mlist.moderators.members)
+    *Empty*
+    >>> dump_list(mlist.administrators.members)
+    *Empty*
+    >>> dump_list(mlist.nonmembers.members)
+    *Empty*
 
 
 Administrators
 ==============
 
 A mailing list's administrators are defined as union of the list's owners and
-the list's moderators.  We can add new owners or moderators to this list by
-assigning roles to users.  First we have to create the user, because there are
-no users in the user database yet.
+moderators.  We can add new owners or moderators to this list by assigning
+roles to users.  First we have to create the user, because there are no users
+in the user database yet.
 
     >>> from mailman.interfaces.usermanager import IUserManager
     >>> from zope.component import getUtility
     >>> user_manager = getUtility(IUserManager)
     >>> user_1 = user_manager.create_user('aperson@example.com', 'Anne Person')
-    >>> print user_1.real_name
-    Anne Person
-    >>> print_addresses(user_1.addresses)
-    aperson@example.com
+    >>> print user_1
+    <User "Anne Person" at ...>
 
 We can add Anne as an owner of the mailing list, by creating a member role for
 her.
 
     >>> from mailman.interfaces.member import MemberRole
     >>> address_1 = list(user_1.addresses)[0]
-    >>> print address_1.address
-    aperson@example.com
     >>> address_1.subscribe(mlist, MemberRole.owner)
     <Member: Anne Person <aperson@example.com> on
              test@example.com as MemberRole.owner>
-    >>> print_addresses(mlist.owners.members)
+    >>> dump_list(member.address for member in mlist.owners.members)
     Anne Person <aperson@example.com>
-    >>> sorted(user.real_name for user in mlist.owners.users)
-    [u'Anne Person']
-    >>> print_addresses(mlist.owners.addresses)
-    aperson@example.com
 
 Adding Anne as a list owner also makes her an administrator, but does not make
 her a moderator.  Nor does it make her a member of the list.
 
-    >>> sorted(user.real_name for user in mlist.administrators.users)
-    [u'Anne Person']
-    >>> sorted(user.real_name for user in mlist.moderators.users)
-    []
-    >>> sorted(user.real_name for user in mlist.members.users)
-    []
+    >>> dump_list(member.address for member in mlist.administrators.members)
+    Anne Person <aperson@example.com>
+    >>> dump_list(member.address for member in mlist.moderators.members)
+    *Empty*
+    >>> dump_list(member.address for member in mlist.members.members)
+    *Empty*
 
-We can add Bart as a moderator of the list, by creating a different member role
-for him.
+Bart becomes a moderator of the list.
 
     >>> user_2 = user_manager.create_user('bperson@example.com', 'Bart Person')
-    >>> print user_2.real_name
-    Bart Person
+    >>> print user_2
+    <User "Bart Person" at ...>
     >>> address_2 = list(user_2.addresses)[0]
-    >>> print address_2.address
-    bperson@example.com
     >>> address_2.subscribe(mlist, MemberRole.moderator)
     <Member: Bart Person <bperson@example.com>
              on test@example.com as MemberRole.moderator>
-    >>> print_addresses(mlist.moderators.members)
+    >>> dump_list(member.address for member in mlist.moderators.members)
     Bart Person <bperson@example.com>
-    >>> sorted(user.real_name for user in mlist.moderators.users)
-    [u'Bart Person']
-    >>> print_addresses(mlist.moderators.addresses)
-    bperson@example.com
 
 Now, both Anne and Bart are list administrators.
+::
+
+    >>> from operator import attrgetter
+    >>> def dump_members(roster):
+    ...     all_addresses = list(member.address for member in roster)
+    ...     sorted_addresses = sorted(all_addresses, key=attrgetter('address'))
+    ...     dump_list(sorted_addresses)
 
-    >>> print_addresses(mlist.administrators.members)
+    >>> dump_members(mlist.administrators.members)
     Anne Person <aperson@example.com>
     Bart Person <bperson@example.com>
-    >>> sorted(user.real_name for user in mlist.administrators.users)
-    [u'Anne Person', u'Bart Person']
-    >>> print_addresses(mlist.administrators.addresses)
-    aperson@example.com
-    bperson@example.com
 
 
 Members
 =======
 
-Similarly, list members are born of users being given the proper role.  It's
-more interesting here because these roles should have a preference which can
-be used to decide whether the member is to get regular delivery or digest
-delivery.  Without a preference, Mailman will fall back first to the address's
-preference, then the user's preference, then the list's preference.  Start
-without any member preference to see the system defaults.
+Similarly, list members are born of users being subscribed with the proper
+role.
 
     >>> user_3 = user_manager.create_user(
     ...     'cperson@example.com', 'Cris Person')
-    >>> print user_3.real_name
-    Cris Person
     >>> address_3 = list(user_3.addresses)[0]
-    >>> print address_3.address
-    cperson@example.com
     >>> address_3.subscribe(mlist, MemberRole.member)
     <Member: Cris Person <cperson@example.com>
              on test@example.com as MemberRole.member>
 
 Cris will be a regular delivery member but not a digest member.
 
-    >>> print_addresses(mlist.members.addresses)
-    cperson@example.com
-    >>> print_addresses(mlist.regular_members.addresses)
-    cperson@example.com
-    >>> print_addresses(mlist.digest_members.addresses)
-    No addresses
+    >>> dump_members(mlist.members.members)
+    Cris Person <cperson@example.com>
+    >>> dump_members(mlist.regular_members.members)
+    Cris Person <cperson@example.com>
+    >>> dump_members(mlist.digest_members.addresses)
+    *Empty*
 
 It's easy to make the list administrators members of the mailing list too.
 
@@ -184,21 +121,21 @@ It's easy to make the list administrators members of the mailing list too.
     >>> for address in mlist.administrators.addresses:
     ...     member = address.subscribe(mlist, MemberRole.member)
     ...     members.append(member)
-    >>> sorted(members, key=lambda m: m.address.address)
-    [<Member: Anne Person <aperson@example.com> on
-              test@example.com as MemberRole.member>,
-     <Member: Bart Person <bperson@example.com> on
-              test@example.com as MemberRole.member>]
-    >>> print_addresses(mlist.members.addresses)
-    aperson@example.com
-    bperson@example.com
-    cperson@example.com
-    >>> print_addresses(mlist.regular_members.addresses)
-    aperson@example.com
-    bperson@example.com
-    cperson@example.com
-    >>> print_addresses(mlist.digest_members.addresses)
-    No addresses
+    >>> dump_list(members, key=attrgetter('address.address'))
+    <Member: Anne Person <aperson@example.com> on
+             test@example.com as MemberRole.member>
+    <Member: Bart Person <bperson@example.com> on
+             test@example.com as MemberRole.member>
+    >>> dump_members(mlist.members.members)
+    Anne Person <aperson@example.com>
+    Bart Person <bperson@example.com>
+    Cris Person <cperson@example.com>
+    >>> dump_members(mlist.regular_members.members)
+    Anne Person <aperson@example.com>
+    Bart Person <bperson@example.com>
+    Cris Person <cperson@example.com>
+    >>> dump_members(mlist.digest_members.members)
+    *Empty*
 
 
 Nonmembers
@@ -207,33 +144,32 @@ Nonmembers
 Nonmembers are used to represent people who have posted to the mailing list
 but are not subscribed to the mailing list.  These may be legitimate users who
 have found the mailing list and wish to interact without a direct
-subscription.  It may also be spammers who should never be allowed to contact
+subscription, or they may be spammers who should never be allowed to contact
 the mailing list.  Because all the same moderation rules can be applied to
 nonmembers, we represent them as the same type of object but with a different
 role.
 
-    >>> user_6 = user_manager.create_user('fperson@example.com', 'Fred')
+    >>> user_6 = user_manager.create_user('fperson@example.com', 'Fred Person')
     >>> address_6 = list(user_6.addresses)[0]
     >>> member_6 = address_6.subscribe(mlist, MemberRole.nonmember)
     >>> member_6
-    <Member: Fred <fperson@example.com> on test@example.com
+    <Member: Fred Person <fperson@example.com> on test@example.com
              as MemberRole.nonmember>
-    >>> print_addresses(mlist.nonmembers.addresses)
-    fperson@example.com
+    >>> dump_members(mlist.nonmembers.members)
+    Fred Person <fperson@example.com>
 
-Nonmembers do not get delivery of any messages.  See that Fred does not show
-up in any of the delivery rosters.
+Nonmembers do not get delivery of any messages.
 
-    >>> print_addresses(mlist.members.addresses)
-    aperson@example.com
-    bperson@example.com
-    cperson@example.com
-    >>> print_addresses(mlist.regular_members.addresses)
-    aperson@example.com
-    bperson@example.com
-    cperson@example.com
-    >>> print_addresses(mlist.digest_members.addresses)
-    No addresses
+    >>> dump_members(mlist.members.members)
+    Anne Person <aperson@example.com>
+    Bart Person <bperson@example.com>
+    Cris Person <cperson@example.com>
+    >>> dump_members(mlist.regular_members.members)
+    Anne Person <aperson@example.com>
+    Bart Person <bperson@example.com>
+    Cris Person <cperson@example.com>
+    >>> dump_members(mlist.digest_members.members)
+    *Empty*
 
 
 Finding members
@@ -252,7 +188,7 @@ text email address by using the ``IRoster.get_member()`` method.
     <Member: Anne Person <aperson@example.com> on
              test@example.com as MemberRole.member>
     >>> mlist.nonmembers.get_member('fperson@example.com')
-    <Member: Fred <fperson@example.com> on
+    <Member: Fred Person <fperson@example.com> on
              test@example.com as MemberRole.nonmember>
 
 However, if the address is not subscribed with the appropriate role, then None
@@ -305,11 +241,10 @@ All members of any role have a *moderation action* which specifies how
 postings from that member are handled.  By default, owners and moderators are
 automatically accepted for posting to the mailing list.
 
-    >>> from operator import attrgetter
     >>> for member in sorted(mlist.administrators.members,
     ...                      key=attrgetter('address.address')):
     ...     print member.address.address, member.role, member.moderation_action
-    aperson@example.com MemberRole.owner Action.accept
+    aperson@example.com MemberRole.owner     Action.accept
     bperson@example.com MemberRole.moderator Action.accept
 
 By default, members have a *deferred* action which specifies that the posting
diff --git a/src/mailman/queue/docs/incoming.txt b/src/mailman/queue/docs/incoming.txt
index fd875105c..b0f644098 100644
--- a/src/mailman/queue/docs/incoming.txt
+++ b/src/mailman/queue/docs/incoming.txt
@@ -17,22 +17,40 @@ above.
     built-in
 
 
-Accepted messages
-=================
+Sender addresses
+================
 
-We have a message that is going to be sent to the mailing list.  This message
-is so perfectly fine for posting that it will be accepted and forward to the
-pipeline queue.
+The incoming queue runner ensures that the sender addresses on the message are
+registered with the system.  This is used for determining nonmember posting
+privileges.  The addresses will not be linked to a user and will be
+unverified, so if the real user comes along later and claims the address, it
+will be linked to their user account (and must be verified).
+
+While configurable, the *sender addresses* by default are those named in the
+`From:`, `Sender:`, and `Reply-To:` headers, as well as the envelope sender
+(though we won't worry about the latter).
+::
 
     >>> msg = message_from_string("""\
-    ... From: aperson@example.com
+    ... From: zperson@example.com
+    ... Reply-To: yperson@example.com
+    ... Sender: xperson@example.com
     ... To: test@example.com
-    ... Subject: My first post
-    ... Message-ID: <first>
+    ... Subject: This is spiced ham
+    ... Message-ID: <bogus>
     ...
-    ... First post!
     ... """)
 
+    >>> from zope.component import getUtility
+    >>> from mailman.interfaces.usermanager import IUserManager
+    >>> user_manager = getUtility(IUserManager)
+    >>> print user_manager.get_address('xperson@example.com')
+    None
+    >>> print user_manager.get_address('yperson@example.com')
+    None
+    >>> print user_manager.get_address('zperson@example.com')
+    None
+
 Inject the message into the incoming queue, similar to the way the upstream
 mail server normally would.
 
@@ -46,7 +64,48 @@ The incoming queue runner runs until it is empty.
     >>> incoming = make_testable_runner(IncomingRunner, 'in')
     >>> incoming.run()
 
-And now the message is in the pipeline queue.
+And now the addresses are known to the system.  As mentioned above, they are
+not linked to a user and are unverified.
+
+    >>> for localpart in ('xperson', 'yperson', 'zperson'):
+    ...     email = '{0}@example.com'.format(localpart)
+    ...     address = user_manager.get_address(email)
+    ...     print '{0}; verified? {1}; user? {2}'.format(
+    ...         address.address,
+    ...         ('No' if address.verified_on is None else 'Yes'),
+    ...         user_manager.get_user(email))
+    xperson@example.com; verified? No; user? None
+    yperson@example.com; verified? No; user? None
+    zperson@example.com; verified? No; user? None
+
+..
+    Clear the pipeline queue of artifacts that affect the following tests.
+    >>> from mailman.testing.helpers import get_queue_messages
+    >>> ignore = get_queue_messages('pipeline')
+
+
+Accepted messages
+=================
+
+We have a message that is going to be sent to the mailing list.  This message
+is so perfectly fine for posting that it will be accepted and forward to the
+pipeline queue.
+
+    >>> msg = message_from_string("""\
+    ... From: aperson@example.com
+    ... To: test@example.com
+    ... Subject: My first post
+    ... Message-ID: <first>
+    ...
+    ... First post!
+    ... """)
+
+Inject the message into the incoming queue and run until the queue is empty.
+
+    >>> inject_message(mlist, msg)
+    >>> incoming.run()
+
+Now the message is in the pipeline queue.
 
     >>> pipeline_queue = config.switchboards['pipeline']
     >>> len(pipeline_queue.files)
@@ -54,7 +113,6 @@ And now the message is in the pipeline queue.
     >>> incoming_queue = config.switchboards['in']
     >>> len(incoming_queue.files)
     0
-    >>> from mailman.testing.helpers import get_queue_messages
     >>> item = get_queue_messages('pipeline')[0]
     >>> print item.msg.as_string()
     From: aperson@example.com
diff --git a/src/mailman/queue/incoming.py b/src/mailman/queue/incoming.py
index f91f6c90c..418723814 100644
--- a/src/mailman/queue/incoming.py
+++ b/src/mailman/queue/incoming.py
@@ -34,7 +34,12 @@ __all__ = [
     ]
 
 
+from zope.component import getUtility
+
+from mailman.config import config
 from mailman.core.chains import process
+from mailman.interfaces.address import ExistingAddressError
+from mailman.interfaces.usermanager import IUserManager
 from mailman.queue import Runner
 
 
@@ -46,6 +51,15 @@ class IncomingRunner(Runner):
         """See `IRunner`."""
         if msgdata.get('envsender') is None:
             msgdata['envsender'] = mlist.no_reply_address
+        # Ensure that the email addresses of the message's senders are known
+        # to Mailman.  This will be used in nonmember posting dispositions.
+        user_manager = getUtility(IUserManager)
+        for sender in msg.senders:
+            try:
+                user_manager.create_address(sender)
+            except ExistingAddressError:
+                pass
+        config.db.commit()
         # Process the message through the mailing list's start chain.
         process(mlist, msg, msgdata, mlist.start_chain)
         # Do not keep this message queued.
diff --git a/src/mailman/queue/lmtp.py b/src/mailman/queue/lmtp.py
index dcca201e8..7fcd0c70c 100644
--- a/src/mailman/queue/lmtp.py
+++ b/src/mailman/queue/lmtp.py
@@ -146,7 +146,7 @@ class LMTPRunner(Runner, smtpd.SMTPServer):
 
     def handle_accept(self):
         conn, addr = self.accept()
-        channel = Channel(self, conn, addr)
+        Channel(self, conn, addr)
         qlog.debug('LMTP accept from %s', addr)
 
     @txn
@@ -163,7 +163,7 @@ class LMTPRunner(Runner, smtpd.SMTPServer):
                 return ERR_501
             msg['X-MailFrom'] = mailfrom
             message_id = msg['message-id']
-        except Exception, e:
+        except Exception:
             elog.exception('LMTP message parsing')
             config.db.abort()
             return CRLF.join(ERR_451 for to in rcpttos)
diff --git a/src/mailman/tests/test_documentation.py b/src/mailman/tests/test_documentation.py
index f4b382714..f89979ba2 100644
--- a/src/mailman/tests/test_documentation.py
+++ b/src/mailman/tests/test_documentation.py
@@ -110,6 +110,18 @@ def dump_msgdata(msgdata, *additional_skips):
         print '{0:{2}}: {1}'.format(key, msgdata[key], longest)
 
 
+def dump_list(list_of_things, key=None):
+    """Print items in a string to get rid of stupid u'' prefixes."""
+    # List of things may be a generator.
+    list_of_things = list(list_of_things)
+    if len(list_of_things) == 0:
+        print '*Empty*'
+    if key is not None:
+        list_of_things = sorted(list_of_things, key=key)
+    for item in list_of_things:
+        print item
+
+
 def call_http(url, data=None, method=None, username=None, password=None):
     """'Call' a URL with a given HTTP method and return the resulting object.
 
@@ -202,6 +214,7 @@ def setup(testobj):
     testobj.globs['create_list'] = create_list
     testobj.globs['dump_json'] = dump_json
     testobj.globs['dump_msgdata'] = dump_msgdata
+    testobj.globs['dump_list'] = dump_list
     testobj.globs['message_from_string'] = specialized_message_from_string
     testobj.globs['smtpd'] = SMTPLayer.smtpd
     testobj.globs['stop'] = stop