merge trunk and fix merge conflicts

2 files changed, 117 insertions, 486 deletions

diff --git a/src/mailman/app/docs/chains.rst b/src/mailman/app/docs/chains.rst
deleted file mode 100644
index 328d0b624..000000000
--- a/src/mailman/app/docs/chains.rst
+++ /dev/null
@@ -1,341 +0,0 @@
-======
-Chains
-======
-
-When a new message is posted to a mailing list, Mailman uses a set of rule
-chains to decide whether the message gets accepted for posting, rejected,
-discarded, or held for moderator approval.
-
-There are a number of built-in chains available that act as end-points in the
-processing of messages.
-
-
-The Discard chain
-=================
-
-The `discard` chain simply throws the message away.
-::
-
-    >>> chain = config.chains['discard']
-    >>> print(chain.name)
-    discard
-    >>> print(chain.description)
-    Discard a message and stop processing.
-
-    >>> mlist = create_list('test@example.com')
-    >>> msg = message_from_string("""\
-    ... From: aperson@example.com
-    ... To: test@example.com
-    ... Subject: My first post
-    ... Message-ID: <first>
-    ...
-    ... An important message.
-    ... """)
-
-    >>> def print_msgid(event):
-    ...     print('{0}: {1}'.format(
-    ...         event.chain.name.upper(), event.msg.get('message-id', 'n/a')))
-
-    >>> from mailman.core.chains import process
-    >>> from mailman.testing.helpers import event_subscribers
-
-    >>> with event_subscribers(print_msgid):
-    ...     process(mlist, msg, {}, 'discard')
-    DISCARD: <first>
-
-
-The Reject chain
-================
-
-The `reject` chain bounces the message back to the original sender, and logs
-this action.
-::
-
-    >>> chain = config.chains['reject']
-    >>> print(chain.name)
-    reject
-    >>> print(chain.description)
-    Reject/bounce a message and stop processing.
-
-    >>> with event_subscribers(print_msgid):
-    ...     process(mlist, msg, {}, 'reject')
-    REJECT: <first>
-
-The bounce message is now sitting in the `virgin` queue.
-
-    >>> from mailman.testing.helpers import get_queue_messages
-    >>> qfiles = get_queue_messages('virgin')
-    >>> len(qfiles)
-    1
-    >>> print(qfiles[0].msg.as_string())
-    Subject: My first post
-    From: test-owner@example.com
-    To: aperson@example.com
-    ...
-    [No bounce details are available]
-    ...
-    Content-Type: message/rfc822
-    MIME-Version: 1.0
-    <BLANKLINE>
-    From: aperson@example.com
-    To: test@example.com
-    Subject: My first post
-    Message-ID: <first>
-    <BLANKLINE>
-    An important message.
-    <BLANKLINE>
-    ...
-
-
-The Hold Chain
-==============
-
-The `hold` chain places the message into the administrative request database
-and depending on the list's settings, sends a notification to both the
-original sender and the list moderators.  ::
-
-    >>> chain = config.chains['hold']
-    >>> print(chain.name)
-    hold
-    >>> print(chain.description)
-    Hold a message and stop processing.
-
-    >>> with event_subscribers(print_msgid):
-    ...     process(mlist, msg, {}, 'hold')
-    HOLD: <first>
-
-There are now two messages in the virgin queue, one to the list moderators and
-one to the original author.
-
-    >>> qfiles = get_queue_messages('virgin', sort_on='to')
-    >>> len(qfiles)
-    2
-
-One of the message is addressed to the mailing list moderators, and the other
-is addressed to the original sender.
-
-    >>> from operator import itemgetter
-    >>> messages = sorted((item.msg for item in qfiles),
-    ...                   key=itemgetter('to'), reverse=True)
-
-This one is addressed to the list moderators.
-
-    >>> print(messages[0].as_string())
-    Subject: test@example.com post from aperson@example.com requires approval
-    From: test-owner@example.com
-    To: test-owner@example.com
-    MIME-Version: 1.0
-    ...
-    As list administrator, your authorization is requested for the
-    following mailing list posting:
-    <BLANKLINE>
-        List:    test@example.com
-        From:    aperson@example.com
-        Subject: My first post
-        Reason:  N/A
-    <BLANKLINE>
-    At your convenience, visit your dashboard to approve or deny the
-    request.
-    <BLANKLINE>
-    ...
-    Content-Type: message/rfc822
-    MIME-Version: 1.0
-    <BLANKLINE>
-    From: aperson@example.com
-    To: test@example.com
-    Subject: My first post
-    Message-ID: <first>
-    X-Message-ID-Hash: RXJU4JL6N2OUN3OYMXXPPSCR7P7JE2BW
-    <BLANKLINE>
-    An important message.
-    <BLANKLINE>
-    ...
-    Content-Type: message/rfc822
-    MIME-Version: 1.0
-    <BLANKLINE>
-    Content-Type: text/plain; charset="us-ascii"
-    MIME-Version: 1.0
-    Content-Transfer-Encoding: 7bit
-    Subject: confirm ...
-    From: test-request@example.com
-    ...
-    <BLANKLINE>
-    If you reply to this message, keeping the Subject: header intact,
-    Mailman will discard the held message.  Do this if the message is
-    spam.  If you reply to this message and include an Approved: header
-    with the list password in it, the message will be approved for posting
-    to the list.  The Approved: header can also appear in the first line
-    of the body of the reply.
-    ...
-
-This message is addressed to the sender of the message.
-
-    >>> print(messages[1].as_string())
-    MIME-Version: 1.0
-    Content-Type: text/plain; charset="us-ascii"
-    Content-Transfer-Encoding: 7bit
-    Subject: Your message to test@example.com awaits moderator approval
-    From: test-bounces@example.com
-    To: aperson@example.com
-    ...
-    Your mail to 'test@example.com' with the subject
-    <BLANKLINE>
-        My first post
-    <BLANKLINE>
-    Is being held until the list moderator can review it for approval.
-    <BLANKLINE>
-    The reason it is being held:
-    <BLANKLINE>
-        N/A
-    <BLANKLINE>
-    Either the message will get posted to the list, or you will receive
-    notification of the moderator's decision.  If you would like to cancel
-    this posting, please visit the following URL:
-    <BLANKLINE>
-        http://lists.example.com/confirm/test@example.com/...
-    <BLANKLINE>
-    <BLANKLINE>
-
-In addition, the pending database is holding the original messages, waiting
-for them to be disposed of by the original author or the list moderators.  The
-database is essentially a dictionary, with the keys being the randomly
-selected tokens included in the urls and the values being a 2-tuple where the
-first item is a type code and the second item is a message id.
-::
-
-    >>> import re
-    >>> cookie = None
-    >>> for line in messages[1].get_payload().splitlines():
-    ...     mo = re.search('confirm/[^/]+/(?P<cookie>.*)$', line)
-    ...     if mo:
-    ...         cookie = mo.group('cookie')
-    ...         break
-    >>> assert cookie is not None, 'No confirmation token found'
-
-    >>> from mailman.interfaces.pending import IPendings
-    >>> from zope.component import getUtility
-
-    >>> data = getUtility(IPendings).confirm(cookie)
-    >>> dump_msgdata(data)
-    id  : 1
-    type: held message
-
-The message itself is held in the message store.
-::
-
-    >>> from mailman.interfaces.requests import IListRequests
-    >>> list_requests = IListRequests(mlist)
-    >>> rkey, rdata = list_requests.get_request(data['id'])
-
-    >>> from mailman.interfaces.messages import IMessageStore
-    >>> from zope.component import getUtility
-    >>> msg = getUtility(IMessageStore).get_message_by_id(
-    ...     rdata['_mod_message_id'])
-
-    >>> print(msg.as_string())
-    From: aperson@example.com
-    To: test@example.com
-    Subject: My first post
-    Message-ID: <first>
-    X-Message-ID-Hash: RXJU4JL6N2OUN3OYMXXPPSCR7P7JE2BW
-    <BLANKLINE>
-    An important message.
-    <BLANKLINE>
-
-
-The Accept chain
-================
-
-The `accept` chain sends the message on the `pipeline` queue, where it will be
-processed and sent on to the list membership.
-::
-
-    >>> chain = config.chains['accept']
-    >>> print(chain.name)
-    accept
-    >>> print(chain.description)
-    Accept a message.
-
-    >>> with event_subscribers(print_msgid):
-    ...     process(mlist, msg, {}, 'accept')
-    ACCEPT: <first>
-
-    >>> qfiles = get_queue_messages('pipeline')
-    >>> len(qfiles)
-    1
-    >>> print(qfiles[0].msg.as_string())
-    From: aperson@example.com
-    To: test@example.com
-    Subject: My first post
-    Message-ID: <first>
-    X-Message-ID-Hash: RXJU4JL6N2OUN3OYMXXPPSCR7P7JE2BW
-    <BLANKLINE>
-    An important message.
-    <BLANKLINE>
-
-
-Run-time chains
-===============
-
-We can also define chains at run time, and these chains can be mutated.
-Run-time chains are made up of links where each link associates both a rule
-and a `jump`.  The rule is really a rule name, which is looked up when
-needed.  The jump names a chain which is jumped to if the rule matches.
-
-There is one built-in posting chain.  This is the default chain to use when no
-other input chain is defined for a mailing list.  It runs through the default
-rules.
-
-    >>> chain = config.chains['default-posting-chain']
-    >>> print(chain.name)
-    default-posting-chain
-    >>> print(chain.description)
-    The built-in moderation chain.
-
-Once the sender is a member of the mailing list, the previously created
-message is innocuous enough that it should pass through all default rules.
-This message will end up in the `pipeline` queue.
-::
-
-    >>> from mailman.testing.helpers import subscribe
-    >>> subscribe(mlist, 'Anne')
-    <Member: aperson@example.com on test@example.com as MemberRole.member>
-
-    >>> with event_subscribers(print_msgid):
-    ...     process(mlist, msg, {})
-    ACCEPT: <first>
-
-    >>> qfiles = get_queue_messages('pipeline')
-    >>> len(qfiles)
-    1
-    >>> print(qfiles[0].msg.as_string())
-    From: aperson@example.com
-    To: test@example.com
-    Subject: My first post
-    Message-ID: <first>
-    X-Message-ID-Hash: RXJU4JL6N2OUN3OYMXXPPSCR7P7JE2BW
-    X-Mailman-Rule-Misses: approved; emergency; loop; member-moderation;
-        administrivia; implicit-dest; max-recipients; max-size;
-        news-moderation; no-subject; suspicious-header; nonmember-moderation
-    <BLANKLINE>
-    An important message.
-    <BLANKLINE>
-
-In addition, the message metadata now contains lists of all rules that have
-hit and all rules that have missed.
-
-    >>> dump_list(qfiles[0].msgdata['rule_hits'])
-    *Empty*
-    >>> dump_list(qfiles[0].msgdata['rule_misses'])
-    administrivia
-    approved
-    emergency
-    implicit-dest
-    loop
-    max-recipients
-    max-size
-    member-moderation
-    news-moderation
-    no-subject
-    nonmember-moderation
-    suspicious-header
diff --git a/src/mailman/app/docs/subscriptions.rst b/src/mailman/app/docs/subscriptions.rst
index eaccdc3cc..2fc59d9a7 100644
--- a/src/mailman/app/docs/subscriptions.rst
+++ b/src/mailman/app/docs/subscriptions.rst
@@ -2,9 +2,10 @@
 Subscription services
 =====================
 
-The `ISubscriptionService` utility provides higher level convenience methods
-useful for searching, retrieving, iterating, adding, and removing
-memberships.
+The ``ISubscriptionService`` utility provides higher level convenience methods
+useful for searching, retrieving, iterating, and removing memberships across
+all mailing lists on th esystem.  Adding new users is handled by the
+``IRegistrar`` interface.
 
     >>> from mailman.interfaces.subscriptions import ISubscriptionService
     >>> from zope.component import getUtility
@@ -22,183 +23,154 @@ membership role.  At first, there are no memberships.
     None
 
 
-Adding new members
-==================
-
-The service can be used to subscribe new members, by default with the `member`
-role.  At a minimum, a mailing list and an address for the new user is
-required.
-
-    >>> mlist = create_list('test@example.com')
-    >>> anne = service.join('test.example.com', 'anne@example.com')
-    >>> anne
-    <Member: anne <anne@example.com> on test@example.com as MemberRole.member>
+Listing members
+===============
 
-The real name of the new member can be given.
+When there are some members, of any role on any mailing list, they can be
+retrieved through the subscription service.
 
-    >>> bart = service.join('test.example.com', 'bart@example.com',
-    ...                     'Bart Person')
-    >>> bart
-    <Member: Bart Person <bart@example.com>
-             on test@example.com as MemberRole.member>
+    >>> from mailman.app.lifecycle import create_list
+    >>> ant = create_list('ant@example.com')
+    >>> bee = create_list('bee@example.com')
+    >>> cat = create_list('cat@example.com')
 
-Other roles can also be subscribed.
+Some people become members.
 
     >>> from mailman.interfaces.member import MemberRole
-    >>> anne_owner = service.join('test.example.com', 'anne@example.com',
-    ...                           role=MemberRole.owner)
-    >>> anne_owner
-    <Member: anne <anne@example.com> on test@example.com as MemberRole.owner>
+    >>> from mailman.testing.helpers import subscribe
+    >>> anne_1 = subscribe(ant, 'Anne')
+    >>> anne_2 = subscribe(ant, 'Anne', MemberRole.owner)
+    >>> bart_1 = subscribe(ant, 'Bart', MemberRole.moderator)
+    >>> bart_2 = subscribe(bee, 'Bart', MemberRole.owner)
+    >>> anne_3 = subscribe(cat, 'Anne', email='anne@example.com')
+    >>> cris_1 = subscribe(cat, 'Cris')
 
-And all the subscribed members can now be displayed.
+The service can be used to iterate over them.
 
-    >>> service.get_members()
-    [<Member: anne <anne@example.com> on test@example.com as MemberRole.owner>,
-     <Member: anne <anne@example.com> on test@example.com
-              as MemberRole.member>,
-     <Member: Bart Person <bart@example.com> on test@example.com
-              as MemberRole.member>]
-    >>> sum(1 for member in service)
-    3
-    >>> print(service.get_member(UUID(int=3)))
-    <Member: anne <anne@example.com> on test@example.com as MemberRole.owner>
+    >>> for member in service.get_members():
+    ...     print(member)
+    <Member: Anne Person <aperson@example.com>
+        on ant@example.com as MemberRole.owner>
+    <Member: Bart Person <bperson@example.com>
+        on ant@example.com as MemberRole.moderator>
+    <Member: Anne Person <aperson@example.com>
+        on ant@example.com as MemberRole.member>
+    <Member: Bart Person <bperson@example.com>
+        on bee@example.com as MemberRole.owner>
+    <Member: Anne Person <anne@example.com>
+        on cat@example.com as MemberRole.member>
+    <Member: Cris Person <cperson@example.com>
+        on cat@example.com as MemberRole.member>
 
-New members can also be added by providing an existing user id instead of an
-email address.  However, the user must have a preferred email address.
-::
+The service can also be used to get the information about a single member.
 
-    >>> from mailman.utilities.datetime import now
-    >>> address = list(bart.user.addresses)[0]
-    >>> address.verified_on = now()
-    >>> bart.user.preferred_address = address
-    >>> service.join('test.example.com', bart.user.user_id,
-    ...              role=MemberRole.owner)
-    <Member: Bart Person <bart@example.com>
-             on test@example.com as MemberRole.owner>
+    >>> print(service.get_member(bart_2.member_id))
+    <Member: Bart Person <bperson@example.com>
+        on bee@example.com as MemberRole.owner>
 
+There is an iteration shorthand for getting all the members.
 
-Removing members
-================
-
-Regular members can also be removed.
-
-    >>> cris = service.join('test.example.com', 'cris@example.com')
-    >>> service.get_members()
-    [<Member: anne <anne@example.com> on test@example.com
-              as MemberRole.owner>,
-     <Member: Bart Person <bart@example.com> on test@example.com
-              as MemberRole.owner>,
-     <Member: anne <anne@example.com> on test@example.com
-              as MemberRole.member>,
-     <Member: Bart Person <bart@example.com> on test@example.com
-              as MemberRole.member>,
-     <Member: cris <cris@example.com> on test@example.com
-              as MemberRole.member>]
-    >>> sum(1 for member in service)
-    5
-    >>> service.leave('test.example.com', 'cris@example.com')
-    >>> service.get_members()
-    [<Member: anne <anne@example.com> on test@example.com
-              as MemberRole.owner>,
-     <Member: Bart Person <bart@example.com> on test@example.com
-              as MemberRole.owner>,
-     <Member: anne <anne@example.com> on test@example.com
-              as MemberRole.member>,
-     <Member: Bart Person <bart@example.com> on test@example.com
-              as MemberRole.member>]
-    >>> sum(1 for member in service)
-    4
+    >>> for member in service:
+    ...     print(member)
+    <Member: Anne Person <aperson@example.com>
+        on ant@example.com as MemberRole.owner>
+    <Member: Bart Person <bperson@example.com>
+        on ant@example.com as MemberRole.moderator>
+    <Member: Anne Person <aperson@example.com>
+        on ant@example.com as MemberRole.member>
+    <Member: Bart Person <bperson@example.com>
+        on bee@example.com as MemberRole.owner>
+    <Member: Anne Person <anne@example.com>
+        on cat@example.com as MemberRole.member>
+    <Member: Cris Person <cperson@example.com>
+        on cat@example.com as MemberRole.member>
 
 
 Finding members
 ===============
 
-If you know the member id for a specific member, you can get that member.
-
-    >>> service.get_member(UUID(int=3))
-    <Member: anne <anne@example.com> on test@example.com as MemberRole.owner>
-
-If you know the member's address, you can find all their memberships, based on
-specific search criteria.  We start by subscribing Anne to a couple of new
-mailing lists.
+The subscription service can be used to find memberships based on specific
+search criteria.  For example, we can find all the mailing lists that Anne is
+a member of with her ``aperson@example.com`` address.
 
-    >>> mlist2 = create_list('foo@example.com')
-    >>> mlist3 = create_list('bar@example.com')
-    >>> address = list(anne.user.addresses)[0]
-    >>> address.verified_on = now()
-    >>> anne.user.preferred_address = address
-    >>> mlist.subscribe(anne.user, MemberRole.moderator)
-    <Member: anne <anne@example.com> on test@example.com
-             as MemberRole.moderator>
-    >>> mlist2.subscribe(anne.user, MemberRole.member)
-    <Member: anne <anne@example.com> on foo@example.com as MemberRole.member>
-    >>> mlist3.subscribe(anne.user, MemberRole.owner)
-    <Member: anne <anne@example.com> on bar@example.com as MemberRole.owner>
-
-And now we can find all of Anne's memberships.
-
-    >>> service.find_members('anne@example.com')
-    [<Member: anne <anne@example.com> on bar@example.com as MemberRole.owner>,
-     <Member: anne <anne@example.com> on foo@example.com as MemberRole.member>,
-     <Member: anne <anne@example.com> on test@example.com
-              as MemberRole.member>,
-     <Member: anne <anne@example.com> on test@example.com
-              as MemberRole.owner>,
-     <Member: anne <anne@example.com> on test@example.com
-              as MemberRole.moderator>]
+    >>> for member in service.find_members('aperson@example.com'):
+    ...     print(member)
+    <Member: Anne Person <aperson@example.com>
+        on ant@example.com as MemberRole.member>
+    <Member: Anne Person <aperson@example.com>
+        on ant@example.com as MemberRole.owner>
 
 There may be no matching memberships.
 
-    >>> service.find_members('cris@example.com')
+    >>> service.find_members('dave@example.com')
     []
 
 Memberships can also be searched for by user id.
 
-    >>> service.find_members(UUID(int=1))
-    [<Member: anne <anne@example.com> on bar@example.com as MemberRole.owner>,
-     <Member: anne <anne@example.com> on foo@example.com as MemberRole.member>,
-     <Member: anne <anne@example.com> on test@example.com
-              as MemberRole.member>,
-     <Member: anne <anne@example.com> on test@example.com
-              as MemberRole.owner>,
-     <Member: anne <anne@example.com> on test@example.com
-              as MemberRole.moderator>]
+    >>> for member in service.find_members(anne_1.user.user_id):
+    ...     print(member)
+    <Member: Anne Person <aperson@example.com>
+        on ant@example.com as MemberRole.member>
+    <Member: Anne Person <aperson@example.com>
+        on ant@example.com as MemberRole.owner>
 
 You can find all the memberships for a specific mailing list.
 
-    >>> service.find_members(list_id='test.example.com')
-    [<Member: anne <anne@example.com> on test@example.com
-              as MemberRole.member>,
-     <Member: anne <anne@example.com> on test@example.com as MemberRole.owner>,
-     <Member: anne <anne@example.com> on test@example.com
-              as MemberRole.moderator>,
-     <Member: Bart Person <bart@example.com> on test@example.com
-              as MemberRole.member>,
-     <Member: Bart Person <bart@example.com> on test@example.com
-              as MemberRole.owner>]
+    >>> for member in service.find_members(list_id='ant.example.com'):
+    ...     print(member)
+    <Member: Anne Person <aperson@example.com>
+        on ant@example.com as MemberRole.member>
+    <Member: Anne Person <aperson@example.com>
+        on ant@example.com as MemberRole.owner>
+    <Member: Bart Person <bperson@example.com>
+        on ant@example.com as MemberRole.moderator>
 
 You can find all the memberships for an address on a specific mailing list,
 but you have to give it the list id, not the fqdn listname since the former is
 stable but the latter could change if the list is moved.
 
-    >>> service.find_members('anne@example.com', 'test.example.com')
-    [<Member: anne <anne@example.com> on test@example.com
-              as MemberRole.member>,
-     <Member: anne <anne@example.com> on test@example.com
-              as MemberRole.owner>,
-     <Member: anne <anne@example.com> on test@example.com
-              as MemberRole.moderator>]
+    >>> for member in service.find_members(
+    ...         'bperson@example.com', 'ant.example.com'):
+    ...     print(member)
+    <Member: Bart Person <bperson@example.com>
+        on ant@example.com as MemberRole.moderator>
 
 You can find all the memberships for an address with a specific role.
 
-    >>> service.find_members('anne@example.com', role=MemberRole.owner)
-    [<Member: anne <anne@example.com> on bar@example.com as MemberRole.owner>,
-     <Member: anne <anne@example.com> on test@example.com
-              as MemberRole.owner>]
+    >>> for member in service.find_members(
+    ...         list_id='ant.example.com', role=MemberRole.owner):
+    ...     print(member)
+    <Member: Anne Person <aperson@example.com>
+        on ant@example.com as MemberRole.owner>
 
 You can also find a specific membership by all three criteria.
 
-    >>> service.find_members('anne@example.com', 'test.example.com',
-    ...                      MemberRole.owner)
-    [<Member: anne <anne@example.com> on test@example.com
-              as MemberRole.owner>]
+    >>> for member in service.find_members(
+    ...         'bperson@example.com', 'bee.example.com', MemberRole.owner):
+    ...     print(member)
+    <Member: Bart Person <bperson@example.com>
+        on bee@example.com as MemberRole.owner>
+
+
+Removing members
+================
+
+Members can be removed via this service.
+
+    >>> len(service.get_members())
+    6
+    >>> service.leave('cat.example.com', 'cperson@example.com')
+    >>> len(service.get_members())
+    5
+    >>> for member in service:
+    ...     print(member)
+    <Member: Anne Person <aperson@example.com>
+        on ant@example.com as MemberRole.owner>
+    <Member: Bart Person <bperson@example.com>
+        on ant@example.com as MemberRole.moderator>
+    <Member: Anne Person <aperson@example.com>
+        on ant@example.com as MemberRole.member>
+    <Member: Bart Person <bperson@example.com>
+        on bee@example.com as MemberRole.owner>
+    <Member: Anne Person <anne@example.com>
+        on cat@example.com as MemberRole.member>