* Fix a test based on updated output.

* Add a stub for the -confirm email command
* Add stubs for -leave and -unsubscribe
* Remove the crufty (and broken) 'tojoin' 'toleave' and 'toconfirm' metadata
  keys for synchronizing between lmtp and the command runner.  Replace this by
  putting the subaddress recognized by lmtp into the metadata and having the
  command runner look at the subaddress.

4 files changed, 358 insertions, 36 deletions

diff --git a/src/mailman/queue/command.py b/src/mailman/queue/command.py
index 178d2ba0b..f36f9a31d 100644
--- a/src/mailman/queue/command.py
+++ b/src/mailman/queue/command.py
@@ -60,11 +60,12 @@ class CommandFinder:
         # commands.  For example, if this was sent to the -join or -leave
         # addresses, it's the same as if 'join' or 'leave' commands were sent
         # to the -request address.
-        if msgdata.get('tojoin'):
+        subaddress = msgdata.get('subaddress')
+        if subaddress == 'join':
             self.command_lines.append('join')
-        elif msgdata.get('toleave'):
+        elif subaddress == 'leave':
             self.command_lines.append('leave')
-        elif msgdata.get('toconfirm'):
+        elif subaddress == 'confirm':
             mo = re.match(config.mta.verp_confirm_regexp, msg.get('to', ''))
             if mo:
                 self.command_lines.append('confirm ' + mo.group('cookie'))
diff --git a/src/mailman/queue/docs/command.txt b/src/mailman/queue/docs/command.txt
index 88e4387d4..fd0236a8c 100644
--- a/src/mailman/queue/docs/command.txt
+++ b/src/mailman/queue/docs/command.txt
@@ -108,6 +108,105 @@ message is plain text.
     <BLANKLINE>
 
 
+Implicit commands
+=================
+
+For some commands, specifically for joining and leaving a mailing list, there
+are email aliases that act like commands, even when there's nothing else in
+the Subject or body.  For example, to join a mailing list, a user need only
+email the -join address or -subscribe address (the latter is deprecated).
+
+Because Dirk has never registered with Mailman before, he gets two responses.
+The first is a confirmation message so that Dirk can validate his email
+address, and the other is the results of his email command.
+
+    >>> msg = message_from_string("""\
+    ... From: Dirk Person <dperson@example.com>
+    ... To: test-join@example.com
+    ...
+    ... """)
+
+    >>> inject_message(mlist, msg, switchboard='command', subaddress='join')
+    >>> command.run()
+    >>> len(virgin_queue.files)
+    2
+
+    >>> def sortkey(item):
+    ...     return item.msg['subject']
+    >>> messages = sorted(get_queue_messages('virgin'), key=sortkey)
+    >>> for item in messages:
+    ...     print 'Subject:', item.msg['subject']
+    Subject: confirm ...
+    Subject: The results of your email commands
+
+Similarly, to leave a mailing list, the user need only email the -leave or
+-unsubscribe address (the latter is deprecated).
+
+    >>> msg = message_from_string("""\
+    ... From: dperson@example.com
+    ... To: test-leave@example.com
+    ...
+    ... """)
+
+    >>> inject_message(mlist, msg, switchboard='command', subaddress='leave')
+    >>> command.run()
+    >>> len(virgin_queue.files)
+    1
+    >>> item = get_queue_messages('virgin')[0]
+    >>> print item.msg.as_string()
+    Subject: The results of your email commands
+    From: test-bounces@example.com
+    To: dperson@example.com
+    ...
+    <BLANKLINE>
+    The results of your email command are provided below.
+    <BLANKLINE>
+    - Original message details:
+    From: dperson@example.com
+    Subject: n/a
+    Date: ...
+    Message-ID: ...
+    <BLANKLINE>
+    - Results:
+    dperson@example.com left test@example.com
+    <BLANKLINE>
+    - Done.
+    <BLANKLINE>
+
+The -confirm address is also available as an implicit command.
+
+    >>> msg = message_from_string("""\
+    ... From: dperson@example.com
+    ... To: test-confirm+123@example.com
+    ...
+    ... """)
+
+    >>> inject_message(mlist, msg, switchboard='command', subaddress='confirm')
+    >>> command.run()
+    >>> len(virgin_queue.files)
+    1
+    >>> item = get_queue_messages('virgin')[0]
+    >>> print item.msg.as_string()
+    Subject: The results of your email commands
+    From: test-bounces@example.com
+    To: dperson@example.com
+    ...
+    <BLANKLINE>
+    The results of your email command are provided below.
+    <BLANKLINE>
+    - Original message details:
+    From: dperson@example.com
+    Subject: n/a
+    Date: ...
+    Message-ID: ...
+    <BLANKLINE>
+    - Results:
+    Confirmed
+    <BLANKLINE>
+    - Done.
+    <BLANKLINE>
+
+
 Stopping command processing
 ===========================
 
diff --git a/src/mailman/queue/docs/lmtp.txt b/src/mailman/queue/docs/lmtp.txt
index 5961bea50..549161b08 100644
--- a/src/mailman/queue/docs/lmtp.txt
+++ b/src/mailman/queue/docs/lmtp.txt
@@ -61,6 +61,25 @@ Once the mailing list is created, the posting address is valid.
     ... """)
     {}
 
+    >>> from mailman.testing.helpers import get_queue_messages
+    >>> messages = get_queue_messages('in')
+    >>> len(messages)
+    1
+    >>> print messages[0].msg.as_string()
+    From: anne.person@example.com
+    To: mylist@example.com
+    Subject: An interesting message
+    Message-ID: <badger>
+    X-MailFrom: anne.person@example.com
+    <BLANKLINE>
+    This is an interesting message.
+    >>> dump_msgdata(messages[0].msgdata)
+    _parsemsg    : False
+    listname     : mylist@example.com
+    original_size: ...
+    to_list      : True
+    version      : ...
+
 
 Sub-addresses
 =============
@@ -98,6 +117,193 @@ But the message is accepted if posted to a valid sub-address.
     {}
 
 
+Request subaddress
+------------------
+
+Depending on the subaddress, there is a message in the appropriate queue for
+later processing.  For example, all -request messages are put into the command
+queue for processing.
+
+    >>> messages = get_queue_messages('command')
+    >>> len(messages)
+    1
+    >>> print messages[0].msg.as_string()
+    From: anne.person@example.com
+    To: mylist-request@example.com
+    Subject: Help
+    Message-ID: <dog>
+    X-MailFrom: anne.person@example.com
+    <BLANKLINE>
+    Please help me.
+    >>> dump_msgdata(messages[0].msgdata)
+    _parsemsg    : False
+    listname     : mylist@example.com
+    original_size: ...
+    subaddress   : request
+    version      : ...
+
+
+Bounce processor
+----------------
+
+A message to the -bounces address goes to the bounce processor.
+
+    >>> lmtp.sendmail(
+    ...     'mail-daemon@example.com',
+    ...     ['mylist-bounces@example.com'], """\
+    ... From: mail-daemon@example.com
+    ... To: mylist-bounces@example.com
+    ... Subject: A bounce
+    ... Message-ID: <elephant>
+    ...
+    ... Bouncy bouncy.
+    ... """)
+    {}
+    >>> messages = get_queue_messages('bounces')
+    >>> len(messages)
+    1
+    >>> dump_msgdata(messages[0].msgdata)
+    _parsemsg    : False
+    listname     : mylist@example.com
+    original_size: ...
+    subaddress   : bounces
+    version      : ...
+
+
+Command processor
+-----------------
+
+Confirmation messages go to the command processor...
+
+    >>> lmtp.sendmail(
+    ...     'anne.person@example.com',
+    ...     ['mylist-confirm@example.com'], """\
+    ... From: anne.person@example.com
+    ... To: mylist-confirm@example.com
+    ... Subject: A bounce
+    ... Message-ID: <falcon>
+    ...
+    ... confirm 123
+    ... """)
+    {}
+    >>> messages = get_queue_messages('command')
+    >>> len(messages)
+    1
+    >>> dump_msgdata(messages[0].msgdata)
+    _parsemsg    : False
+    listname     : mylist@example.com
+    original_size: ...
+    subaddress   : confirm
+    version      : ...
+
+...as do join messages...
+
+    >>> lmtp.sendmail(
+    ...     'anne.person@example.com',
+    ...     ['mylist-join@example.com'], """\
+    ... From: anne.person@example.com
+    ... To: mylist-join@example.com
+    ... Message-ID: <giraffe>
+    ...
+    ... """)
+    {}
+    >>> messages = get_queue_messages('command')
+    >>> len(messages)
+    1
+    >>> dump_msgdata(messages[0].msgdata)
+    _parsemsg    : False
+    listname     : mylist@example.com
+    original_size: ...
+    subaddress   : join
+    version      : ...
+
+    >>> lmtp.sendmail(
+    ...     'anne.person@example.com',
+    ...     ['mylist-subscribe@example.com'], """\
+    ... From: anne.person@example.com
+    ... To: mylist-subscribe@example.com
+    ... Message-ID: <hippopotamus>
+    ...
+    ... """)
+    {}
+    >>> messages = get_queue_messages('command')
+    >>> len(messages)
+    1
+    >>> dump_msgdata(messages[0].msgdata)
+    _parsemsg    : False
+    listname     : mylist@example.com
+    original_size: ...
+    subaddress   : join
+    version      : ...
+
+...and leave messages.
+
+    >>> lmtp.sendmail(
+    ...     'anne.person@example.com',
+    ...     ['mylist-leave@example.com'], """\
+    ... From: anne.person@example.com
+    ... To: mylist-leave@example.com
+    ... Message-ID: <iguana>
+    ...
+    ... """)
+    {}
+    >>> messages = get_queue_messages('command')
+    >>> len(messages)
+    1
+    >>> dump_msgdata(messages[0].msgdata)
+    _parsemsg    : False
+    listname     : mylist@example.com
+    original_size: ...
+    subaddress   : leave
+    version      : ...
+
+    >>> lmtp.sendmail(
+    ...     'anne.person@example.com',
+    ...     ['mylist-unsubscribe@example.com'], """\
+    ... From: anne.person@example.com
+    ... To: mylist-unsubscribe@example.com
+    ... Message-ID: <jackal>
+    ...
+    ... """)
+    {}
+    >>> messages = get_queue_messages('command')
+    >>> len(messages)
+    1
+    >>> dump_msgdata(messages[0].msgdata)
+    _parsemsg    : False
+    listname     : mylist@example.com
+    original_size: ...
+    subaddress   : leave
+    version      : ...
+
+
+Incoming processor
+------------------
+
+Messages to the -owner address go to the incoming processor.
+
+    >>> lmtp.sendmail(
+    ...     'anne.person@example.com',
+    ...     ['mylist-owner@example.com'], """\
+    ... From: anne.person@example.com
+    ... To: mylist-owner@example.com
+    ... Message-ID: <kangaroo>
+    ...
+    ... """)
+    {}
+    >>> messages = get_queue_messages('in')
+    >>> len(messages)
+    1
+    >>> dump_msgdata(messages[0].msgdata)
+    _parsemsg    : False
+    envsender    : changeme@example.com
+    listname     : mylist@example.com
+    original_size: ...
+    subaddress   : owner
+    to_owner     : True
+    version      : ...
+
+
 Clean up
 ========
 
diff --git a/src/mailman/queue/lmtp.py b/src/mailman/queue/lmtp.py
index c07594c0e..e93fa6624 100644
--- a/src/mailman/queue/lmtp.py
+++ b/src/mailman/queue/lmtp.py
@@ -50,10 +50,29 @@ qlog = logging.getLogger('mailman.qrunner')
 
 
 # We only care about the listname and the sub-addresses as in listname@ or
-# listname-request@
-SUBADDRESS_NAMES = (
-    'bounces',  'confirm',  'join', '       leave',
-    'owner',    'request',  'subscribe',    'unsubscribe',
+# listname-request@.  This maps user visible subaddress names (which may
+# include aliases) to the internal canonical subaddress name.
+SUBADDRESS_NAMES = dict(
+    admin='bounces',
+    bounces='bounces',
+    confirm='confirm',
+    join='join',
+    leave='leave',
+    owner='owner',
+    request='request',
+    subscribe='join',
+    unsubscribe='leave',
+    )
+
+# This maps subaddress canonical name to the destination queue that handles
+# messages sent to that subaddress.
+SUBADDRESS_QUEUES = dict(
+    bounces='bounces',
+    confirm='command',
+    join='command',
+    leave='command',
+    owner='in',
+    request='command',
     )
 
 DASH    = '-'
@@ -136,7 +155,6 @@ class LMTPRunner(Runner, smtpd.SMTPServer):
             # Refresh the list of list names every time we process a message
             # since the set of mailing lists could have changed.
             listnames = set(getUtility(IListManager).names)
-            qlog.debug('listnames: %s', listnames)
             # Parse the message data.  If there are any defects in the
             # message, reject it right away; it's probably spam. 
             msg = email.message_from_string(data, Message)
@@ -144,6 +162,7 @@ class LMTPRunner(Runner, smtpd.SMTPServer):
             if msg.defects:
                 return ERR_501
             msg['X-MailFrom'] = mailfrom
+            message_id = msg['message-id']
         except Exception, e:
             elog.exception('LMTP message parsing')
             config.db.abort()
@@ -158,48 +177,45 @@ class LMTPRunner(Runner, smtpd.SMTPServer):
             try:
                 to = parseaddr(to)[1].lower()
                 listname, subaddress, domain = split_recipient(to)
-                qlog.debug('to: %s, list: %s, sub: %s, dom: %s',
-                           to, listname, subaddress, domain)
+                qlog.debug('%s to: %s, list: %s, sub: %s, dom: %s',
+                           message_id, to, listname, subaddress, domain)
                 listname += '@' + domain
                 if listname not in listnames:
                     status.append(ERR_550)
                     continue
-                # The recipient is a valid mailing list; see if it's a valid
-                # sub-address, and if so, enqueue it.
+                # The recipient is a valid mailing list.  Find the subaddress
+                # if there is one, and set things up to enqueue to the proper
+                # queue runner.
                 queue = None
                 msgdata = dict(listname=listname,
                                original_size=msg.original_size)
-                if subaddress in ('bounces', 'admin'):
-                    queue = 'bounce'
-                elif subaddress == 'confirm':
-                    msgdata['to_confirm'] = True
-                    queue = 'command'
-                elif subaddress in ('join', 'subscribe'):
-                    msgdata['to_join'] = True
-                    queue = 'command'
-                elif subaddress in ('leave', 'unsubscribe'):
-                    msgdata['to_leave'] = True
-                    queue = 'command'
-                elif subaddress == 'owner':
-                    msgdata.update(dict(
-                        to_owner=True,
-                        envsender=config.mailman.site_owner,
-                        ))
-                    queue = 'in'
-                elif subaddress is None:
+                canonical_subaddress = SUBADDRESS_NAMES.get(subaddress)
+                queue = SUBADDRESS_QUEUES.get(canonical_subaddress)
+                if subaddress is None:
+                    # The message is destined for the mailing list.
                     msgdata['to_list'] = True
                     queue = 'in'
-                elif subaddress == 'request':
-                    msgdata['to_request'] = True
-                    queue = 'command'
-                else:
-                    elog.error('Unknown sub-address: %s', subaddress)
+                elif canonical_subaddress is None:
+                    # The subaddress was bogus.
+                    elog.error('%s unknown sub-address: %s',
+                               message_id, subaddress)
                     status.append(ERR_550)
                     continue
-                # If we found a valid subaddress, enqueue the message and add
+                else:
+                    # A valid subaddress.
+                    msgdata['subaddress'] = canonical_subaddress
+                    if canonical_subaddress == 'owner':
+                        msgdata.update(dict(
+                            to_owner=True,
+                            envsender=config.mailman.site_owner,
+                            ))
+                        queue = 'in'
+                # If we found a valid destination, enqueue the message and add
                 # a success status for this recipient.
                 if queue is not None:
                     config.switchboards[queue].enqueue(msg, msgdata)
+                    qlog.debug('%s subaddress: %s, queue: %s',
+                               message_id, canonical_subaddress, queue)
                     status.append('250 Ok')
             except Exception, e:
                 elog.exception('Queue detection: %s', msg['message-id'])