All three scripts have now been changed to always quickly queue their

messages to the qfiles directory.  This once and for all avoids the
possibility that we hit the MTA's command time limit.  The mailing
list objects are never locked so we can't time out there.  They don't
need to be locked for message queuing.

The penalty is that we do more disk i/o for every message destined to
the list, the list-owner or list-request, and messages are not
delivered immediately.  Both are probably worthy tradeoffs for
absolutely guaranteeing that messages never get lost.

5 files changed, 75 insertions, 127 deletions

diff --git a/scripts/mailcmd b/scripts/mailcmd
index 979030d4a..4201e18c8 100755
--- a/scripts/mailcmd
+++ b/scripts/mailcmd
@@ -28,12 +28,9 @@ Errors are redirected to logs/errors.
 import sys
 
 import paths
-from Mailman import mm_cfg
 from Mailman import MailList
-from Mailman import Utils
 from Mailman import Message
 from Mailman import Errors
-from Mailman.pythonlib.StringIO import StringIO
 from Mailman.Logging.Utils import LogStdErr
 
 LogStdErr("error", "mailcmd")
@@ -52,21 +49,19 @@ def main():
         sys.stderr.write('Mailman error: mailcmd got bad listname: %s\n%s' %
                          (listname, e))
         sys.exit(1)
-    # Try to obtain the mailing list lock.  If that fails, enqueue the message
-    # for delivery by the qrunner cron job.
-    try:
-        mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT)
-    except LockFile.TimeOutError:
-        msg.Enqueue(mlist)
-        return
-    try:
-        s = StringIO(sys.stdin.read())
-        msg = Message.Message(s)
-        # Moved the setting of 'torequest' into MailCommandHandler
-        mlist.ParseMailCommands(msg)
-    finally:
-        mlist.Save()
-        mlist.Unlock()
+    # Create the message object
+    msg = Message.Message(sys.stdin)
+    # Immediately queue the message for disposition by qrunner, most likely in
+    # about a minute from now.  The advantage to this approach is that
+    # messages should never get lost -- some MTAs have a hard limit to the
+    # time a filter prog can run.  Postfix is a good example; if the limit is
+    # hit, the proc is SIGKILL'd giving us no chance to save the message.  It
+    # could take a long time to acquire the lock.  This way we're fairly safe
+    # against catastrophe at the expense of more disk I/O.
+    #
+    # The `torequest' flag is a message to qrunner that an alternative route
+    # should be taken for this message.
+    msg.Enqueue(mlist, torequest=1)
 
 
 
diff --git a/scripts/mailowner b/scripts/mailowner
index 68e577105..a302b3ae8 100755
--- a/scripts/mailowner
+++ b/scripts/mailowner
@@ -18,25 +18,22 @@
 
 """Send a message to the mailing list owner.
 
-This script gets called by the wrapper.
+The -owner address for alist should be filtered to this program, through the
+mail wrapper.  E.g. for list `test@yourdomain.com', the `test-owner' alias
+would deliver to this script.
 
-Stdin is the mail message, and argv[1] is the name of the mailing list
-whose owner(s) to send mail to.
-"""
+Stdin is the mail message, and argv[1] is the name of the target mailing list.
 
+"""
 
 import sys
 import mimetools
 
 import paths
-from Mailman import mm_cfg
 from Mailman import MailList
 from Mailman import Message
 from Mailman import Errors
-from Mailman.Handlers import HandlerAPI
-from Mailman.Bouncers import BouncerAPI
 from Mailman.Logging.Utils import LogStdErr
-from Mailman.pythonlib.StringIO import StringIO
 
 LogStdErr('error', 'post')
 
@@ -54,34 +51,19 @@ def main():
         sys.stderr.write('Mailman error: mailowner got bad listname: %s\n%s' %
                          (listname, e))
         sys.exit(1)
-    # Try to obtain the mailing list lock.  If that fails, enqueue the message
-    # for delivery by the qrunner cron job.
-    try:
-        mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT)
-    except LockFile.TimeOutError:
-        msg.Enqueue(mlist)
-        return
-    try:
-        s = StringIO(sys.stdin.read())
-        msg = mimetools.Message(s)
-        if mlist.bounce_processing:
-            if BouncerAPI.ScanMessages(mlist, msg):
-                return
-        # okay, no bounces were detected, but we have to convert this
-        # mimetools.Message thingie to one of our OutgoingMessages.
-        msg = Message.OutgoingMessage(s.getvalue())
-        recips = mlist.owner[:]
-        # debugging
-        #recips.append(mm_cfg.MAILMAN_OWNER)
-        msgdata = {'recips' : recips,
-                   'toadmin': 1,
-                   }
-        enqueue = HandlerAPI.DeliverToUser(mlist, msg, msgdata)
-        if enqueue:
-            msg.Enqueue(mlist, newdata=msgdata)
-    finally:
-        mlist.Save()
-        mlist.Unlock()
+    # Create the message object
+    msg = Message.Message(sys.stdin)
+    # Immediately queue the message for disposition by qrunner, most likely in
+    # about a minute from now.  The advantage to this approach is that
+    # messages should never get lost -- some MTAs have a hard limit to the
+    # time a filter prog can run.  Postfix is a good example; if the limit is
+    # hit, the proc is SIGKILL'd giving us no chance to save the message.  It
+    # could take a long time to acquire the lock.  This way we're fairly safe
+    # against catastrophe at the expense of more disk I/O.
+    #
+    # The `toadmin' flag is a message to qrunner that an alternative route
+    # should be taken for this message.
+    msg.Enqueue(mlist, toadmin=1)
 
 
 
diff --git a/scripts/owner b/scripts/owner
index 68e577105..a302b3ae8 100755
--- a/scripts/owner
+++ b/scripts/owner
@@ -18,25 +18,22 @@
 
 """Send a message to the mailing list owner.
 
-This script gets called by the wrapper.
+The -owner address for alist should be filtered to this program, through the
+mail wrapper.  E.g. for list `test@yourdomain.com', the `test-owner' alias
+would deliver to this script.
 
-Stdin is the mail message, and argv[1] is the name of the mailing list
-whose owner(s) to send mail to.
-"""
+Stdin is the mail message, and argv[1] is the name of the target mailing list.
 
+"""
 
 import sys
 import mimetools
 
 import paths
-from Mailman import mm_cfg
 from Mailman import MailList
 from Mailman import Message
 from Mailman import Errors
-from Mailman.Handlers import HandlerAPI
-from Mailman.Bouncers import BouncerAPI
 from Mailman.Logging.Utils import LogStdErr
-from Mailman.pythonlib.StringIO import StringIO
 
 LogStdErr('error', 'post')
 
@@ -54,34 +51,19 @@ def main():
         sys.stderr.write('Mailman error: mailowner got bad listname: %s\n%s' %
                          (listname, e))
         sys.exit(1)
-    # Try to obtain the mailing list lock.  If that fails, enqueue the message
-    # for delivery by the qrunner cron job.
-    try:
-        mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT)
-    except LockFile.TimeOutError:
-        msg.Enqueue(mlist)
-        return
-    try:
-        s = StringIO(sys.stdin.read())
-        msg = mimetools.Message(s)
-        if mlist.bounce_processing:
-            if BouncerAPI.ScanMessages(mlist, msg):
-                return
-        # okay, no bounces were detected, but we have to convert this
-        # mimetools.Message thingie to one of our OutgoingMessages.
-        msg = Message.OutgoingMessage(s.getvalue())
-        recips = mlist.owner[:]
-        # debugging
-        #recips.append(mm_cfg.MAILMAN_OWNER)
-        msgdata = {'recips' : recips,
-                   'toadmin': 1,
-                   }
-        enqueue = HandlerAPI.DeliverToUser(mlist, msg, msgdata)
-        if enqueue:
-            msg.Enqueue(mlist, newdata=msgdata)
-    finally:
-        mlist.Save()
-        mlist.Unlock()
+    # Create the message object
+    msg = Message.Message(sys.stdin)
+    # Immediately queue the message for disposition by qrunner, most likely in
+    # about a minute from now.  The advantage to this approach is that
+    # messages should never get lost -- some MTAs have a hard limit to the
+    # time a filter prog can run.  Postfix is a good example; if the limit is
+    # hit, the proc is SIGKILL'd giving us no chance to save the message.  It
+    # could take a long time to acquire the lock.  This way we're fairly safe
+    # against catastrophe at the expense of more disk I/O.
+    #
+    # The `toadmin' flag is a message to qrunner that an alternative route
+    # should be taken for this message.
+    msg.Enqueue(mlist, toadmin=1)
 
 
 
diff --git a/scripts/post b/scripts/post
index b9720f03e..d9464ff69 100755
--- a/scripts/post
+++ b/scripts/post
@@ -18,8 +18,11 @@
 
 """Accept posts to a list and handle them properly.
 
-This script is invoked via the mail wrapper.  stdin is the mail message, and
-argv[1] is the name of the target mailing list.
+The main advertised address for a list should be filtered to this program,
+through the mail wrapper.  E.g. for list `test@yourdomain.com', the `test'
+alias would deliver to this script.
+
+Stdin is the mail message, and argv[1] is the name of the target mailing list.
 
 """
 
@@ -27,13 +30,10 @@ import sys
 
 import paths
 from Mailman import mm_cfg
-from Mailman import Utils
 from Mailman import MailList
 from Mailman import Message
 from Mailman import Errors
-from Mailman import LockFile
 from Mailman.Logging.Utils import LogStdErr
-from Mailman.Handlers import HandlerAPI
 from Mailman.pythonlib.StringIO import StringIO
 
 LogStdErr("error", "post")
@@ -79,20 +79,14 @@ def main():
     # object in a usable form.  From here on out, we should never lose
     # messages.
     msg = get_message(mlist)
-    # Try to obtain the mailing list lock.  If that fails, enqueue the message
-    # for delivery by the qrunner cron job.
-    try:
-        mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT)
-    except LockFile.TimeOutError:
-        msg.Enqueue(mlist)
-        return
-    try:
-        msgdata = {}
-        # ignore return value
-        HandlerAPI.DeliverToList(mlist, msg, msgdata)
-    finally:
-        mlist.Save()
-        mlist.Unlock()
+    # Immediately queue the message for the qrunner to deliver, mostly likely
+    # about a minute from now.  The advantage to this approach is that
+    # messages should never get lost -- some MTAs have a hard limit to the
+    # time a filter prog can run.  Postfix is a good example; if the limit is
+    # hit, the proc is SIGKILL'd giving us no chance to save the message.  It
+    # could take a long time to acquire the lock.  This way we're fairly safe
+    # against catastrophe at the expense of more disk I/O.
+    msg.Enqueue(mlist)
 
 
 
diff --git a/scripts/request b/scripts/request
index 979030d4a..4201e18c8 100755
--- a/scripts/request
+++ b/scripts/request
@@ -28,12 +28,9 @@ Errors are redirected to logs/errors.
 import sys
 
 import paths
-from Mailman import mm_cfg
 from Mailman import MailList
-from Mailman import Utils
 from Mailman import Message
 from Mailman import Errors
-from Mailman.pythonlib.StringIO import StringIO
 from Mailman.Logging.Utils import LogStdErr
 
 LogStdErr("error", "mailcmd")
@@ -52,21 +49,19 @@ def main():
         sys.stderr.write('Mailman error: mailcmd got bad listname: %s\n%s' %
                          (listname, e))
         sys.exit(1)
-    # Try to obtain the mailing list lock.  If that fails, enqueue the message
-    # for delivery by the qrunner cron job.
-    try:
-        mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT)
-    except LockFile.TimeOutError:
-        msg.Enqueue(mlist)
-        return
-    try:
-        s = StringIO(sys.stdin.read())
-        msg = Message.Message(s)
-        # Moved the setting of 'torequest' into MailCommandHandler
-        mlist.ParseMailCommands(msg)
-    finally:
-        mlist.Save()
-        mlist.Unlock()
+    # Create the message object
+    msg = Message.Message(sys.stdin)
+    # Immediately queue the message for disposition by qrunner, most likely in
+    # about a minute from now.  The advantage to this approach is that
+    # messages should never get lost -- some MTAs have a hard limit to the
+    # time a filter prog can run.  Postfix is a good example; if the limit is
+    # hit, the proc is SIGKILL'd giving us no chance to save the message.  It
+    # could take a long time to acquire the lock.  This way we're fairly safe
+    # against catastrophe at the expense of more disk I/O.
+    #
+    # The `torequest' flag is a message to qrunner that an alternative route
+    # should be taken for this message.
+    msg.Enqueue(mlist, torequest=1)