diff options
| -rw-r--r-- | Mailman/Queue/.cvsignore | 1 | ||||
| -rw-r--r-- | Mailman/Queue/IncomingRunner.py | 175 | ||||
| -rw-r--r-- | Mailman/Queue/Makefile.in | 69 | ||||
| -rw-r--r-- | Mailman/Queue/NewsRunner.py | 47 | ||||
| -rw-r--r-- | Mailman/Queue/OutgoingRunner.py | 57 | ||||
| -rw-r--r-- | Mailman/Queue/Runner.py | 149 | ||||
| -rw-r--r-- | Mailman/Queue/__init__.py | 15 |
7 files changed, 513 insertions, 0 deletions
diff --git a/Mailman/Queue/.cvsignore b/Mailman/Queue/.cvsignore new file mode 100644 index 000000000..f3c7a7c5d --- /dev/null +++ b/Mailman/Queue/.cvsignore @@ -0,0 +1 @@ +Makefile diff --git a/Mailman/Queue/IncomingRunner.py b/Mailman/Queue/IncomingRunner.py new file mode 100644 index 000000000..262bb9c7c --- /dev/null +++ b/Mailman/Queue/IncomingRunner.py @@ -0,0 +1,175 @@ +#! /usr/bin/env python +# +# Copyright (C) 1998,1999,2000 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +"""Incoming queue runner.""" + +# A typical Mailman list exposes four aliases which point to three different +# wrapped scripts. E.g. for a list named `mylist', you'd have: +# +# mylist -> post +# mylist-admin -> mailowner +# mylist-request -> mailcmd +# mylist-owner -> mailowner (through an alias to mylist-admin) +# +# Only 3 scripts are used for historical purposes, and this is unlikely to +# change to due backwards compatibility. That's immaterial though since the +# mailowner script can determine which alias it received the message on. +# +# mylist-request is a robot address; it's sole purpose is to process emailed +# commands in a Majordomo-like fashion. mylist-admin is supposed to reach the +# list owners, but it performs one vital step before list owner delivery - it +# examines the message for bounce content. mylist-owner is the fallback for +# delivery to the list owners; it performs no bounce detection, but it /does/ +# look for bounce loops, which can occur if a list owner address is bouncing. +# +# So delivery flow of messages look like this: +# +# joerandom ---> mylist ---> list members +# | | +# | |[bounces] +# +-------> mylist-admin <----+ <-------------------------------+ +# | | | +# | +--->[internal bounce processing] | +# | | | +# | | [bounce found] | +# | +--->[register and discard] | +# | | | +# | | [no bounce found] | +# | +---> list owners <------+ | +# | | | | +# | |[bounces] | | +# +-------> mylist-owner <-------------------+ | | +# | | | | +# | | [bounce loop detected] | | +# | +---> [log and discard] | | +# | | | | +# | +-----------------------------------------+ | +# | [no bounce loop detected] | +# | | +# | | +# +-------> mylist-request | +# | | +# +---> [command processor] | +# | | +# +---> joerandom | +# | | +# |[bounces] | +# +----------------------+ + + + +import mimetools + +from Mailman import mm_cfg +from Mailman.Queue import Runner +from Mailman.Handlers import HandlerAPI +from Mailman.Bouncers import BouncerAPI +from Mailman.Logging.Syslog import syslog +from Mailman.pythonlib.StringIO import StringIO + + + +class IncomingRunner(Runner.Runner): + def __init__(self, cachelists=1): + Runner.Runner.__init__(self, mm_cfg.INQUEUE_DIR) + + def _dispose_message(self, msg, msgdata): + # TBD: refactor this stanza. + # Find out which mailing list this message is destined for + listname = msgdata.get('listname') + if not listname: + syslog('qrunner', 'qfile metadata specifies no list: %s' % root) + return 1 + mlist = self._open_list(listname) + if not mlist: + syslog('qrunner', + 'Dequeuing message destined for missing list: %s' % root) + self._dequeue(root) + return 1 + # Now try to get the list lock + try: + mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT) + except LockFile.TimeOutError: + # oh well, try again later + return 1 + # + # runner specific code + try: + # The message may be destined for one of three subsystems: the + # list delivery subsystem (i.e. the message gets delivered to + # every member of the list), the bounce detector (i.e. this was a + # message to the -owner address), or the mail command handler + # (i.e. this was a message to the -request address). The flags + # controlling this path are found in the message metadata. + # + # post - no `toadmin' or `torequest' key + # mailowner - `toadmin' == true + # mailcmd - `torequest' == true + # + if msgdata.get('toadmin'): + s = StringIO(str(msg)) + mimemsg = mimetools.Message(s) + if mlist.bounce_processing: + if BouncerAPI.ScanMessages(mlist, mimemsg): + return 0 + # Either bounce processing isn't turned on or the bounce + # detector found no recognized bounce format in the message. + # In either case, forward the dang thing off to the list + # owners. Be sure to munge the headers so that any bounces + # from the list owners goes to the -owner address instead of + # the -admin address. This will avoid bounce loops. + msgdata.update({'recips' : mlist.owner[:], + 'errorsto': mlist.GetOwnerEmail(), + 'noack' : 0, # enable Replybot + }) + return HandlerAPI.DeliverToUser(mlist, msg, msgdata) + elif msgdata.get('toowner'): + # The message could have been a bounce from a broken list + # admin address. About the only other test we can do is to + # see if the message is appearing to come from a well-known + # MTA generated address. + sender = msg.GetSender() + i = sender.find('@') + if i >= 0: + senderlhs = sender[:i].lower() + else: + senderlhs = sender + if senderlhs in mm_cfg.LIKELY_BOUNCE_SENDERS: + syslog('error', 'bounce loop detected from: %s' % sender) + return 0 + # Any messages to the owner address must have Errors-To: set + # back to the owners address so bounce loops can be broken, as + # per the code above. + msgdata.update({'recips' : mlist.owner[:], + 'errorsto': mlist.GetOwnerEmail(), + 'noack' : 0, # enable Replybot + }) + return HandlerAPI.DeliverToUser(mlist, msg, msgdata) + elif msgdata.get('torequest'): + mlist.ParseMailCommands(msg) + return 0 + else: + # Pre 2.0beta3 qfiles have no schema version number + if msgdata.get('version', 0) < 1: + msg.Requeue(mlist, newdata=msgdata, + pipeline = [mm_cfg.DELIVERY_MODULE]) + return + return HandlerAPI.DeliverToList(mlist, msg, msgdata) + finally: + mlist.Save() + mlist.Unlock() diff --git a/Mailman/Queue/Makefile.in b/Mailman/Queue/Makefile.in new file mode 100644 index 000000000..220377149 --- /dev/null +++ b/Mailman/Queue/Makefile.in @@ -0,0 +1,69 @@ +# Copyright (C) 1998,1999,2000 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +# NOTE: Makefile.in is converted into Makefile by the configure script +# in the parent directory. Once configure has run, you can recreate +# the Makefile by running just config.status. + +# Variables set by configure + +VPATH= @srcdir@ +srcdir= @srcdir@ +bindir= @bindir@ +prefix= @prefix@ +exec_prefix= @exec_prefix@ + +CC= @CC@ +CHMOD= @CHMOD@ +INSTALL= @INSTALL@ + +DEFS= @DEFS@ + +# Customizable but not set by configure + +OPT= @OPT@ +CFLAGS= $(OPT) $(DEFS) +PACKAGEDIR= $(prefix)/Mailman/Queue +SHELL= /bin/sh + +MODULES= *.py + +# Modes for directories and executables created by the install +# process. Default to group-writable directories but +# user-only-writable for executables. +DIRMODE= 775 +EXEMODE= 755 +FILEMODE= 644 +INSTALL_PROGRAM=$(INSTALL) -m $(EXEMODE) + + +# Rules + +all: + +install: + for f in $(MODULES); \ + do \ + $(INSTALL) -m $(FILEMODE) $$f $(PACKAGEDIR); \ + done + +finish: + +clean: + +distclean: + -rm *.pyc + -rm Makefile diff --git a/Mailman/Queue/NewsRunner.py b/Mailman/Queue/NewsRunner.py new file mode 100644 index 000000000..a441b959a --- /dev/null +++ b/Mailman/Queue/NewsRunner.py @@ -0,0 +1,47 @@ +#! /usr/bin/env python +# +# Copyright (C) 2000 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +"""NNTP queue runner.""" + +from Mailman import mm_cfg +from Mailman.Queue import Runner +from Mailman.Handlers import HandlerAPI + + + +class NewsRunner(Runner.Runner): + def __init__(self, cachelists=1): + Runner.Runner.__init__(self, mm_cfg.NEWSQUEUE_DIR) + + def _dispose_message(self, msg, msgdata): + # TBD: refactor this stanza. + # Find out which mailing list this message is destined for + listname = msgdata.get('listname') + if not listname: + syslog('qrunner', 'qfile metadata specifies no list: %s' % root) + return 1 + mlist = self._open_list(listname) + if not mlist: + syslog('qrunner', + 'Dequeuing message destined for missing list: %s' % root) + self._dequeue(root) + return 1 + # The mailing list object does not need to be locked + pipeline = HandlerAPI.do_pipeline(mlist, msg, msgdata, + ['NNTPDirect']) + return len(pipeline) diff --git a/Mailman/Queue/OutgoingRunner.py b/Mailman/Queue/OutgoingRunner.py new file mode 100644 index 000000000..2bd7cc38c --- /dev/null +++ b/Mailman/Queue/OutgoingRunner.py @@ -0,0 +1,57 @@ +#! /usr/bin/env python +# +# Copyright (C) 2000 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +"""Outgoing queue runner.""" + +from Mailman import mm_cfg +from Mailman.Queue import Runner +from Mailman.Handlers import HandlerAPI + + + +class OutgoingRunner(Runner.Runner): + def __init__(self, cachelists=1): + Runner.Runner.__init__(self, mm_cfg.OUTQUEUE_DIR) + + def _dispose_message(self, msg, msgdata): + # TBD: refactor this stanza. + # Find out which mailing list this message is destined for + listname = msgdata.get('listname') + if not listname: + syslog('qrunner', 'qfile metadata specifies no list: %s' % root) + return 1 + mlist = self._open_list(listname) + if not mlist: + syslog('qrunner', + 'Dequeuing message destined for missing list: %s' % root) + self._dequeue(root) + return 1 + # Now try to get the list lock + try: + mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT) + except LockFile.TimeOutError: + # oh well, try again later + return 1 + # + # runner specific code + try: + msgdata['pipeline'] = [mm_cfg.DELIVERY_MODULE] + return HandlerAPI.DeliverToList(mlist, msg, msgdata) + finally: + mlist.Save() + mlist.Unlock() diff --git a/Mailman/Queue/Runner.py b/Mailman/Queue/Runner.py new file mode 100644 index 000000000..bac76d762 --- /dev/null +++ b/Mailman/Queue/Runner.py @@ -0,0 +1,149 @@ +#! /usr/bin/env python +# +# Copyright (C) 1998,1999,2000 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + +"""Generic queue runner class. +""" + +import os +import marshal +import random +import time + +from Mailman import mm_cfg +from Mailman import Utils +from Mailman import Errors +from Mailman import MailList +from Mailman import Message +from Mailman import LockFile +from Mailman.Logging.Syslog import syslog + + + +class Runner: + def __init__(self, qdir, cachelists=1): + self._qdir = qdir + self._kids = {} + self._cachelists = cachelists + self._lock = LockFile.LockFile(os.path.join(qdir, 'qrunner.lock'), + lifetime = mm_cfg.QRUNNER_LOCK_LIFETIME) + + def _dequeue(self, filebase): + os.unlink(filebase + '.db') + os.unlink(filebase + '.msg') + + _listcache = {} + + def _open_list(self, listname, lockp=1): + if self._cachelists: + mlist = self._listcache.get(listname) + else: + mlist = None + if not mlist: + try: + mlist = MailList.MailList(listname, lock=0) + if self._cachelists: + self._listcache[listname] = mlist + except Errors.MMListError, e: + syslog('qrunner', 'error opening list: %s\n%s' % (listname, e)) + return None + return mlist + + def _start(self): + self._msgcount = 0 + self._t0 = time.time() + try: + self._lock.lock(timeout=0.5) + except LockFile.TimeOutError: + # Some other qrunner process is running, which is fine. + syslog('qrunner', 'Could not acquire %s lock' % + self.__class__) + return 0 + return 1 + + def _cleanup(self): + Utils.reap(self._kids) + self._listcache.clear() + + def _onefile(self, filebase): + msgfp = dbfp = None + try: + dbfp = open(filebase + '.db') + msgdata = marshal.load(dbfp) + dbfp.close() + dbfp = None + msgfp = open(filebase + '.msg') + # re-establish the file base for re-queuing + msg = Message.Message(msgfp, filebase=msgdata.get('filebase')) + msgfp.close() + msgfp = None + except (EOFError, ValueError, TypeError, IOError), e: + # For some reason we had trouble getting all the information out + # of the queued files. log this and move on (we figure it's a + # temporary problem) + syslog('qrunner', + 'Exception reading qfiles: %s\n%s' % (filebase, e)) + if msgfp: + msgfp.close() + if dbfp: + dbfp.close() + return + keepqueued = self._dispose_message(msg, msgdata) + # Did the delivery generate child processes? + kids = msgdata.get('_kids') + if kids: + self._kids.update(kids) + del msgdata['_kids'] + if not keepqueued: + # We're done with this message + self._dequeue(filebase) + + def _dispose_message(self, msg, msgdata): + raise UnimplementedError + + def _doperiodic(self): + if mm_cfg.QRUNNER_MAX_MESSAGES is not None and \ + self._msgcount > mm_cfg.QRUNNER_MAX_MESSAGES: + return 0 + if mm_cfg.QRUNNER_PROCESS_LIFETIME is not None and \ + (time.time() - self._t0) > mm_cfg.QRUNNER_PROCESS_LIFETIME: + return 0 + self._msgcount += 1 + return 1 + + def run(self): + # Give us the absolute path to all the unique filebase file names in + # the current directory. + files = [] + for file in os.listdir(self._qdir): + root, ext = os.path.splitext(file) + if ext == '.db': + files.append(os.path.join(self._qdir, root)) + # Randomize this list so we're more likely to touch them all + # eventually, even if we're hitting resource limits. + random.shuffle(files) + # initialize the resource counters + okaytostart = self._start() + if not okaytostart: + return + for filebase in files: + keepgoing = self._doperiodic() + if not keepgoing: + break + self._onefile(filebase) + # clean up after ourselves + self._cleanup() diff --git a/Mailman/Queue/__init__.py b/Mailman/Queue/__init__.py new file mode 100644 index 000000000..a46d82f55 --- /dev/null +++ b/Mailman/Queue/__init__.py @@ -0,0 +1,15 @@ +# Copyright (C) 1998,1999,2000 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. |