diff options
| author | Barry Warsaw | 2009-01-01 17:58:39 -0500 |
|---|---|---|
| committer | Barry Warsaw | 2009-01-01 17:58:39 -0500 |
| commit | 1c285f110d8e98597453c6b4b69ea01163033547 (patch) | |
| tree | 00c7ec16711b2073e40f593658f652726a9d4231 /mailman/queue | |
| parent | 12513c7d0fc1f5d2a1aabda349637309f6e8300b (diff) | |
| parent | 600ddb503a391d70230d96ee91a631888d11b35a (diff) | |
| download | mailman-1c285f110d8e98597453c6b4b69ea01163033547.tar.gz mailman-1c285f110d8e98597453c6b4b69ea01163033547.tar.zst mailman-1c285f110d8e98597453c6b4b69ea01163033547.zip | |
Two major structural conversions.
* Use zc.buildout and zc.testing frameworks for building and testing Mailman.
* Use lazr.config as the configuration system, though this conversion is not
yet complete.
Also: move a bunch of old bin scripts to the attic.
Diffstat (limited to 'mailman/queue')
| -rw-r--r-- | mailman/queue/__init__.py | 124 | ||||
| -rw-r--r-- | mailman/queue/archive.py | 24 | ||||
| -rw-r--r-- | mailman/queue/bounce.py | 31 | ||||
| -rw-r--r-- | mailman/queue/command.py | 13 | ||||
| -rw-r--r-- | mailman/queue/docs/archiver.txt | 4 | ||||
| -rw-r--r-- | mailman/queue/docs/command.txt | 19 | ||||
| -rw-r--r-- | mailman/queue/docs/incoming.txt | 16 | ||||
| -rw-r--r-- | mailman/queue/docs/lmtp.txt | 2 | ||||
| -rw-r--r-- | mailman/queue/docs/news.txt | 1 | ||||
| -rw-r--r-- | mailman/queue/docs/outgoing.txt | 6 | ||||
| -rw-r--r-- | mailman/queue/docs/runner.txt | 24 | ||||
| -rw-r--r-- | mailman/queue/docs/switchboard.txt | 1 | ||||
| -rw-r--r-- | mailman/queue/http.py | 4 | ||||
| -rw-r--r-- | mailman/queue/incoming.py | 6 | ||||
| -rw-r--r-- | mailman/queue/lmtp.py | 36 | ||||
| -rw-r--r-- | mailman/queue/maildir.py | 4 | ||||
| -rw-r--r-- | mailman/queue/news.py | 14 | ||||
| -rw-r--r-- | mailman/queue/outgoing.py | 17 | ||||
| -rw-r--r-- | mailman/queue/pipeline.py | 8 | ||||
| -rw-r--r-- | mailman/queue/retry.py | 15 | ||||
| -rw-r--r-- | mailman/queue/virgin.py | 8 |
21 files changed, 196 insertions, 181 deletions
diff --git a/mailman/queue/__init__.py b/mailman/queue/__init__.py index 961a46283..52686b3ab 100644 --- a/mailman/queue/__init__.py +++ b/mailman/queue/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2008 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2009 by the Free Software Foundation, Inc. # # This file is part of GNU Mailman. # @@ -32,6 +32,7 @@ __all__ = [ import os +import sys import time import email import errno @@ -43,13 +44,16 @@ import marshal import traceback from cStringIO import StringIO +from lazr.config import as_boolean, as_timedelta +from string import Template from zope.interface import implements -from mailman import i18n from mailman import Message from mailman import Utils -from mailman.configuration import config -from mailman.interfaces import IRunner, ISwitchboard +from mailman import i18n +from mailman.config import config +from mailman.interfaces.runner import IRunner +from mailman.interfaces.switchboard import ISwitchboard # 20 bytes of all bits set, maximum hashlib.sha.digest() value shamax = 0xffffffffffffffffffffffffffffffffffffffffL @@ -57,6 +61,7 @@ shamax = 0xffffffffffffffffffffffffffffffffffffffffL # Small increment to add to time in case two entries have the same time. This # prevents skipping one of two entries with the same time until the next pass. DELTA = .0001 +DOT = '.' elog = logging.getLogger('mailman.error') dlog = logging.getLogger('mailman.debug') @@ -66,11 +71,24 @@ dlog = logging.getLogger('mailman.debug') class Switchboard: implements(ISwitchboard) - def __init__(self, whichq, slice=None, numslices=1, recover=False): + @staticmethod + def initialize(): + """Initialize the global switchboards for input/output.""" + for conf in config.qrunner_configs: + name = conf.name.split('.')[-1] + assert name not in config.switchboards, ( + 'Duplicate qrunner name: %s' % name) + substitutions = config.paths + substitutions['name'] = name + path = Template(conf.path).safe_substitute(substitutions) + config.switchboards[name] = Switchboard(path) + + def __init__(self, queue_directory, + slice=None, numslices=1, recover=False): """Create a switchboard object. - :param whichq: The queue directory. - :type whichq: str + :param queue_directory: The queue directory. + :type queue_directory: str :param slice: The slice number for this switchboard, or None. If not None, it must be [0..`numslices`). :type slice: int or None @@ -80,9 +98,11 @@ class Switchboard: :param recover: True if backup files should be recovered. :type recover: bool """ - self._whichq = whichq + assert (numslices & (numslices - 1)) == 0, ( + 'Not a power of 2: %s' % numslices) + self.queue_directory = queue_directory # Create the directory if it doesn't yet exist. - Utils.makedirs(self._whichq, 0770) + Utils.makedirs(self.queue_directory, 0770) # Fast track for no slices self._lower = None self._upper = None @@ -93,11 +113,6 @@ class Switchboard: if recover: self.recover_backup_files() - @property - def queue_directory(self): - """See `ISwitchboard`.""" - return self._whichq - def enqueue(self, _msg, _metadata=None, **_kws): """See `ISwitchboard`.""" if _metadata is None: @@ -125,7 +140,7 @@ class Switchboard: # and the sha hex digest. rcvtime = data.setdefault('received_time', now) filebase = repr(rcvtime) + '+' + hashlib.sha1(hashfood).hexdigest() - filename = os.path.join(self._whichq, filebase + '.pck') + filename = os.path.join(self.queue_directory, filebase + '.pck') tmpfile = filename + '.tmp' # Always add the metadata schema version number data['version'] = config.QFILE_SCHEMA_VERSION @@ -148,8 +163,8 @@ class Switchboard: def dequeue(self, filebase): """See `ISwitchboard`.""" # Calculate the filename from the given filebase. - filename = os.path.join(self._whichq, filebase + '.pck') - backfile = os.path.join(self._whichq, filebase + '.bak') + filename = os.path.join(self.queue_directory, filebase + '.pck') + backfile = os.path.join(self.queue_directory, filebase + '.bak') # Read the message object and metadata. with open(filename) as fp: # Move the file to the backup file name for processing. If this @@ -172,13 +187,13 @@ class Switchboard: return msg, data def finish(self, filebase, preserve=False): - bakfile = os.path.join(self._whichq, filebase + '.bak') + bakfile = os.path.join(self.queue_directory, filebase + '.bak') try: if preserve: - psvfile = os.path.join(config.SHUNTQUEUE_DIR, - filebase + '.psv') + shunt_dir = config.switchboards['shunt'].queue_directory + psvfile = os.path.join(shunt_dir, filebase + '.psv') # Create the directory if it doesn't yet exist. - Utils.makedirs(config.SHUNTQUEUE_DIR, 0770) + Utils.makedirs(shunt_dir, 0770) os.rename(bakfile, psvfile) else: os.unlink(bakfile) @@ -196,7 +211,7 @@ class Switchboard: times = {} lower = self._lower upper = self._upper - for f in os.listdir(self._whichq): + for f in os.listdir(self.queue_directory): # By ignoring anything that doesn't end in .pck, we ignore # tempfiles and avoid a race condition. filebase, ext = os.path.splitext(f) @@ -220,8 +235,8 @@ class Switchboard: # to exist at the same time, so the move is enough to ensure that our # normal dequeuing process will handle them. for filebase in self.get_files('.bak'): - src = os.path.join(self._whichq, filebase + '.bak') - dst = os.path.join(self._whichq, filebase + '.pck') + src = os.path.join(self.queue_directory, filebase + '.bak') + dst = os.path.join(self.queue_directory, filebase + '.pck') os.rename(src, dst) @@ -229,27 +244,31 @@ class Switchboard: class Runner: implements(IRunner) - QDIR = None - SLEEPTIME = None - - def __init__(self, slice=None, numslices=1): + def __init__(self, name, slice=None): """Create a queue runner. :param slice: The slice number for this queue runner. This is passed directly to the underlying `ISwitchboard` object. :type slice: int or None - :param numslices: The number of slices for this queue. Must be a - power of 2. - :type numslices: int """ - # Create our own switchboard. Don't use the switchboard cache because - # we want to provide slice and numslice arguments. - self._switchboard = Switchboard(self.QDIR, slice, numslices, True) - # Create the shunt switchboard - self._shunt = Switchboard(config.SHUNTQUEUE_DIR) + # Grab the configuration section. + self.name = name + section = getattr(config, 'qrunner.' + name) + substitutions = config.paths + substitutions['name'] = name + self.queue_directory = Template(section.path).safe_substitute( + substitutions) + numslices = int(section.instances) + self.switchboard = Switchboard( + self.queue_directory, slice, numslices, True) + self.sleep_time = as_timedelta(section.sleep_time) + # sleep_time is a timedelta; turn it into a float for time.sleep(). + self.sleep_float = (86400 * self.sleep_time.days + + self.sleep_time.seconds + + self.sleep_time.microseconds / 1000000.0) + self.max_restarts = int(section.max_restarts) + self.start = as_boolean(section.start) self._stop = False - if self.SLEEPTIME is None: - self.SLEEPTIME = config.QRUNNER_SLEEP_TIME def __repr__(self): return '<%s at %s>' % (self.__class__.__name__, id(self)) @@ -286,13 +305,13 @@ class Runner: dlog.debug('[%s] starting oneloop', me) # List all the files in our queue directory. The switchboard is # guaranteed to hand us the files in FIFO order. - files = self._switchboard.files + files = self.switchboard.files for filebase in files: dlog.debug('[%s] processing filebase: %s', me, filebase) try: # Ask the switchboard for the message and metadata objects # associated with this queue file. - msg, msgdata = self._switchboard.dequeue(filebase) + msg, msgdata = self.switchboard.dequeue(filebase) except Exception, e: # This used to just catch email.Errors.MessageParseError, but # other problems can occur in message parsing, e.g. @@ -302,14 +321,14 @@ class Runner: self._log(e) elog.error('Skipping and preserving unparseable message: %s', filebase) - self._switchboard.finish(filebase, preserve=True) + self.switchboard.finish(filebase, preserve=True) config.db.abort() continue try: dlog.debug('[%s] processing onefile', me) self._process_one_file(msg, msgdata) dlog.debug('[%s] finishing filebase: %s', me, filebase) - self._switchboard.finish(filebase) + self.switchboard.finish(filebase) except Exception, e: # All runners that implement _dispose() must guarantee that # exceptions are caught and dealt with properly. Still, there @@ -319,14 +338,15 @@ class Runner: # intervention. self._log(e) # Put a marker in the metadata for unshunting. - msgdata['whichq'] = self._switchboard.queue_directory + msgdata['whichq'] = self.switchboard.queue_directory # It is possible that shunting can throw an exception, e.g. a # permissions problem or a MemoryError due to a really large # message. Try to be graceful. try: - new_filebase = self._shunt.enqueue(msg, msgdata) + shunt = config.switchboards['shunt'] + new_filebase = shunt.enqueue(msg, msgdata) elog.error('SHUNTING: %s', new_filebase) - self._switchboard.finish(filebase) + self.switchboard.finish(filebase) except Exception, e: # The message wasn't successfully shunted. Log the # exception and try to preserve the original queue entry @@ -335,13 +355,13 @@ class Runner: elog.error( 'SHUNTING FAILED, preserving original entry: %s', filebase) - self._switchboard.finish(filebase, preserve=True) + self.switchboard.finish(filebase, preserve=True) config.db.abort() # Other work we want to do each time through the loop. dlog.debug('[%s] doing periodic', me) self._do_periodic() dlog.debug('[%s] checking short circuit', me) - if self._short_curcuit(): + if self._short_circuit(): dlog.debug('[%s] short circuiting', me) break dlog.debug('[%s] commiting', me) @@ -363,7 +383,7 @@ class Runner: if mlist is None: elog.error('Dequeuing message destined for missing list: %s', listname) - self._shunt.enqueue(msg, msgdata) + config.switchboards['shunt'].enqueue(msg, msgdata) return # Now process this message. We also want to set up the language # context for this message. The context will be the preferred @@ -380,7 +400,7 @@ class Runner: msgdata['lang'] = language keepqueued = self._dispose(mlist, msg, msgdata) if keepqueued: - self._switchboard.enqueue(msg, msgdata) + self.switchboard.enqueue(msg, msgdata) def _log(self, exc): elog.error('Uncaught runner exception: %s', exc) @@ -401,10 +421,10 @@ class Runner: def _snooze(self, filecnt): """See `IRunner`.""" - if filecnt or float(self.SLEEPTIME) <= 0: + if filecnt or self.sleep_float <= 0: return - time.sleep(float(self.SLEEPTIME)) + time.sleep(self.sleep_float) - def _short_curcuit(self): + def _short_circuit(self): """See `IRunner`.""" return self._stop diff --git a/mailman/queue/archive.py b/mailman/queue/archive.py index 6dda70387..69ec46f4b 100644 --- a/mailman/queue/archive.py +++ b/mailman/queue/archive.py @@ -1,4 +1,4 @@ -# Copyright (C) 2000-2008 by the Free Software Foundation, Inc. +# Copyright (C) 2000-2009 by the Free Software Foundation, Inc. # # This file is part of GNU Mailman. # @@ -25,19 +25,22 @@ __all__ = [ import os import time +import logging from datetime import datetime from email.Utils import parsedate_tz, mktime_tz, formatdate from locknix.lockfile import Lock -from mailman.configuration import config +from mailman import Defaults from mailman.core.plugins import get_plugins from mailman.queue import Runner +log = logging.getLogger('mailman.error') + class ArchiveRunner(Runner): - QDIR = config.ARCHQUEUE_DIR + """The archive runner.""" def _dispose(self, mlist, msg, msgdata): # Support clobber_date, i.e. setting the date in the archive to the @@ -48,9 +51,9 @@ class ArchiveRunner(Runner): received_time = formatdate(msgdata['received_time']) if not original_date: clobber = True - elif config.ARCHIVER_CLOBBER_DATE_POLICY == 1: + elif Defaults.ARCHIVER_CLOBBER_DATE_POLICY == 1: clobber = True - elif config.ARCHIVER_CLOBBER_DATE_POLICY == 2: + elif Defaults.ARCHIVER_CLOBBER_DATE_POLICY == 2: # What's the timestamp on the original message? timetup = parsedate_tz(original_date) now = datetime.now() @@ -60,7 +63,7 @@ class ArchiveRunner(Runner): else: utc_timestamp = datetime.fromtimestamp(mktime_tz(timetup)) clobber = (abs(now - utc_timestamp) > - config.ARCHIVER_ALLOWABLE_SANE_DATE_SKEW) + Defaults.ARCHIVER_ALLOWABLE_SANE_DATE_SKEW) except (ValueError, OverflowError): # The likely cause of this is that the year in the Date: field # is horribly incorrect, e.g. (from SF bug # 571634): @@ -78,5 +81,10 @@ class ArchiveRunner(Runner): # While a list archiving lock is acquired, archive the message. with Lock(os.path.join(mlist.data_path, 'archive.lck')): for archive_factory in get_plugins('mailman.archiver'): - archive_factory().archive_message(mlist, msg) - + # A problem in one archiver should not prevent any other + # archiver from running. + try: + archive = archive_factory() + archive.archive_message(mlist, msg) + except Exception: + log.exception('Broken archiver: %s' % archive.name) diff --git a/mailman/queue/bounce.py b/mailman/queue/bounce.py index 0c5788174..e098ba866 100644 --- a/mailman/queue/bounce.py +++ b/mailman/queue/bounce.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2008 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2009 by the Free Software Foundation, Inc. # # This file is part of GNU Mailman. # @@ -27,10 +27,11 @@ from email.MIMEMessage import MIMEMessage from email.MIMEText import MIMEText from email.Utils import parseaddr +from mailman import Defaults from mailman import Utils from mailman.Bouncers import BouncerAPI from mailman.Message import UserNotification -from mailman.configuration import config +from mailman.config import config from mailman.i18n import _ from mailman.queue import Runner, Switchboard @@ -80,7 +81,7 @@ class BounceMixin: self._bounce_events_fp = None self._bouncecnt = 0 self._nextaction = (datetime.datetime.now() + - config.REGISTER_BOUNCES_EVERY) + Defaults.REGISTER_BOUNCES_EVERY) def _queue_bounces(self, listname, addrs, msg): today = datetime.date.today() @@ -132,7 +133,7 @@ class BounceMixin: if self._nextaction > now or self._bouncecnt == 0: return # Let's go ahead and register the bounces we've got stored up - self._nextaction = now + config.REGISTER_BOUNCES_EVERY + self._nextaction = now + Defaults.REGISTER_BOUNCES_EVERY self._register_bounces() def _probe_bounce(self, mlist, token): @@ -153,7 +154,7 @@ class BounceMixin: class BounceRunner(Runner, BounceMixin): - QDIR = config.BOUNCEQUEUE_DIR + """The bounce runner.""" def __init__(self, slice=None, numslices=1): Runner.__init__(self, slice, numslices) @@ -162,7 +163,6 @@ class BounceRunner(Runner, BounceMixin): def _dispose(self, mlist, msg, msgdata): # Make sure we have the most up-to-date state mlist.Load() - outq = Switchboard(config.OUTQUEUE_DIR) # There are a few possibilities here: # # - the message could have been VERP'd in which case, we know exactly @@ -176,14 +176,15 @@ class BounceRunner(Runner, BounceMixin): # owner address. That way, if a list owner address bounces, at least # some human has a chance to deal with it. Is this a bounce for a # message to a list owner, coming to the site owner? - if msg.get('to', '') == config.SITE_OWNER_ADDRESS: + if msg.get('to', '') == config.mailman.site_owner: # Send it on to the site owners, but craft the envelope sender to # be the noreply address, so if the site owner bounce, we won't # get stuck in a bounce loop. - outq.enqueue(msg, msgdata, - recips=[config.SITE_OWNER_ADDRESS], - envsender=config.NO_REPLY_ADDRESS, - ) + config.switchboards['out'].enqueue( + msg, msgdata, + recips=[config.mailman.site_owner], + envsender=config.mailman.noreply_address, + ) # List isn't doing bounce processing? if not mlist.bounce_processing: return @@ -241,7 +242,7 @@ def verp_bounce(mlist, msg): to = parseaddr(field)[1] if not to: continue # empty header - mo = re.search(config.VERP_REGEXP, to) + mo = re.search(Defaults.VERP_REGEXP, to) if not mo: continue # no match of regexp try: @@ -251,7 +252,7 @@ def verp_bounce(mlist, msg): addr = '%s@%s' % mo.group('mailbox', 'host') except IndexError: elog.error("VERP_REGEXP doesn't yield the right match groups: %s", - config.VERP_REGEXP) + Defaults.VERP_REGEXP) return [] return [addr] @@ -272,7 +273,7 @@ def verp_probe(mlist, msg): to = parseaddr(field)[1] if not to: continue # empty header - mo = re.search(config.VERP_PROBE_REGEXP, to) + mo = re.search(Defaults.VERP_PROBE_REGEXP, to) if not mo: continue # no match of regexp try: @@ -286,7 +287,7 @@ def verp_probe(mlist, msg): except IndexError: elog.error( "VERP_PROBE_REGEXP doesn't yield the right match groups: %s", - config.VERP_PROBE_REGEXP) + Defaults.VERP_PROBE_REGEXP) return None diff --git a/mailman/queue/command.py b/mailman/queue/command.py index a02827de0..db6feba57 100644 --- a/mailman/queue/command.py +++ b/mailman/queue/command.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2008 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2009 by the Free Software Foundation, Inc. # # This file is part of GNU Mailman. # @@ -40,10 +40,11 @@ from email.MIMEMessage import MIMEMessage from email.MIMEText import MIMEText from zope.interface import implements +from mailman import Defaults from mailman import Message from mailman import Utils from mailman.app.replybot import autorespond_to_sender -from mailman.configuration import config +from mailman.config import config from mailman.i18n import _ from mailman.interfaces import ContinueProcessing, IEmailResults from mailman.queue import Runner @@ -70,7 +71,7 @@ class CommandFinder: elif msgdata.get('toleave'): self.command_lines.append('leave') elif msgdata.get('toconfirm'): - mo = re.match(config.VERP_CONFIRM_REGEXP, msg.get('to', '')) + mo = re.match(Defaults.VERP_CONFIRM_REGEXP, msg.get('to', '')) if mo: self.command_lines.append('confirm ' + mo.group('cookie')) # Extract the subject header and do RFC 2047 decoding. @@ -99,8 +100,8 @@ class CommandFinder: assert isinstance(body, basestring), 'Non-string decoded payload' lines = body.splitlines() # Use no more lines than specified - self.command_lines.extend(lines[:config.EMAIL_COMMANDS_MAX_LINES]) - self.ignored_lines.extend(lines[config.EMAIL_COMMANDS_MAX_LINES:]) + self.command_lines.extend(lines[:Defaults.EMAIL_COMMANDS_MAX_LINES]) + self.ignored_lines.extend(lines[Defaults.EMAIL_COMMANDS_MAX_LINES:]) def __iter__(self): """Return each command line, split into commands and arguments. @@ -141,7 +142,7 @@ The results of your email command are provided below. class CommandRunner(Runner): - QDIR = config.CMDQUEUE_DIR + """The email command runner.""" def _dispose(self, mlist, msg, msgdata): message_id = msg.get('message-id', 'n/a') diff --git a/mailman/queue/docs/archiver.txt b/mailman/queue/docs/archiver.txt index 8c34e3537..ed7c26d45 100644 --- a/mailman/queue/docs/archiver.txt +++ b/mailman/queue/docs/archiver.txt @@ -5,7 +5,6 @@ Mailman can archive to any number of archivers that adhere to the IArchiver interface. By default, there's a Pipermail archiver. >>> from mailman.app.lifecycle import create_list - >>> from mailman.configuration import config >>> mlist = create_list(u'test@example.com') >>> mlist.web_page_url = u'http://www.example.com/' >>> config.db.commit() @@ -19,8 +18,7 @@ interface. By default, there's a Pipermail archiver. ... First post! ... """) - >>> from mailman.queue import Switchboard - >>> archiver_queue = Switchboard(config.ARCHQUEUE_DIR) + >>> archiver_queue = config.switchboards['archive'] >>> ignore = archiver_queue.enqueue(msg, {}, listname=mlist.fqdn_listname) >>> from mailman.queue.archive import ArchiveRunner diff --git a/mailman/queue/docs/command.txt b/mailman/queue/docs/command.txt index 470a632b7..0b384de01 100644 --- a/mailman/queue/docs/command.txt +++ b/mailman/queue/docs/command.txt @@ -24,9 +24,8 @@ sender. The command can be in the Subject header. ... ... """) - >>> from mailman.configuration import config >>> from mailman.inject import inject_message - >>> inject_message(mlist, msg, qdir=config.CMDQUEUE_DIR) + >>> inject_message(mlist, msg, switchboard='command') >>> from mailman.queue.command import CommandRunner >>> from mailman.testing.helpers import make_testable_runner >>> command = make_testable_runner(CommandRunner) @@ -35,11 +34,11 @@ sender. The command can be in the Subject header. And now the response is in the virgin queue. >>> from mailman.queue import Switchboard - >>> virgin_queue = Switchboard(config.VIRGINQUEUE_DIR) + >>> virgin_queue = config.switchboards['virgin'] >>> len(virgin_queue.files) 1 >>> from mailman.testing.helpers import get_queue_messages - >>> item = get_queue_messages(virgin_queue)[0] + >>> item = get_queue_messages('virgin')[0] >>> print item.msg.as_string() Subject: The results of your email commands From: test-bounces@example.com @@ -79,11 +78,11 @@ message is plain text. ... echo foo bar ... """) - >>> inject_message(mlist, msg, qdir=config.CMDQUEUE_DIR) + >>> inject_message(mlist, msg, switchboard='command') >>> command.run() >>> len(virgin_queue.files) 1 - >>> item = get_queue_messages(virgin_queue)[0] + >>> item = get_queue_messages('virgin')[0] >>> print item.msg.as_string() Subject: The results of your email commands From: test-bounces@example.com @@ -122,11 +121,11 @@ at by the command queue. ... echo baz qux ... """) - >>> inject_message(mlist, msg, qdir=config.CMDQUEUE_DIR) + >>> inject_message(mlist, msg, switchboard='command') >>> command.run() >>> len(virgin_queue.files) 1 - >>> item = get_queue_messages(virgin_queue)[0] + >>> item = get_queue_messages('virgin')[0] >>> print item.msg.as_string() Subject: The results of your email commands ... @@ -152,11 +151,11 @@ The 'stop' command is an alias for 'end'. ... echo baz qux ... """) - >>> inject_message(mlist, msg, qdir=config.CMDQUEUE_DIR) + >>> inject_message(mlist, msg, switchboard='command') >>> command.run() >>> len(virgin_queue.files) 1 - >>> item = get_queue_messages(virgin_queue)[0] + >>> item = get_queue_messages('virgin')[0] >>> print item.msg.as_string() Subject: The results of your email commands ... diff --git a/mailman/queue/docs/incoming.txt b/mailman/queue/docs/incoming.txt index 997eac07e..22b32d828 100644 --- a/mailman/queue/docs/incoming.txt +++ b/mailman/queue/docs/incoming.txt @@ -43,21 +43,19 @@ The incoming queue runner runs until it is empty. >>> from mailman.queue.incoming import IncomingRunner >>> from mailman.testing.helpers import make_testable_runner - >>> incoming = make_testable_runner(IncomingRunner) + >>> incoming = make_testable_runner(IncomingRunner, 'in') >>> incoming.run() And now the message is in the pipeline queue. - >>> from mailman.configuration import config - >>> from mailman.queue import Switchboard - >>> pipeline_queue = Switchboard(config.PIPELINEQUEUE_DIR) + >>> pipeline_queue = config.switchboards['pipeline'] >>> len(pipeline_queue.files) 1 - >>> incoming_queue = Switchboard(config.INQUEUE_DIR) + >>> incoming_queue = config.switchboards['in'] >>> len(incoming_queue.files) 0 >>> from mailman.testing.helpers import get_queue_messages - >>> item = get_queue_messages(pipeline_queue)[0] + >>> item = get_queue_messages('pipeline')[0] >>> print item.msg.as_string() From: aperson@example.com To: _xtest@example.com @@ -153,8 +151,8 @@ just create a new chain that does. The virgin queue needs to be cleared out due to artifacts from the previous tests above. - >>> virgin_queue = Switchboard(config.VIRGINQUEUE_DIR) - >>> ignore = get_queue_messages(virgin_queue) + >>> virgin_queue = config.switchboards['virgin'] + >>> ignore = get_queue_messages('virgin') >>> inject_message(mlist, msg) >>> file_pos = fp.tell() @@ -166,7 +164,7 @@ tests above. >>> len(virgin_queue.files) 1 - >>> item = get_queue_messages(virgin_queue)[0] + >>> item = get_queue_messages('virgin')[0] >>> print item.msg.as_string() Subject: My first post From: _xtest-owner@example.com diff --git a/mailman/queue/docs/lmtp.txt b/mailman/queue/docs/lmtp.txt index bb77203b4..75e91fd4e 100644 --- a/mailman/queue/docs/lmtp.txt +++ b/mailman/queue/docs/lmtp.txt @@ -25,7 +25,7 @@ It also helps to have a nice LMTP client. Posting address --------------- -If the mail server tries to send a message to a non-existant mailing list, it +If the mail server tries to send a message to a nonexistent mailing list, it will get a 550 error. >>> lmtp.sendmail( diff --git a/mailman/queue/docs/news.txt b/mailman/queue/docs/news.txt index 0b89de2bc..3375b3d54 100644 --- a/mailman/queue/docs/news.txt +++ b/mailman/queue/docs/news.txt @@ -6,7 +6,6 @@ NNTP newsgroup. One of the most important things this runner does is prepare the message for Usenet (yes, I know that NNTP is not Usenet, but this runner was originally written to gate to Usenet, which has its own rules). - >>> from mailman.configuration import config >>> from mailman.queue.news import prepare_message >>> mlist = config.db.list_manager.create(u'_xtest@example.com') >>> mlist.linked_newsgroup = u'comp.lang.python' diff --git a/mailman/queue/docs/outgoing.txt b/mailman/queue/docs/outgoing.txt index edc806d47..851bdf474 100644 --- a/mailman/queue/docs/outgoing.txt +++ b/mailman/queue/docs/outgoing.txt @@ -43,13 +43,11 @@ Normally, messages would show up in the outgoing queue after the message has been processed by the rule set and pipeline. But we can simulate that here by injecting a message directly into the outgoing queue. - >>> from mailman.configuration import config >>> msgdata = {} >>> handler = config.handlers['calculate-recipients'] >>> handler.process(mlist, msg, msgdata) - >>> from mailman.queue import Switchboard - >>> outgoing_queue = Switchboard(config.OUTQUEUE_DIR) + >>> outgoing_queue = config.switchboards['out'] >>> ignore = outgoing_queue.enqueue( ... msg, msgdata, ... verp=True, listname=mlist.fqdn_listname, tolist=True, @@ -60,7 +58,7 @@ upstream SMTP, which happens to be our test server. >>> from mailman.queue.outgoing import OutgoingRunner >>> from mailman.testing.helpers import make_testable_runner - >>> outgoing = make_testable_runner(OutgoingRunner) + >>> outgoing = make_testable_runner(OutgoingRunner, 'out') >>> outgoing.run() Three messages have been delivered to our SMTP server, one for each recipient. diff --git a/mailman/queue/docs/runner.txt b/mailman/queue/docs/runner.txt index a9e17370b..db493110c 100644 --- a/mailman/queue/docs/runner.txt +++ b/mailman/queue/docs/runner.txt @@ -14,21 +14,21 @@ The basic architecture of qrunner is implemented in the base class that all runners inherit from. This base class implements a .run() method that runs continuously in a loop until the .stop() method is called. - >>> import os - >>> from mailman.queue import Runner, Switchboard - >>> from mailman.configuration import config >>> mlist = config.db.list_manager.create(u'_xtest@example.com') >>> mlist.preferred_language = u'en' -Here is a very simple derived qrunner class. The class attribute QDIR tells -the qrunner which queue directory it is responsible for. Derived classes -should also implement various methods to provide the special functionality. -This is about as simple as a qrunner can be. +Here is a very simple derived qrunner class. Queue runners use a +configuration section in the configuration files to determine run +characteristics, such as the queue directory to use. Here we push a +configuration section for the test runner. - >>> queue_directory = os.path.join(config.QUEUE_DIR, 'test') + >>> config.push('test-runner', """ + ... [qrunner.test] + ... max_restarts: 1 + ... """) + + >>> from mailman.queue import Runner >>> class TestableRunner(Runner): - ... QDIR = queue_directory - ... ... def _dispose(self, mlist, msg, msgdata): ... self.msg = msg ... self.msgdata = msgdata @@ -40,8 +40,7 @@ This is about as simple as a qrunner can be. ... def _snooze(self, filecnt): ... return - >>> runner = TestableRunner() - >>> switchboard = Switchboard(queue_directory) + >>> runner = TestableRunner('test') This qrunner doesn't do much except run once, storing the message and metadata on instance variables. @@ -52,6 +51,7 @@ on instance variables. ... ... A test message. ... """) + >>> switchboard = config.switchboards['test'] >>> filebase = switchboard.enqueue(msg, listname=mlist.fqdn_listname, ... foo='yes', bar='no') >>> runner.run() diff --git a/mailman/queue/docs/switchboard.txt b/mailman/queue/docs/switchboard.txt index 633bdabe6..7baee7b54 100644 --- a/mailman/queue/docs/switchboard.txt +++ b/mailman/queue/docs/switchboard.txt @@ -14,7 +14,6 @@ instance of a switchboard is responsible for one queue directory. Create a switchboard by giving its queue directory. >>> import os - >>> from mailman.configuration import config >>> queue_directory = os.path.join(config.QUEUE_DIR, 'test') >>> from mailman.queue import Switchboard >>> switchboard = Switchboard(queue_directory) diff --git a/mailman/queue/http.py b/mailman/queue/http.py index cb6940b29..941b6d131 100644 --- a/mailman/queue/http.py +++ b/mailman/queue/http.py @@ -1,4 +1,4 @@ -# Copyright (C) 2006-2008 by the Free Software Foundation, Inc. +# Copyright (C) 2006-2009 by the Free Software Foundation, Inc. # # This file is part of GNU Mailman. # @@ -25,7 +25,7 @@ from cStringIO import StringIO from wsgiref.simple_server import make_server, WSGIRequestHandler from mailman.Cgi.wsgi_app import mailman_app -from mailman.configuration import config +from mailman.config import config from mailman.queue import Runner hlog = logging.getLogger('mailman.http') diff --git a/mailman/queue/incoming.py b/mailman/queue/incoming.py index d4decd435..ae78e1a81 100644 --- a/mailman/queue/incoming.py +++ b/mailman/queue/incoming.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2008 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2009 by the Free Software Foundation, Inc. # # This file is part of GNU Mailman. # @@ -26,14 +26,14 @@ prepared for delivery. Rejections, discards, and holds are processed immediately. """ -from mailman.configuration import config +from mailman.config import config from mailman.core.chains import process from mailman.queue import Runner class IncomingRunner(Runner): - QDIR = config.INQUEUE_DIR + """The incoming queue runner.""" def _dispose(self, mlist, msg, msgdata): if msgdata.get('envsender') is None: diff --git a/mailman/queue/lmtp.py b/mailman/queue/lmtp.py index b4688653b..f0895ee1f 100644 --- a/mailman/queue/lmtp.py +++ b/mailman/queue/lmtp.py @@ -1,4 +1,4 @@ -# Copyright (C) 2006-2008 by the Free Software Foundation, Inc. +# Copyright (C) 2006-2009 by the Free Software Foundation, Inc. # # This file is part of GNU Mailman. # @@ -42,10 +42,11 @@ import asyncore from email.utils import parseaddr +from mailman import Defaults from mailman.Message import Message -from mailman.configuration import config +from mailman.config import config from mailman.database.transaction import txn -from mailman.queue import Runner, Switchboard +from mailman.queue import Runner elog = logging.getLogger('mailman.error') qlog = logging.getLogger('mailman.qrunner') @@ -63,7 +64,7 @@ CRLF = '\r\n' ERR_451 = '451 Requested action aborted: error in processing' ERR_501 = '501 Message has defects' ERR_502 = '502 Error: command HELO not implemented' -ERR_550 = config.LMTP_ERR_550 +ERR_550 = Defaults.LMTP_ERR_550 # XXX Blech smtpd.__version__ = 'Python LMTP queue runner 1.0' @@ -86,7 +87,7 @@ def split_recipient(address): subaddress may be None if this is the list's posting address. """ localpart, domain = address.split('@', 1) - localpart = localpart.split(config.VERP_DELIMITER, 1)[0] + localpart = localpart.split(Defaults.VERP_DELIMITER, 1)[0] parts = localpart.split(DASH) if parts[-1] in SUBADDRESS_NAMES: listname = DASH.join(parts[:-1]) @@ -121,11 +122,11 @@ class LMTPRunner(Runner, smtpd.SMTPServer): # connections from the MTA. slice and numslices are ignored and are # necessary only to satisfy the API. def __init__(self, slice=None, numslices=1): - localaddr = config.LMTP_HOST, config.LMTP_PORT + localaddr = config.mta.lmtp_host, int(config.mta.lmtp_port) # Do not call Runner's constructor because there's no QDIR to create smtpd.SMTPServer.__init__(self, localaddr, remoteaddr=None) qlog.debug('LMTP server listening on %s:%s', - config.LMTP_HOST, config.LMTP_PORT) + localaddr[0], localaddr[1]) def handle_accept(self): conn, addr = self.accept() @@ -138,6 +139,7 @@ class LMTPRunner(Runner, smtpd.SMTPServer): # Refresh the list of list names every time we process a message # since the set of mailing lists could have changed. listnames = set(config.db.list_manager.names) + qlog.debug('listnames: %s', listnames) # Parse the message data. If there are any defects in the # message, reject it right away; it's probably spam. msg = email.message_from_string(data, Message) @@ -169,29 +171,29 @@ class LMTPRunner(Runner, smtpd.SMTPServer): queue = None msgdata = dict(listname=listname) if subaddress in ('bounces', 'admin'): - queue = Switchboard(config.BOUNCEQUEUE_DIR) + queue = 'bounce' elif subaddress == 'confirm': msgdata['toconfirm'] = True - queue = Switchboard(config.CMDQUEUE_DIR) + queue = 'command' elif subaddress in ('join', 'subscribe'): msgdata['tojoin'] = True - queue = Switchboard(config.CMDQUEUE_DIR) + queue = 'command' elif subaddress in ('leave', 'unsubscribe'): msgdata['toleave'] = True - queue = Switchboard(config.CMDQUEUE_DIR) + queue = 'command' elif subaddress == 'owner': msgdata.update(dict( toowner=True, - envsender=config.SITE_OWNER_ADDRESS, - pipeline=config.OWNER_PIPELINE, + envsender=config.mailman.site_owner, + pipeline=Defaults.OWNER_PIPELINE, )) - queue = Switchboard(config.INQUEUE_DIR) + queue = 'in' elif subaddress is None: msgdata['tolist'] = True - queue = Switchboard(config.INQUEUE_DIR) + queue = 'in' elif subaddress == 'request': msgdata['torequest'] = True - queue = Switchboard(config.CMDQUEUE_DIR) + queue = 'command' else: elog.error('Unknown sub-address: %s', subaddress) status.append(ERR_550) @@ -199,7 +201,7 @@ class LMTPRunner(Runner, smtpd.SMTPServer): # If we found a valid subaddress, enqueue the message and add # a success status for this recipient. if queue is not None: - queue.enqueue(msg, msgdata) + config.switchboards[queue].enqueue(msg, msgdata) status.append('250 Ok') except Exception, e: elog.exception('Queue detection: %s', msg['message-id']) diff --git a/mailman/queue/maildir.py b/mailman/queue/maildir.py index 71bac67dc..e12550cb5 100644 --- a/mailman/queue/maildir.py +++ b/mailman/queue/maildir.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002-2008 by the Free Software Foundation, Inc. +# Copyright (C) 2002-2009 by the Free Software Foundation, Inc. # # This file is part of GNU Mailman. # @@ -57,7 +57,7 @@ from email.Parser import Parser from email.Utils import parseaddr from mailman.Message import Message -from mailman.configuration import config +from mailman.config import config from mailman.queue import Runner log = logging.getLogger('mailman.error') diff --git a/mailman/queue/news.py b/mailman/queue/news.py index 8415f7d95..98735d498 100644 --- a/mailman/queue/news.py +++ b/mailman/queue/news.py @@ -1,4 +1,4 @@ -# Copyright (C) 2000-2008 by the Free Software Foundation, Inc. +# Copyright (C) 2000-2009 by the Free Software Foundation, Inc. # # This file is part of GNU Mailman. # @@ -28,8 +28,8 @@ from email.utils import getaddresses, make_msgid COMMASPACE = ', ' +from mailman import Defaults from mailman import Utils -from mailman.configuration import config from mailman.interfaces import NewsModeration from mailman.queue import Runner @@ -51,8 +51,6 @@ mcre = re.compile(r""" class NewsRunner(Runner): - QDIR = config.NEWSQUEUE_DIR - def _dispose(self, mlist, msg, msgdata): # Make sure we have the most up-to-date state mlist.Load() @@ -67,8 +65,8 @@ class NewsRunner(Runner): nntp_host, nntp_port = Utils.nntpsplit(mlist.nntp_host) conn = nntplib.NNTP(nntp_host, nntp_port, readermode=True, - user=config.NNTP_USERNAME, - password=config.NNTP_PASSWORD) + user=Defaults.NNTP_USERNAME, + password=Defaults.NNTP_PASSWORD) conn.post(fp) except nntplib.error_temp, e: log.error('(NNTPDirect) NNTP error for list "%s": %s', @@ -150,9 +148,9 @@ def prepare_message(mlist, msg, msgdata): # woon't completely sanitize the message, but it will eliminate the bulk # of the rejections based on message headers. The NNTP server may still # reject the message because of other problems. - for header in config.NNTP_REMOVE_HEADERS: + for header in Defaults.NNTP_REMOVE_HEADERS: del msg[header] - for header, rewrite in config.NNTP_REWRITE_DUPLICATE_HEADERS: + for header, rewrite in Defaults.NNTP_REWRITE_DUPLICATE_HEADERS: values = msg.get_all(header, []) if len(values) < 2: # We only care about duplicates diff --git a/mailman/queue/outgoing.py b/mailman/queue/outgoing.py index 3ab67eaad..14ba869dc 100644 --- a/mailman/queue/outgoing.py +++ b/mailman/queue/outgoing.py @@ -1,4 +1,4 @@ -# Copyright (C) 2000-2008 by the Free Software Foundation, Inc. +# Copyright (C) 2000-2009 by the Free Software Foundation, Inc. # # This file is part of GNU Mailman. # @@ -26,8 +26,9 @@ import logging from datetime import datetime +from mailman import Defaults from mailman import Message -from mailman.configuration import config +from mailman.config import config from mailman.core import errors from mailman.queue import Runner, Switchboard from mailman.queue.bounce import BounceMixin @@ -41,19 +42,19 @@ log = logging.getLogger('mailman.error') class OutgoingRunner(Runner, BounceMixin): - QDIR = config.OUTQUEUE_DIR + """The outgoing queue runner.""" def __init__(self, slice=None, numslices=1): Runner.__init__(self, slice, numslices) BounceMixin.__init__(self) # We look this function up only at startup time - handler = config.handlers[config.DELIVERY_MODULE] + handler = config.handlers[Defaults.DELIVERY_MODULE] self._func = handler.process # This prevents smtp server connection problems from filling up the # error log. It gets reset if the message was successfully sent, and # set if there was a socket.error. self._logged = False - self._retryq = Switchboard(config.RETRYQUEUE_DIR) + self._retryq = config.switchboards['retry'] def _dispose(self, mlist, msg, msgdata): # See if we should retry delivery of this message again. @@ -73,13 +74,13 @@ class OutgoingRunner(Runner, BounceMixin): # There was a problem connecting to the SMTP server. Log this # once, but crank up our sleep time so we don't fill the error # log. - port = config.SMTPPORT + port = int(config.mta.port) if port == 0: port = 'smtp' # Log this just once. if not self._logged: log.error('Cannot connect to SMTP server %s on port %s', - config.SMTPHOST, port) + config.mta.host, port) self._logged = True return True except errors.SomeRecipientsFailed, e: @@ -115,7 +116,7 @@ class OutgoingRunner(Runner, BounceMixin): return False else: # Keep trying to delivery this message for a while - deliver_until = now + config.DELIVERY_RETRY_PERIOD + deliver_until = now + Defaults.DELIVERY_RETRY_PERIOD msgdata['last_recip_count'] = len(recips) msgdata['deliver_until'] = deliver_until msgdata['recips'] = recips diff --git a/mailman/queue/pipeline.py b/mailman/queue/pipeline.py index 665bfce77..f0a42f729 100644 --- a/mailman/queue/pipeline.py +++ b/mailman/queue/pipeline.py @@ -1,4 +1,4 @@ -# Copyright (C) 2008 by the Free Software Foundation, Inc. +# Copyright (C) 2008-2009 by the Free Software Foundation, Inc. # # This file is part of GNU Mailman. # @@ -22,15 +22,13 @@ through the 'preparation pipeline'. This pipeline adds, deletes and modifies headers, calculates message recipients, and more. """ -from mailman.app.pipelines import process -from mailman.configuration import config +from mailman.core.pipelines import process +from mailman.config import config from mailman.queue import Runner class PipelineRunner(Runner): - QDIR = config.PIPELINEQUEUE_DIR - def _dispose(self, mlist, msg, msgdata): # Process the message through the mailing list's pipeline. process(mlist, msg, msgdata, mlist.pipeline) diff --git a/mailman/queue/retry.py b/mailman/queue/retry.py index f8e1b4665..2b5a6afad 100644 --- a/mailman/queue/retry.py +++ b/mailman/queue/retry.py @@ -1,4 +1,4 @@ -# Copyright (C) 2003-2008 by the Free Software Foundation, Inc. +# Copyright (C) 2003-2009 by the Free Software Foundation, Inc. # # This file is part of GNU Mailman. # @@ -17,24 +17,21 @@ import time -from mailman.configuration import config -from mailman.queue import Runner, Switchboard +from mailman.config import config +from mailman.queue import Runner class RetryRunner(Runner): - QDIR = config.RETRYQUEUE_DIR - SLEEPTIME = config.minutes(15) - def __init__(self, slice=None, numslices=1): Runner.__init__(self, slice, numslices) - self.__outq = Switchboard(config.OUTQUEUE_DIR) + self._outq = config.switchboards['out'] def _dispose(self, mlist, msg, msgdata): # Move it to the out queue for another retry - self.__outq.enqueue(msg, msgdata) + self._outq.enqueue(msg, msgdata) return False def _snooze(self, filecnt): # We always want to snooze - time.sleep(float(self.SLEEPTIME)) + time.sleep(self.sleep_float) diff --git a/mailman/queue/virgin.py b/mailman/queue/virgin.py index 503337ea1..ca61074bf 100644 --- a/mailman/queue/virgin.py +++ b/mailman/queue/virgin.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2008 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2009 by the Free Software Foundation, Inc. # # This file is part of GNU Mailman. # @@ -23,15 +23,13 @@ to go through some minimal processing before they can be sent out to the recipient. """ -from mailman.app.pipelines import process -from mailman.configuration import config +from mailman.core.pipelines import process +from mailman.config import config from mailman.queue import Runner class VirginRunner(Runner): - QDIR = config.VIRGINQUEUE_DIR - def _dispose(self, mlist, msg, msgdata): # We need to fast track this message through any pipeline handlers # that touch it, e.g. especially cook-headers. |
