summaryrefslogtreecommitdiff
path: root/mailman/queue
diff options
context:
space:
mode:
authorBarry Warsaw2009-01-01 17:58:39 -0500
committerBarry Warsaw2009-01-01 17:58:39 -0500
commit1c285f110d8e98597453c6b4b69ea01163033547 (patch)
tree00c7ec16711b2073e40f593658f652726a9d4231 /mailman/queue
parent12513c7d0fc1f5d2a1aabda349637309f6e8300b (diff)
parent600ddb503a391d70230d96ee91a631888d11b35a (diff)
downloadmailman-1c285f110d8e98597453c6b4b69ea01163033547.tar.gz
mailman-1c285f110d8e98597453c6b4b69ea01163033547.tar.zst
mailman-1c285f110d8e98597453c6b4b69ea01163033547.zip
Two major structural conversions.
* Use zc.buildout and zc.testing frameworks for building and testing Mailman. * Use lazr.config as the configuration system, though this conversion is not yet complete. Also: move a bunch of old bin scripts to the attic.
Diffstat (limited to 'mailman/queue')
-rw-r--r--mailman/queue/__init__.py124
-rw-r--r--mailman/queue/archive.py24
-rw-r--r--mailman/queue/bounce.py31
-rw-r--r--mailman/queue/command.py13
-rw-r--r--mailman/queue/docs/archiver.txt4
-rw-r--r--mailman/queue/docs/command.txt19
-rw-r--r--mailman/queue/docs/incoming.txt16
-rw-r--r--mailman/queue/docs/lmtp.txt2
-rw-r--r--mailman/queue/docs/news.txt1
-rw-r--r--mailman/queue/docs/outgoing.txt6
-rw-r--r--mailman/queue/docs/runner.txt24
-rw-r--r--mailman/queue/docs/switchboard.txt1
-rw-r--r--mailman/queue/http.py4
-rw-r--r--mailman/queue/incoming.py6
-rw-r--r--mailman/queue/lmtp.py36
-rw-r--r--mailman/queue/maildir.py4
-rw-r--r--mailman/queue/news.py14
-rw-r--r--mailman/queue/outgoing.py17
-rw-r--r--mailman/queue/pipeline.py8
-rw-r--r--mailman/queue/retry.py15
-rw-r--r--mailman/queue/virgin.py8
21 files changed, 196 insertions, 181 deletions
diff --git a/mailman/queue/__init__.py b/mailman/queue/__init__.py
index 961a46283..52686b3ab 100644
--- a/mailman/queue/__init__.py
+++ b/mailman/queue/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2001-2008 by the Free Software Foundation, Inc.
+# Copyright (C) 2001-2009 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
@@ -32,6 +32,7 @@ __all__ = [
import os
+import sys
import time
import email
import errno
@@ -43,13 +44,16 @@ import marshal
import traceback
from cStringIO import StringIO
+from lazr.config import as_boolean, as_timedelta
+from string import Template
from zope.interface import implements
-from mailman import i18n
from mailman import Message
from mailman import Utils
-from mailman.configuration import config
-from mailman.interfaces import IRunner, ISwitchboard
+from mailman import i18n
+from mailman.config import config
+from mailman.interfaces.runner import IRunner
+from mailman.interfaces.switchboard import ISwitchboard
# 20 bytes of all bits set, maximum hashlib.sha.digest() value
shamax = 0xffffffffffffffffffffffffffffffffffffffffL
@@ -57,6 +61,7 @@ shamax = 0xffffffffffffffffffffffffffffffffffffffffL
# Small increment to add to time in case two entries have the same time. This
# prevents skipping one of two entries with the same time until the next pass.
DELTA = .0001
+DOT = '.'
elog = logging.getLogger('mailman.error')
dlog = logging.getLogger('mailman.debug')
@@ -66,11 +71,24 @@ dlog = logging.getLogger('mailman.debug')
class Switchboard:
implements(ISwitchboard)
- def __init__(self, whichq, slice=None, numslices=1, recover=False):
+ @staticmethod
+ def initialize():
+ """Initialize the global switchboards for input/output."""
+ for conf in config.qrunner_configs:
+ name = conf.name.split('.')[-1]
+ assert name not in config.switchboards, (
+ 'Duplicate qrunner name: %s' % name)
+ substitutions = config.paths
+ substitutions['name'] = name
+ path = Template(conf.path).safe_substitute(substitutions)
+ config.switchboards[name] = Switchboard(path)
+
+ def __init__(self, queue_directory,
+ slice=None, numslices=1, recover=False):
"""Create a switchboard object.
- :param whichq: The queue directory.
- :type whichq: str
+ :param queue_directory: The queue directory.
+ :type queue_directory: str
:param slice: The slice number for this switchboard, or None. If not
None, it must be [0..`numslices`).
:type slice: int or None
@@ -80,9 +98,11 @@ class Switchboard:
:param recover: True if backup files should be recovered.
:type recover: bool
"""
- self._whichq = whichq
+ assert (numslices & (numslices - 1)) == 0, (
+ 'Not a power of 2: %s' % numslices)
+ self.queue_directory = queue_directory
# Create the directory if it doesn't yet exist.
- Utils.makedirs(self._whichq, 0770)
+ Utils.makedirs(self.queue_directory, 0770)
# Fast track for no slices
self._lower = None
self._upper = None
@@ -93,11 +113,6 @@ class Switchboard:
if recover:
self.recover_backup_files()
- @property
- def queue_directory(self):
- """See `ISwitchboard`."""
- return self._whichq
-
def enqueue(self, _msg, _metadata=None, **_kws):
"""See `ISwitchboard`."""
if _metadata is None:
@@ -125,7 +140,7 @@ class Switchboard:
# and the sha hex digest.
rcvtime = data.setdefault('received_time', now)
filebase = repr(rcvtime) + '+' + hashlib.sha1(hashfood).hexdigest()
- filename = os.path.join(self._whichq, filebase + '.pck')
+ filename = os.path.join(self.queue_directory, filebase + '.pck')
tmpfile = filename + '.tmp'
# Always add the metadata schema version number
data['version'] = config.QFILE_SCHEMA_VERSION
@@ -148,8 +163,8 @@ class Switchboard:
def dequeue(self, filebase):
"""See `ISwitchboard`."""
# Calculate the filename from the given filebase.
- filename = os.path.join(self._whichq, filebase + '.pck')
- backfile = os.path.join(self._whichq, filebase + '.bak')
+ filename = os.path.join(self.queue_directory, filebase + '.pck')
+ backfile = os.path.join(self.queue_directory, filebase + '.bak')
# Read the message object and metadata.
with open(filename) as fp:
# Move the file to the backup file name for processing. If this
@@ -172,13 +187,13 @@ class Switchboard:
return msg, data
def finish(self, filebase, preserve=False):
- bakfile = os.path.join(self._whichq, filebase + '.bak')
+ bakfile = os.path.join(self.queue_directory, filebase + '.bak')
try:
if preserve:
- psvfile = os.path.join(config.SHUNTQUEUE_DIR,
- filebase + '.psv')
+ shunt_dir = config.switchboards['shunt'].queue_directory
+ psvfile = os.path.join(shunt_dir, filebase + '.psv')
# Create the directory if it doesn't yet exist.
- Utils.makedirs(config.SHUNTQUEUE_DIR, 0770)
+ Utils.makedirs(shunt_dir, 0770)
os.rename(bakfile, psvfile)
else:
os.unlink(bakfile)
@@ -196,7 +211,7 @@ class Switchboard:
times = {}
lower = self._lower
upper = self._upper
- for f in os.listdir(self._whichq):
+ for f in os.listdir(self.queue_directory):
# By ignoring anything that doesn't end in .pck, we ignore
# tempfiles and avoid a race condition.
filebase, ext = os.path.splitext(f)
@@ -220,8 +235,8 @@ class Switchboard:
# to exist at the same time, so the move is enough to ensure that our
# normal dequeuing process will handle them.
for filebase in self.get_files('.bak'):
- src = os.path.join(self._whichq, filebase + '.bak')
- dst = os.path.join(self._whichq, filebase + '.pck')
+ src = os.path.join(self.queue_directory, filebase + '.bak')
+ dst = os.path.join(self.queue_directory, filebase + '.pck')
os.rename(src, dst)
@@ -229,27 +244,31 @@ class Switchboard:
class Runner:
implements(IRunner)
- QDIR = None
- SLEEPTIME = None
-
- def __init__(self, slice=None, numslices=1):
+ def __init__(self, name, slice=None):
"""Create a queue runner.
:param slice: The slice number for this queue runner. This is passed
directly to the underlying `ISwitchboard` object.
:type slice: int or None
- :param numslices: The number of slices for this queue. Must be a
- power of 2.
- :type numslices: int
"""
- # Create our own switchboard. Don't use the switchboard cache because
- # we want to provide slice and numslice arguments.
- self._switchboard = Switchboard(self.QDIR, slice, numslices, True)
- # Create the shunt switchboard
- self._shunt = Switchboard(config.SHUNTQUEUE_DIR)
+ # Grab the configuration section.
+ self.name = name
+ section = getattr(config, 'qrunner.' + name)
+ substitutions = config.paths
+ substitutions['name'] = name
+ self.queue_directory = Template(section.path).safe_substitute(
+ substitutions)
+ numslices = int(section.instances)
+ self.switchboard = Switchboard(
+ self.queue_directory, slice, numslices, True)
+ self.sleep_time = as_timedelta(section.sleep_time)
+ # sleep_time is a timedelta; turn it into a float for time.sleep().
+ self.sleep_float = (86400 * self.sleep_time.days +
+ self.sleep_time.seconds +
+ self.sleep_time.microseconds / 1000000.0)
+ self.max_restarts = int(section.max_restarts)
+ self.start = as_boolean(section.start)
self._stop = False
- if self.SLEEPTIME is None:
- self.SLEEPTIME = config.QRUNNER_SLEEP_TIME
def __repr__(self):
return '<%s at %s>' % (self.__class__.__name__, id(self))
@@ -286,13 +305,13 @@ class Runner:
dlog.debug('[%s] starting oneloop', me)
# List all the files in our queue directory. The switchboard is
# guaranteed to hand us the files in FIFO order.
- files = self._switchboard.files
+ files = self.switchboard.files
for filebase in files:
dlog.debug('[%s] processing filebase: %s', me, filebase)
try:
# Ask the switchboard for the message and metadata objects
# associated with this queue file.
- msg, msgdata = self._switchboard.dequeue(filebase)
+ msg, msgdata = self.switchboard.dequeue(filebase)
except Exception, e:
# This used to just catch email.Errors.MessageParseError, but
# other problems can occur in message parsing, e.g.
@@ -302,14 +321,14 @@ class Runner:
self._log(e)
elog.error('Skipping and preserving unparseable message: %s',
filebase)
- self._switchboard.finish(filebase, preserve=True)
+ self.switchboard.finish(filebase, preserve=True)
config.db.abort()
continue
try:
dlog.debug('[%s] processing onefile', me)
self._process_one_file(msg, msgdata)
dlog.debug('[%s] finishing filebase: %s', me, filebase)
- self._switchboard.finish(filebase)
+ self.switchboard.finish(filebase)
except Exception, e:
# All runners that implement _dispose() must guarantee that
# exceptions are caught and dealt with properly. Still, there
@@ -319,14 +338,15 @@ class Runner:
# intervention.
self._log(e)
# Put a marker in the metadata for unshunting.
- msgdata['whichq'] = self._switchboard.queue_directory
+ msgdata['whichq'] = self.switchboard.queue_directory
# It is possible that shunting can throw an exception, e.g. a
# permissions problem or a MemoryError due to a really large
# message. Try to be graceful.
try:
- new_filebase = self._shunt.enqueue(msg, msgdata)
+ shunt = config.switchboards['shunt']
+ new_filebase = shunt.enqueue(msg, msgdata)
elog.error('SHUNTING: %s', new_filebase)
- self._switchboard.finish(filebase)
+ self.switchboard.finish(filebase)
except Exception, e:
# The message wasn't successfully shunted. Log the
# exception and try to preserve the original queue entry
@@ -335,13 +355,13 @@ class Runner:
elog.error(
'SHUNTING FAILED, preserving original entry: %s',
filebase)
- self._switchboard.finish(filebase, preserve=True)
+ self.switchboard.finish(filebase, preserve=True)
config.db.abort()
# Other work we want to do each time through the loop.
dlog.debug('[%s] doing periodic', me)
self._do_periodic()
dlog.debug('[%s] checking short circuit', me)
- if self._short_curcuit():
+ if self._short_circuit():
dlog.debug('[%s] short circuiting', me)
break
dlog.debug('[%s] commiting', me)
@@ -363,7 +383,7 @@ class Runner:
if mlist is None:
elog.error('Dequeuing message destined for missing list: %s',
listname)
- self._shunt.enqueue(msg, msgdata)
+ config.switchboards['shunt'].enqueue(msg, msgdata)
return
# Now process this message. We also want to set up the language
# context for this message. The context will be the preferred
@@ -380,7 +400,7 @@ class Runner:
msgdata['lang'] = language
keepqueued = self._dispose(mlist, msg, msgdata)
if keepqueued:
- self._switchboard.enqueue(msg, msgdata)
+ self.switchboard.enqueue(msg, msgdata)
def _log(self, exc):
elog.error('Uncaught runner exception: %s', exc)
@@ -401,10 +421,10 @@ class Runner:
def _snooze(self, filecnt):
"""See `IRunner`."""
- if filecnt or float(self.SLEEPTIME) <= 0:
+ if filecnt or self.sleep_float <= 0:
return
- time.sleep(float(self.SLEEPTIME))
+ time.sleep(self.sleep_float)
- def _short_curcuit(self):
+ def _short_circuit(self):
"""See `IRunner`."""
return self._stop
diff --git a/mailman/queue/archive.py b/mailman/queue/archive.py
index 6dda70387..69ec46f4b 100644
--- a/mailman/queue/archive.py
+++ b/mailman/queue/archive.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2000-2008 by the Free Software Foundation, Inc.
+# Copyright (C) 2000-2009 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
@@ -25,19 +25,22 @@ __all__ = [
import os
import time
+import logging
from datetime import datetime
from email.Utils import parsedate_tz, mktime_tz, formatdate
from locknix.lockfile import Lock
-from mailman.configuration import config
+from mailman import Defaults
from mailman.core.plugins import get_plugins
from mailman.queue import Runner
+log = logging.getLogger('mailman.error')
+
class ArchiveRunner(Runner):
- QDIR = config.ARCHQUEUE_DIR
+ """The archive runner."""
def _dispose(self, mlist, msg, msgdata):
# Support clobber_date, i.e. setting the date in the archive to the
@@ -48,9 +51,9 @@ class ArchiveRunner(Runner):
received_time = formatdate(msgdata['received_time'])
if not original_date:
clobber = True
- elif config.ARCHIVER_CLOBBER_DATE_POLICY == 1:
+ elif Defaults.ARCHIVER_CLOBBER_DATE_POLICY == 1:
clobber = True
- elif config.ARCHIVER_CLOBBER_DATE_POLICY == 2:
+ elif Defaults.ARCHIVER_CLOBBER_DATE_POLICY == 2:
# What's the timestamp on the original message?
timetup = parsedate_tz(original_date)
now = datetime.now()
@@ -60,7 +63,7 @@ class ArchiveRunner(Runner):
else:
utc_timestamp = datetime.fromtimestamp(mktime_tz(timetup))
clobber = (abs(now - utc_timestamp) >
- config.ARCHIVER_ALLOWABLE_SANE_DATE_SKEW)
+ Defaults.ARCHIVER_ALLOWABLE_SANE_DATE_SKEW)
except (ValueError, OverflowError):
# The likely cause of this is that the year in the Date: field
# is horribly incorrect, e.g. (from SF bug # 571634):
@@ -78,5 +81,10 @@ class ArchiveRunner(Runner):
# While a list archiving lock is acquired, archive the message.
with Lock(os.path.join(mlist.data_path, 'archive.lck')):
for archive_factory in get_plugins('mailman.archiver'):
- archive_factory().archive_message(mlist, msg)
-
+ # A problem in one archiver should not prevent any other
+ # archiver from running.
+ try:
+ archive = archive_factory()
+ archive.archive_message(mlist, msg)
+ except Exception:
+ log.exception('Broken archiver: %s' % archive.name)
diff --git a/mailman/queue/bounce.py b/mailman/queue/bounce.py
index 0c5788174..e098ba866 100644
--- a/mailman/queue/bounce.py
+++ b/mailman/queue/bounce.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2001-2008 by the Free Software Foundation, Inc.
+# Copyright (C) 2001-2009 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
@@ -27,10 +27,11 @@ from email.MIMEMessage import MIMEMessage
from email.MIMEText import MIMEText
from email.Utils import parseaddr
+from mailman import Defaults
from mailman import Utils
from mailman.Bouncers import BouncerAPI
from mailman.Message import UserNotification
-from mailman.configuration import config
+from mailman.config import config
from mailman.i18n import _
from mailman.queue import Runner, Switchboard
@@ -80,7 +81,7 @@ class BounceMixin:
self._bounce_events_fp = None
self._bouncecnt = 0
self._nextaction = (datetime.datetime.now() +
- config.REGISTER_BOUNCES_EVERY)
+ Defaults.REGISTER_BOUNCES_EVERY)
def _queue_bounces(self, listname, addrs, msg):
today = datetime.date.today()
@@ -132,7 +133,7 @@ class BounceMixin:
if self._nextaction > now or self._bouncecnt == 0:
return
# Let's go ahead and register the bounces we've got stored up
- self._nextaction = now + config.REGISTER_BOUNCES_EVERY
+ self._nextaction = now + Defaults.REGISTER_BOUNCES_EVERY
self._register_bounces()
def _probe_bounce(self, mlist, token):
@@ -153,7 +154,7 @@ class BounceMixin:
class BounceRunner(Runner, BounceMixin):
- QDIR = config.BOUNCEQUEUE_DIR
+ """The bounce runner."""
def __init__(self, slice=None, numslices=1):
Runner.__init__(self, slice, numslices)
@@ -162,7 +163,6 @@ class BounceRunner(Runner, BounceMixin):
def _dispose(self, mlist, msg, msgdata):
# Make sure we have the most up-to-date state
mlist.Load()
- outq = Switchboard(config.OUTQUEUE_DIR)
# There are a few possibilities here:
#
# - the message could have been VERP'd in which case, we know exactly
@@ -176,14 +176,15 @@ class BounceRunner(Runner, BounceMixin):
# owner address. That way, if a list owner address bounces, at least
# some human has a chance to deal with it. Is this a bounce for a
# message to a list owner, coming to the site owner?
- if msg.get('to', '') == config.SITE_OWNER_ADDRESS:
+ if msg.get('to', '') == config.mailman.site_owner:
# Send it on to the site owners, but craft the envelope sender to
# be the noreply address, so if the site owner bounce, we won't
# get stuck in a bounce loop.
- outq.enqueue(msg, msgdata,
- recips=[config.SITE_OWNER_ADDRESS],
- envsender=config.NO_REPLY_ADDRESS,
- )
+ config.switchboards['out'].enqueue(
+ msg, msgdata,
+ recips=[config.mailman.site_owner],
+ envsender=config.mailman.noreply_address,
+ )
# List isn't doing bounce processing?
if not mlist.bounce_processing:
return
@@ -241,7 +242,7 @@ def verp_bounce(mlist, msg):
to = parseaddr(field)[1]
if not to:
continue # empty header
- mo = re.search(config.VERP_REGEXP, to)
+ mo = re.search(Defaults.VERP_REGEXP, to)
if not mo:
continue # no match of regexp
try:
@@ -251,7 +252,7 @@ def verp_bounce(mlist, msg):
addr = '%s@%s' % mo.group('mailbox', 'host')
except IndexError:
elog.error("VERP_REGEXP doesn't yield the right match groups: %s",
- config.VERP_REGEXP)
+ Defaults.VERP_REGEXP)
return []
return [addr]
@@ -272,7 +273,7 @@ def verp_probe(mlist, msg):
to = parseaddr(field)[1]
if not to:
continue # empty header
- mo = re.search(config.VERP_PROBE_REGEXP, to)
+ mo = re.search(Defaults.VERP_PROBE_REGEXP, to)
if not mo:
continue # no match of regexp
try:
@@ -286,7 +287,7 @@ def verp_probe(mlist, msg):
except IndexError:
elog.error(
"VERP_PROBE_REGEXP doesn't yield the right match groups: %s",
- config.VERP_PROBE_REGEXP)
+ Defaults.VERP_PROBE_REGEXP)
return None
diff --git a/mailman/queue/command.py b/mailman/queue/command.py
index a02827de0..db6feba57 100644
--- a/mailman/queue/command.py
+++ b/mailman/queue/command.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2008 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2009 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
@@ -40,10 +40,11 @@ from email.MIMEMessage import MIMEMessage
from email.MIMEText import MIMEText
from zope.interface import implements
+from mailman import Defaults
from mailman import Message
from mailman import Utils
from mailman.app.replybot import autorespond_to_sender
-from mailman.configuration import config
+from mailman.config import config
from mailman.i18n import _
from mailman.interfaces import ContinueProcessing, IEmailResults
from mailman.queue import Runner
@@ -70,7 +71,7 @@ class CommandFinder:
elif msgdata.get('toleave'):
self.command_lines.append('leave')
elif msgdata.get('toconfirm'):
- mo = re.match(config.VERP_CONFIRM_REGEXP, msg.get('to', ''))
+ mo = re.match(Defaults.VERP_CONFIRM_REGEXP, msg.get('to', ''))
if mo:
self.command_lines.append('confirm ' + mo.group('cookie'))
# Extract the subject header and do RFC 2047 decoding.
@@ -99,8 +100,8 @@ class CommandFinder:
assert isinstance(body, basestring), 'Non-string decoded payload'
lines = body.splitlines()
# Use no more lines than specified
- self.command_lines.extend(lines[:config.EMAIL_COMMANDS_MAX_LINES])
- self.ignored_lines.extend(lines[config.EMAIL_COMMANDS_MAX_LINES:])
+ self.command_lines.extend(lines[:Defaults.EMAIL_COMMANDS_MAX_LINES])
+ self.ignored_lines.extend(lines[Defaults.EMAIL_COMMANDS_MAX_LINES:])
def __iter__(self):
"""Return each command line, split into commands and arguments.
@@ -141,7 +142,7 @@ The results of your email command are provided below.
class CommandRunner(Runner):
- QDIR = config.CMDQUEUE_DIR
+ """The email command runner."""
def _dispose(self, mlist, msg, msgdata):
message_id = msg.get('message-id', 'n/a')
diff --git a/mailman/queue/docs/archiver.txt b/mailman/queue/docs/archiver.txt
index 8c34e3537..ed7c26d45 100644
--- a/mailman/queue/docs/archiver.txt
+++ b/mailman/queue/docs/archiver.txt
@@ -5,7 +5,6 @@ Mailman can archive to any number of archivers that adhere to the IArchiver
interface. By default, there's a Pipermail archiver.
>>> from mailman.app.lifecycle import create_list
- >>> from mailman.configuration import config
>>> mlist = create_list(u'test@example.com')
>>> mlist.web_page_url = u'http://www.example.com/'
>>> config.db.commit()
@@ -19,8 +18,7 @@ interface. By default, there's a Pipermail archiver.
... First post!
... """)
- >>> from mailman.queue import Switchboard
- >>> archiver_queue = Switchboard(config.ARCHQUEUE_DIR)
+ >>> archiver_queue = config.switchboards['archive']
>>> ignore = archiver_queue.enqueue(msg, {}, listname=mlist.fqdn_listname)
>>> from mailman.queue.archive import ArchiveRunner
diff --git a/mailman/queue/docs/command.txt b/mailman/queue/docs/command.txt
index 470a632b7..0b384de01 100644
--- a/mailman/queue/docs/command.txt
+++ b/mailman/queue/docs/command.txt
@@ -24,9 +24,8 @@ sender. The command can be in the Subject header.
...
... """)
- >>> from mailman.configuration import config
>>> from mailman.inject import inject_message
- >>> inject_message(mlist, msg, qdir=config.CMDQUEUE_DIR)
+ >>> inject_message(mlist, msg, switchboard='command')
>>> from mailman.queue.command import CommandRunner
>>> from mailman.testing.helpers import make_testable_runner
>>> command = make_testable_runner(CommandRunner)
@@ -35,11 +34,11 @@ sender. The command can be in the Subject header.
And now the response is in the virgin queue.
>>> from mailman.queue import Switchboard
- >>> virgin_queue = Switchboard(config.VIRGINQUEUE_DIR)
+ >>> virgin_queue = config.switchboards['virgin']
>>> len(virgin_queue.files)
1
>>> from mailman.testing.helpers import get_queue_messages
- >>> item = get_queue_messages(virgin_queue)[0]
+ >>> item = get_queue_messages('virgin')[0]
>>> print item.msg.as_string()
Subject: The results of your email commands
From: test-bounces@example.com
@@ -79,11 +78,11 @@ message is plain text.
... echo foo bar
... """)
- >>> inject_message(mlist, msg, qdir=config.CMDQUEUE_DIR)
+ >>> inject_message(mlist, msg, switchboard='command')
>>> command.run()
>>> len(virgin_queue.files)
1
- >>> item = get_queue_messages(virgin_queue)[0]
+ >>> item = get_queue_messages('virgin')[0]
>>> print item.msg.as_string()
Subject: The results of your email commands
From: test-bounces@example.com
@@ -122,11 +121,11 @@ at by the command queue.
... echo baz qux
... """)
- >>> inject_message(mlist, msg, qdir=config.CMDQUEUE_DIR)
+ >>> inject_message(mlist, msg, switchboard='command')
>>> command.run()
>>> len(virgin_queue.files)
1
- >>> item = get_queue_messages(virgin_queue)[0]
+ >>> item = get_queue_messages('virgin')[0]
>>> print item.msg.as_string()
Subject: The results of your email commands
...
@@ -152,11 +151,11 @@ The 'stop' command is an alias for 'end'.
... echo baz qux
... """)
- >>> inject_message(mlist, msg, qdir=config.CMDQUEUE_DIR)
+ >>> inject_message(mlist, msg, switchboard='command')
>>> command.run()
>>> len(virgin_queue.files)
1
- >>> item = get_queue_messages(virgin_queue)[0]
+ >>> item = get_queue_messages('virgin')[0]
>>> print item.msg.as_string()
Subject: The results of your email commands
...
diff --git a/mailman/queue/docs/incoming.txt b/mailman/queue/docs/incoming.txt
index 997eac07e..22b32d828 100644
--- a/mailman/queue/docs/incoming.txt
+++ b/mailman/queue/docs/incoming.txt
@@ -43,21 +43,19 @@ The incoming queue runner runs until it is empty.
>>> from mailman.queue.incoming import IncomingRunner
>>> from mailman.testing.helpers import make_testable_runner
- >>> incoming = make_testable_runner(IncomingRunner)
+ >>> incoming = make_testable_runner(IncomingRunner, 'in')
>>> incoming.run()
And now the message is in the pipeline queue.
- >>> from mailman.configuration import config
- >>> from mailman.queue import Switchboard
- >>> pipeline_queue = Switchboard(config.PIPELINEQUEUE_DIR)
+ >>> pipeline_queue = config.switchboards['pipeline']
>>> len(pipeline_queue.files)
1
- >>> incoming_queue = Switchboard(config.INQUEUE_DIR)
+ >>> incoming_queue = config.switchboards['in']
>>> len(incoming_queue.files)
0
>>> from mailman.testing.helpers import get_queue_messages
- >>> item = get_queue_messages(pipeline_queue)[0]
+ >>> item = get_queue_messages('pipeline')[0]
>>> print item.msg.as_string()
From: aperson@example.com
To: _xtest@example.com
@@ -153,8 +151,8 @@ just create a new chain that does.
The virgin queue needs to be cleared out due to artifacts from the previous
tests above.
- >>> virgin_queue = Switchboard(config.VIRGINQUEUE_DIR)
- >>> ignore = get_queue_messages(virgin_queue)
+ >>> virgin_queue = config.switchboards['virgin']
+ >>> ignore = get_queue_messages('virgin')
>>> inject_message(mlist, msg)
>>> file_pos = fp.tell()
@@ -166,7 +164,7 @@ tests above.
>>> len(virgin_queue.files)
1
- >>> item = get_queue_messages(virgin_queue)[0]
+ >>> item = get_queue_messages('virgin')[0]
>>> print item.msg.as_string()
Subject: My first post
From: _xtest-owner@example.com
diff --git a/mailman/queue/docs/lmtp.txt b/mailman/queue/docs/lmtp.txt
index bb77203b4..75e91fd4e 100644
--- a/mailman/queue/docs/lmtp.txt
+++ b/mailman/queue/docs/lmtp.txt
@@ -25,7 +25,7 @@ It also helps to have a nice LMTP client.
Posting address
---------------
-If the mail server tries to send a message to a non-existant mailing list, it
+If the mail server tries to send a message to a nonexistent mailing list, it
will get a 550 error.
>>> lmtp.sendmail(
diff --git a/mailman/queue/docs/news.txt b/mailman/queue/docs/news.txt
index 0b89de2bc..3375b3d54 100644
--- a/mailman/queue/docs/news.txt
+++ b/mailman/queue/docs/news.txt
@@ -6,7 +6,6 @@ NNTP newsgroup. One of the most important things this runner does is prepare
the message for Usenet (yes, I know that NNTP is not Usenet, but this runner
was originally written to gate to Usenet, which has its own rules).
- >>> from mailman.configuration import config
>>> from mailman.queue.news import prepare_message
>>> mlist = config.db.list_manager.create(u'_xtest@example.com')
>>> mlist.linked_newsgroup = u'comp.lang.python'
diff --git a/mailman/queue/docs/outgoing.txt b/mailman/queue/docs/outgoing.txt
index edc806d47..851bdf474 100644
--- a/mailman/queue/docs/outgoing.txt
+++ b/mailman/queue/docs/outgoing.txt
@@ -43,13 +43,11 @@ Normally, messages would show up in the outgoing queue after the message has
been processed by the rule set and pipeline. But we can simulate that here by
injecting a message directly into the outgoing queue.
- >>> from mailman.configuration import config
>>> msgdata = {}
>>> handler = config.handlers['calculate-recipients']
>>> handler.process(mlist, msg, msgdata)
- >>> from mailman.queue import Switchboard
- >>> outgoing_queue = Switchboard(config.OUTQUEUE_DIR)
+ >>> outgoing_queue = config.switchboards['out']
>>> ignore = outgoing_queue.enqueue(
... msg, msgdata,
... verp=True, listname=mlist.fqdn_listname, tolist=True,
@@ -60,7 +58,7 @@ upstream SMTP, which happens to be our test server.
>>> from mailman.queue.outgoing import OutgoingRunner
>>> from mailman.testing.helpers import make_testable_runner
- >>> outgoing = make_testable_runner(OutgoingRunner)
+ >>> outgoing = make_testable_runner(OutgoingRunner, 'out')
>>> outgoing.run()
Three messages have been delivered to our SMTP server, one for each recipient.
diff --git a/mailman/queue/docs/runner.txt b/mailman/queue/docs/runner.txt
index a9e17370b..db493110c 100644
--- a/mailman/queue/docs/runner.txt
+++ b/mailman/queue/docs/runner.txt
@@ -14,21 +14,21 @@ The basic architecture of qrunner is implemented in the base class that all
runners inherit from. This base class implements a .run() method that runs
continuously in a loop until the .stop() method is called.
- >>> import os
- >>> from mailman.queue import Runner, Switchboard
- >>> from mailman.configuration import config
>>> mlist = config.db.list_manager.create(u'_xtest@example.com')
>>> mlist.preferred_language = u'en'
-Here is a very simple derived qrunner class. The class attribute QDIR tells
-the qrunner which queue directory it is responsible for. Derived classes
-should also implement various methods to provide the special functionality.
-This is about as simple as a qrunner can be.
+Here is a very simple derived qrunner class. Queue runners use a
+configuration section in the configuration files to determine run
+characteristics, such as the queue directory to use. Here we push a
+configuration section for the test runner.
- >>> queue_directory = os.path.join(config.QUEUE_DIR, 'test')
+ >>> config.push('test-runner', """
+ ... [qrunner.test]
+ ... max_restarts: 1
+ ... """)
+
+ >>> from mailman.queue import Runner
>>> class TestableRunner(Runner):
- ... QDIR = queue_directory
- ...
... def _dispose(self, mlist, msg, msgdata):
... self.msg = msg
... self.msgdata = msgdata
@@ -40,8 +40,7 @@ This is about as simple as a qrunner can be.
... def _snooze(self, filecnt):
... return
- >>> runner = TestableRunner()
- >>> switchboard = Switchboard(queue_directory)
+ >>> runner = TestableRunner('test')
This qrunner doesn't do much except run once, storing the message and metadata
on instance variables.
@@ -52,6 +51,7 @@ on instance variables.
...
... A test message.
... """)
+ >>> switchboard = config.switchboards['test']
>>> filebase = switchboard.enqueue(msg, listname=mlist.fqdn_listname,
... foo='yes', bar='no')
>>> runner.run()
diff --git a/mailman/queue/docs/switchboard.txt b/mailman/queue/docs/switchboard.txt
index 633bdabe6..7baee7b54 100644
--- a/mailman/queue/docs/switchboard.txt
+++ b/mailman/queue/docs/switchboard.txt
@@ -14,7 +14,6 @@ instance of a switchboard is responsible for one queue directory.
Create a switchboard by giving its queue directory.
>>> import os
- >>> from mailman.configuration import config
>>> queue_directory = os.path.join(config.QUEUE_DIR, 'test')
>>> from mailman.queue import Switchboard
>>> switchboard = Switchboard(queue_directory)
diff --git a/mailman/queue/http.py b/mailman/queue/http.py
index cb6940b29..941b6d131 100644
--- a/mailman/queue/http.py
+++ b/mailman/queue/http.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2006-2008 by the Free Software Foundation, Inc.
+# Copyright (C) 2006-2009 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
@@ -25,7 +25,7 @@ from cStringIO import StringIO
from wsgiref.simple_server import make_server, WSGIRequestHandler
from mailman.Cgi.wsgi_app import mailman_app
-from mailman.configuration import config
+from mailman.config import config
from mailman.queue import Runner
hlog = logging.getLogger('mailman.http')
diff --git a/mailman/queue/incoming.py b/mailman/queue/incoming.py
index d4decd435..ae78e1a81 100644
--- a/mailman/queue/incoming.py
+++ b/mailman/queue/incoming.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2008 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2009 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
@@ -26,14 +26,14 @@ prepared for delivery. Rejections, discards, and holds are processed
immediately.
"""
-from mailman.configuration import config
+from mailman.config import config
from mailman.core.chains import process
from mailman.queue import Runner
class IncomingRunner(Runner):
- QDIR = config.INQUEUE_DIR
+ """The incoming queue runner."""
def _dispose(self, mlist, msg, msgdata):
if msgdata.get('envsender') is None:
diff --git a/mailman/queue/lmtp.py b/mailman/queue/lmtp.py
index b4688653b..f0895ee1f 100644
--- a/mailman/queue/lmtp.py
+++ b/mailman/queue/lmtp.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2006-2008 by the Free Software Foundation, Inc.
+# Copyright (C) 2006-2009 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
@@ -42,10 +42,11 @@ import asyncore
from email.utils import parseaddr
+from mailman import Defaults
from mailman.Message import Message
-from mailman.configuration import config
+from mailman.config import config
from mailman.database.transaction import txn
-from mailman.queue import Runner, Switchboard
+from mailman.queue import Runner
elog = logging.getLogger('mailman.error')
qlog = logging.getLogger('mailman.qrunner')
@@ -63,7 +64,7 @@ CRLF = '\r\n'
ERR_451 = '451 Requested action aborted: error in processing'
ERR_501 = '501 Message has defects'
ERR_502 = '502 Error: command HELO not implemented'
-ERR_550 = config.LMTP_ERR_550
+ERR_550 = Defaults.LMTP_ERR_550
# XXX Blech
smtpd.__version__ = 'Python LMTP queue runner 1.0'
@@ -86,7 +87,7 @@ def split_recipient(address):
subaddress may be None if this is the list's posting address.
"""
localpart, domain = address.split('@', 1)
- localpart = localpart.split(config.VERP_DELIMITER, 1)[0]
+ localpart = localpart.split(Defaults.VERP_DELIMITER, 1)[0]
parts = localpart.split(DASH)
if parts[-1] in SUBADDRESS_NAMES:
listname = DASH.join(parts[:-1])
@@ -121,11 +122,11 @@ class LMTPRunner(Runner, smtpd.SMTPServer):
# connections from the MTA. slice and numslices are ignored and are
# necessary only to satisfy the API.
def __init__(self, slice=None, numslices=1):
- localaddr = config.LMTP_HOST, config.LMTP_PORT
+ localaddr = config.mta.lmtp_host, int(config.mta.lmtp_port)
# Do not call Runner's constructor because there's no QDIR to create
smtpd.SMTPServer.__init__(self, localaddr, remoteaddr=None)
qlog.debug('LMTP server listening on %s:%s',
- config.LMTP_HOST, config.LMTP_PORT)
+ localaddr[0], localaddr[1])
def handle_accept(self):
conn, addr = self.accept()
@@ -138,6 +139,7 @@ class LMTPRunner(Runner, smtpd.SMTPServer):
# Refresh the list of list names every time we process a message
# since the set of mailing lists could have changed.
listnames = set(config.db.list_manager.names)
+ qlog.debug('listnames: %s', listnames)
# Parse the message data. If there are any defects in the
# message, reject it right away; it's probably spam.
msg = email.message_from_string(data, Message)
@@ -169,29 +171,29 @@ class LMTPRunner(Runner, smtpd.SMTPServer):
queue = None
msgdata = dict(listname=listname)
if subaddress in ('bounces', 'admin'):
- queue = Switchboard(config.BOUNCEQUEUE_DIR)
+ queue = 'bounce'
elif subaddress == 'confirm':
msgdata['toconfirm'] = True
- queue = Switchboard(config.CMDQUEUE_DIR)
+ queue = 'command'
elif subaddress in ('join', 'subscribe'):
msgdata['tojoin'] = True
- queue = Switchboard(config.CMDQUEUE_DIR)
+ queue = 'command'
elif subaddress in ('leave', 'unsubscribe'):
msgdata['toleave'] = True
- queue = Switchboard(config.CMDQUEUE_DIR)
+ queue = 'command'
elif subaddress == 'owner':
msgdata.update(dict(
toowner=True,
- envsender=config.SITE_OWNER_ADDRESS,
- pipeline=config.OWNER_PIPELINE,
+ envsender=config.mailman.site_owner,
+ pipeline=Defaults.OWNER_PIPELINE,
))
- queue = Switchboard(config.INQUEUE_DIR)
+ queue = 'in'
elif subaddress is None:
msgdata['tolist'] = True
- queue = Switchboard(config.INQUEUE_DIR)
+ queue = 'in'
elif subaddress == 'request':
msgdata['torequest'] = True
- queue = Switchboard(config.CMDQUEUE_DIR)
+ queue = 'command'
else:
elog.error('Unknown sub-address: %s', subaddress)
status.append(ERR_550)
@@ -199,7 +201,7 @@ class LMTPRunner(Runner, smtpd.SMTPServer):
# If we found a valid subaddress, enqueue the message and add
# a success status for this recipient.
if queue is not None:
- queue.enqueue(msg, msgdata)
+ config.switchboards[queue].enqueue(msg, msgdata)
status.append('250 Ok')
except Exception, e:
elog.exception('Queue detection: %s', msg['message-id'])
diff --git a/mailman/queue/maildir.py b/mailman/queue/maildir.py
index 71bac67dc..e12550cb5 100644
--- a/mailman/queue/maildir.py
+++ b/mailman/queue/maildir.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2002-2008 by the Free Software Foundation, Inc.
+# Copyright (C) 2002-2009 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
@@ -57,7 +57,7 @@ from email.Parser import Parser
from email.Utils import parseaddr
from mailman.Message import Message
-from mailman.configuration import config
+from mailman.config import config
from mailman.queue import Runner
log = logging.getLogger('mailman.error')
diff --git a/mailman/queue/news.py b/mailman/queue/news.py
index 8415f7d95..98735d498 100644
--- a/mailman/queue/news.py
+++ b/mailman/queue/news.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2000-2008 by the Free Software Foundation, Inc.
+# Copyright (C) 2000-2009 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
@@ -28,8 +28,8 @@ from email.utils import getaddresses, make_msgid
COMMASPACE = ', '
+from mailman import Defaults
from mailman import Utils
-from mailman.configuration import config
from mailman.interfaces import NewsModeration
from mailman.queue import Runner
@@ -51,8 +51,6 @@ mcre = re.compile(r"""
class NewsRunner(Runner):
- QDIR = config.NEWSQUEUE_DIR
-
def _dispose(self, mlist, msg, msgdata):
# Make sure we have the most up-to-date state
mlist.Load()
@@ -67,8 +65,8 @@ class NewsRunner(Runner):
nntp_host, nntp_port = Utils.nntpsplit(mlist.nntp_host)
conn = nntplib.NNTP(nntp_host, nntp_port,
readermode=True,
- user=config.NNTP_USERNAME,
- password=config.NNTP_PASSWORD)
+ user=Defaults.NNTP_USERNAME,
+ password=Defaults.NNTP_PASSWORD)
conn.post(fp)
except nntplib.error_temp, e:
log.error('(NNTPDirect) NNTP error for list "%s": %s',
@@ -150,9 +148,9 @@ def prepare_message(mlist, msg, msgdata):
# woon't completely sanitize the message, but it will eliminate the bulk
# of the rejections based on message headers. The NNTP server may still
# reject the message because of other problems.
- for header in config.NNTP_REMOVE_HEADERS:
+ for header in Defaults.NNTP_REMOVE_HEADERS:
del msg[header]
- for header, rewrite in config.NNTP_REWRITE_DUPLICATE_HEADERS:
+ for header, rewrite in Defaults.NNTP_REWRITE_DUPLICATE_HEADERS:
values = msg.get_all(header, [])
if len(values) < 2:
# We only care about duplicates
diff --git a/mailman/queue/outgoing.py b/mailman/queue/outgoing.py
index 3ab67eaad..14ba869dc 100644
--- a/mailman/queue/outgoing.py
+++ b/mailman/queue/outgoing.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2000-2008 by the Free Software Foundation, Inc.
+# Copyright (C) 2000-2009 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
@@ -26,8 +26,9 @@ import logging
from datetime import datetime
+from mailman import Defaults
from mailman import Message
-from mailman.configuration import config
+from mailman.config import config
from mailman.core import errors
from mailman.queue import Runner, Switchboard
from mailman.queue.bounce import BounceMixin
@@ -41,19 +42,19 @@ log = logging.getLogger('mailman.error')
class OutgoingRunner(Runner, BounceMixin):
- QDIR = config.OUTQUEUE_DIR
+ """The outgoing queue runner."""
def __init__(self, slice=None, numslices=1):
Runner.__init__(self, slice, numslices)
BounceMixin.__init__(self)
# We look this function up only at startup time
- handler = config.handlers[config.DELIVERY_MODULE]
+ handler = config.handlers[Defaults.DELIVERY_MODULE]
self._func = handler.process
# This prevents smtp server connection problems from filling up the
# error log. It gets reset if the message was successfully sent, and
# set if there was a socket.error.
self._logged = False
- self._retryq = Switchboard(config.RETRYQUEUE_DIR)
+ self._retryq = config.switchboards['retry']
def _dispose(self, mlist, msg, msgdata):
# See if we should retry delivery of this message again.
@@ -73,13 +74,13 @@ class OutgoingRunner(Runner, BounceMixin):
# There was a problem connecting to the SMTP server. Log this
# once, but crank up our sleep time so we don't fill the error
# log.
- port = config.SMTPPORT
+ port = int(config.mta.port)
if port == 0:
port = 'smtp'
# Log this just once.
if not self._logged:
log.error('Cannot connect to SMTP server %s on port %s',
- config.SMTPHOST, port)
+ config.mta.host, port)
self._logged = True
return True
except errors.SomeRecipientsFailed, e:
@@ -115,7 +116,7 @@ class OutgoingRunner(Runner, BounceMixin):
return False
else:
# Keep trying to delivery this message for a while
- deliver_until = now + config.DELIVERY_RETRY_PERIOD
+ deliver_until = now + Defaults.DELIVERY_RETRY_PERIOD
msgdata['last_recip_count'] = len(recips)
msgdata['deliver_until'] = deliver_until
msgdata['recips'] = recips
diff --git a/mailman/queue/pipeline.py b/mailman/queue/pipeline.py
index 665bfce77..f0a42f729 100644
--- a/mailman/queue/pipeline.py
+++ b/mailman/queue/pipeline.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2008 by the Free Software Foundation, Inc.
+# Copyright (C) 2008-2009 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
@@ -22,15 +22,13 @@ through the 'preparation pipeline'. This pipeline adds, deletes and modifies
headers, calculates message recipients, and more.
"""
-from mailman.app.pipelines import process
-from mailman.configuration import config
+from mailman.core.pipelines import process
+from mailman.config import config
from mailman.queue import Runner
class PipelineRunner(Runner):
- QDIR = config.PIPELINEQUEUE_DIR
-
def _dispose(self, mlist, msg, msgdata):
# Process the message through the mailing list's pipeline.
process(mlist, msg, msgdata, mlist.pipeline)
diff --git a/mailman/queue/retry.py b/mailman/queue/retry.py
index f8e1b4665..2b5a6afad 100644
--- a/mailman/queue/retry.py
+++ b/mailman/queue/retry.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2003-2008 by the Free Software Foundation, Inc.
+# Copyright (C) 2003-2009 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
@@ -17,24 +17,21 @@
import time
-from mailman.configuration import config
-from mailman.queue import Runner, Switchboard
+from mailman.config import config
+from mailman.queue import Runner
class RetryRunner(Runner):
- QDIR = config.RETRYQUEUE_DIR
- SLEEPTIME = config.minutes(15)
-
def __init__(self, slice=None, numslices=1):
Runner.__init__(self, slice, numslices)
- self.__outq = Switchboard(config.OUTQUEUE_DIR)
+ self._outq = config.switchboards['out']
def _dispose(self, mlist, msg, msgdata):
# Move it to the out queue for another retry
- self.__outq.enqueue(msg, msgdata)
+ self._outq.enqueue(msg, msgdata)
return False
def _snooze(self, filecnt):
# We always want to snooze
- time.sleep(float(self.SLEEPTIME))
+ time.sleep(self.sleep_float)
diff --git a/mailman/queue/virgin.py b/mailman/queue/virgin.py
index 503337ea1..ca61074bf 100644
--- a/mailman/queue/virgin.py
+++ b/mailman/queue/virgin.py
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2008 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2009 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
@@ -23,15 +23,13 @@ to go through some minimal processing before they can be sent out to the
recipient.
"""
-from mailman.app.pipelines import process
-from mailman.configuration import config
+from mailman.core.pipelines import process
+from mailman.config import config
from mailman.queue import Runner
class VirginRunner(Runner):
- QDIR = config.VIRGINQUEUE_DIR
-
def _dispose(self, mlist, msg, msgdata):
# We need to fast track this message through any pipeline handlers
# that touch it, e.g. especially cook-headers.