diff options
Diffstat (limited to 'src/mailman/queue')
28 files changed, 0 insertions, 5466 deletions
diff --git a/src/mailman/queue/__init__.py b/src/mailman/queue/__init__.py deleted file mode 100644 index 8abc5e9a6..000000000 --- a/src/mailman/queue/__init__.py +++ /dev/null @@ -1,483 +0,0 @@ -# Copyright (C) 2001-2011 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Queing and dequeuing message/metadata pickle files. - -Messages are represented as email.message.Message objects (or an instance ofa -subclass). Metadata is represented as a Python dictionary. For every -message/metadata pair in a queue, a single file containing two pickles is -written. First, the message is written to the pickle, then the metadata -dictionary is written. -""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'Runner', - 'Switchboard', - ] - - -import os -import time -import email -import pickle -import cPickle -import hashlib -import logging -import traceback - -from cStringIO import StringIO -from lazr.config import as_boolean, as_timedelta -from zope.component import getUtility -from zope.interface import implements - -from mailman.config import config -from mailman.core.i18n import _ -from mailman.email.message import Message -from mailman.interfaces.languages import ILanguageManager -from mailman.interfaces.listmanager import IListManager -from mailman.interfaces.runner import IRunner -from mailman.interfaces.switchboard import ISwitchboard -from mailman.utilities.filesystem import makedirs -from mailman.utilities.string import expand - - -# 20 bytes of all bits set, maximum hashlib.sha.digest() value -shamax = 0xffffffffffffffffffffffffffffffffffffffffL - -# Small increment to add to time in case two entries have the same time. This -# prevents skipping one of two entries with the same time until the next pass. -DELTA = .0001 -DOT = '.' -# We count the number of times a file has been moved to .bak and recovered. -# In order to prevent loops and a message flood, when the count reaches this -# value, we move the file to the bad queue as a .psv. -MAX_BAK_COUNT = 3 - -elog = logging.getLogger('mailman.error') -dlog = logging.getLogger('mailman.debug') - - - -class Switchboard: - implements(ISwitchboard) - - @staticmethod - def initialize(): - """Initialize the global switchboards for input/output.""" - for conf in config.qrunner_configs: - name = conf.name.split('.')[-1] - assert name not in config.switchboards, ( - 'Duplicate qrunner name: {0}'.format(name)) - substitutions = config.paths - substitutions['name'] = name - path = expand(conf.path, substitutions) - config.switchboards[name] = Switchboard(name, path) - - def __init__(self, name, queue_directory, - slice=None, numslices=1, recover=False): - """Create a switchboard object. - - :param name: The queue name. - :type name: str - :param queue_directory: The queue directory. - :type queue_directory: str - :param slice: The slice number for this switchboard, or None. If not - None, it must be [0..`numslices`). - :type slice: int or None - :param numslices: The total number of slices to split this queue - directory into. It must be a power of 2. - :type numslices: int - :param recover: True if backup files should be recovered. - :type recover: bool - """ - assert (numslices & (numslices - 1)) == 0, ( - 'Not a power of 2: {0}'.format(numslices)) - self.name = name - self.queue_directory = queue_directory - # If configured to, create the directory if it doesn't yet exist. - if config.create_paths: - makedirs(self.queue_directory, 0770) - # Fast track for no slices - self._lower = None - self._upper = None - # BAW: test performance and end-cases of this algorithm - if numslices <> 1: - self._lower = ((shamax + 1) * slice) / numslices - self._upper = (((shamax + 1) * (slice + 1)) / numslices) - 1 - if recover: - self.recover_backup_files() - - def enqueue(self, _msg, _metadata=None, **_kws): - """See `ISwitchboard`.""" - if _metadata is None: - _metadata = {} - # Calculate the SHA hexdigest of the message to get a unique base - # filename. We're also going to use the digest as a hash into the set - # of parallel qrunner processes. - data = _metadata.copy() - data.update(_kws) - listname = data.get('listname', '--nolist--') - # Get some data for the input to the sha hash. - now = time.time() - if data.get('_plaintext'): - protocol = 0 - msgsave = cPickle.dumps(str(_msg), protocol) - else: - protocol = pickle.HIGHEST_PROTOCOL - msgsave = cPickle.dumps(_msg, protocol) - # listname is unicode but the input to the hash function must be an - # 8-bit string (eventually, a bytes object). - hashfood = msgsave + listname.encode('utf-8') + repr(now) - # Encode the current time into the file name for FIFO sorting. The - # file name consists of two parts separated by a '+': the received - # time for this message (i.e. when it first showed up on this system) - # and the sha hex digest. - rcvtime = data.setdefault('received_time', now) - filebase = repr(rcvtime) + '+' + hashlib.sha1(hashfood).hexdigest() - filename = os.path.join(self.queue_directory, filebase + '.pck') - tmpfile = filename + '.tmp' - # Always add the metadata schema version number - data['version'] = config.QFILE_SCHEMA_VERSION - # Filter out volatile entries. Use .keys() so that we can mutate the - # dictionary during the iteration. - for k in data.keys(): - if k.startswith('_'): - del data[k] - # We have to tell the dequeue() method whether to parse the message - # object or not. - data['_parsemsg'] = (protocol == 0) - # Write to the pickle file the message object and metadata. - with open(tmpfile, 'w') as fp: - fp.write(msgsave) - cPickle.dump(data, fp, protocol) - fp.flush() - os.fsync(fp.fileno()) - os.rename(tmpfile, filename) - return filebase - - def dequeue(self, filebase): - """See `ISwitchboard`.""" - # Calculate the filename from the given filebase. - filename = os.path.join(self.queue_directory, filebase + '.pck') - backfile = os.path.join(self.queue_directory, filebase + '.bak') - # Read the message object and metadata. - with open(filename) as fp: - # Move the file to the backup file name for processing. If this - # process crashes uncleanly the .bak file will be used to - # re-instate the .pck file in order to try again. - os.rename(filename, backfile) - msg = cPickle.load(fp) - data = cPickle.load(fp) - if data.get('_parsemsg'): - # Calculate the original size of the text now so that we won't - # have to generate the message later when we do size restriction - # checking. - original_size = len(msg) - msg = email.message_from_string(msg, Message) - msg.original_size = original_size - data['original_size'] = original_size - return msg, data - - def finish(self, filebase, preserve=False): - """See `ISwitchboard`.""" - bakfile = os.path.join(self.queue_directory, filebase + '.bak') - try: - if preserve: - bad_dir = config.switchboards['bad'].queue_directory - psvfile = os.path.join(bad_dir, filebase + '.psv') - os.rename(bakfile, psvfile) - else: - os.unlink(bakfile) - except EnvironmentError: - elog.exception( - 'Failed to unlink/preserve backup file: %s', bakfile) - - @property - def files(self): - """See `ISwitchboard`.""" - return self.get_files() - - def get_files(self, extension='.pck'): - """See `ISwitchboard`.""" - times = {} - lower = self._lower - upper = self._upper - for f in os.listdir(self.queue_directory): - # By ignoring anything that doesn't end in .pck, we ignore - # tempfiles and avoid a race condition. - filebase, ext = os.path.splitext(f) - if ext <> extension: - continue - when, digest = filebase.split('+', 1) - # Throw out any files which don't match our bitrange. BAW: test - # performance and end-cases of this algorithm. MAS: both - # comparisons need to be <= to get complete range. - if lower is None or (lower <= long(digest, 16) <= upper): - key = float(when) - while key in times: - key += DELTA - times[key] = filebase - # FIFO sort - return [times[key] for key in sorted(times)] - - def recover_backup_files(self): - """See `ISwitchboard`.""" - # Move all .bak files in our slice to .pck. It's impossible for both - # to exist at the same time, so the move is enough to ensure that our - # normal dequeuing process will handle them. We keep count in - # _bak_count in the metadata of the number of times we recover this - # file. When the count reaches MAX_BAK_COUNT, we move the .bak file - # to a .psv file in the bad queue. - for filebase in self.get_files('.bak'): - src = os.path.join(self.queue_directory, filebase + '.bak') - dst = os.path.join(self.queue_directory, filebase + '.pck') - fp = open(src, 'rb+') - try: - try: - msg = cPickle.load(fp) - data_pos = fp.tell() - data = cPickle.load(fp) - except Exception as error: - # If unpickling throws any exception, just log and - # preserve this entry - elog.error('Unpickling .bak exception: %s\n' - 'Preserving file: %s', error, filebase) - self.finish(filebase, preserve=True) - else: - data['_bak_count'] = data.get('_bak_count', 0) + 1 - fp.seek(data_pos) - if data.get('_parsemsg'): - protocol = 0 - else: - protocol = 1 - cPickle.dump(data, fp, protocol) - fp.truncate() - fp.flush() - os.fsync(fp.fileno()) - if data['_bak_count'] >= MAX_BAK_COUNT: - elog.error('.bak file max count, preserving file: %s', - filebase) - self.finish(filebase, preserve=True) - else: - os.rename(src, dst) - finally: - fp.close() - - - -class Runner: - implements(IRunner) - - intercept_signals = True - - def __init__(self, name, slice=None): - """Create a queue runner. - - :param slice: The slice number for this queue runner. This is passed - directly to the underlying `ISwitchboard` object. - :type slice: int or None - """ - # Grab the configuration section. - self.name = name - section = getattr(config, 'qrunner.' + name) - substitutions = config.paths - substitutions['name'] = name - self.queue_directory = expand(section.path, substitutions) - numslices = int(section.instances) - self.switchboard = Switchboard( - name, self.queue_directory, slice, numslices, True) - self.sleep_time = as_timedelta(section.sleep_time) - # sleep_time is a timedelta; turn it into a float for time.sleep(). - self.sleep_float = (86400 * self.sleep_time.days + - self.sleep_time.seconds + - self.sleep_time.microseconds / 1.0e6) - self.max_restarts = int(section.max_restarts) - self.start = as_boolean(section.start) - self._stop = False - - def __repr__(self): - return '<{0} at {1:#x}>'.format(self.__class__.__name__, id(self)) - - def stop(self): - """See `IRunner`.""" - self._stop = True - - def run(self): - """See `IRunner`.""" - # Start the main loop for this queue runner. - try: - while True: - # Once through the loop that processes all the files in the - # queue directory. - filecnt = self._one_iteration() - # Do the periodic work for the subclass. - self._do_periodic() - # If the stop flag is set, we're done. - if self._stop: - break - # Give the runner an opportunity to snooze for a while, but - # pass it the file count so it can decide whether to do more - # work now or not. - self._snooze(filecnt) - except KeyboardInterrupt: - pass - finally: - self._clean_up() - - def _one_iteration(self): - """See `IRunner`.""" - me = self.__class__.__name__ - dlog.debug('[%s] starting oneloop', me) - # List all the files in our queue directory. The switchboard is - # guaranteed to hand us the files in FIFO order. - files = self.switchboard.files - for filebase in files: - dlog.debug('[%s] processing filebase: %s', me, filebase) - try: - # Ask the switchboard for the message and metadata objects - # associated with this queue file. - msg, msgdata = self.switchboard.dequeue(filebase) - except Exception as error: - # This used to just catch email.Errors.MessageParseError, but - # other problems can occur in message parsing, e.g. - # ValueError, and exceptions can occur in unpickling too. We - # don't want the runner to die, so we just log and skip this - # entry, but preserve it for analysis. - self._log(error) - elog.error('Skipping and preserving unparseable message: %s', - filebase) - self.switchboard.finish(filebase, preserve=True) - config.db.abort() - continue - try: - dlog.debug('[%s] processing onefile', me) - self._process_one_file(msg, msgdata) - dlog.debug('[%s] finishing filebase: %s', me, filebase) - self.switchboard.finish(filebase) - except Exception as error: - # All runners that implement _dispose() must guarantee that - # exceptions are caught and dealt with properly. Still, there - # may be a bug in the infrastructure, and we do not want those - # to cause messages to be lost. Any uncaught exceptions will - # cause the message to be stored in the shunt queue for human - # intervention. - self._log(error) - # Put a marker in the metadata for unshunting. - msgdata['whichq'] = self.switchboard.name - # It is possible that shunting can throw an exception, e.g. a - # permissions problem or a MemoryError due to a really large - # message. Try to be graceful. - try: - shunt = config.switchboards['shunt'] - new_filebase = shunt.enqueue(msg, msgdata) - elog.error('SHUNTING: %s', new_filebase) - self.switchboard.finish(filebase) - except Exception as error: - # The message wasn't successfully shunted. Log the - # exception and try to preserve the original queue entry - # for possible analysis. - self._log(error) - elog.error( - 'SHUNTING FAILED, preserving original entry: %s', - filebase) - self.switchboard.finish(filebase, preserve=True) - config.db.abort() - # Other work we want to do each time through the loop. - dlog.debug('[%s] doing periodic', me) - self._do_periodic() - dlog.debug('[%s] checking short circuit', me) - if self._short_circuit(): - dlog.debug('[%s] short circuiting', me) - break - dlog.debug('[%s] commiting', me) - config.db.commit() - dlog.debug('[%s] ending oneloop: %s', me, len(files)) - return len(files) - - def _process_one_file(self, msg, msgdata): - """See `IRunner`.""" - # Do some common sanity checking on the message metadata. It's got to - # be destined for a particular mailing list. This switchboard is used - # to shunt off badly formatted messages. We don't want to just trash - # them because they may be fixable with human intervention. Just get - # them out of our sight. - # - # Find out which mailing list this message is destined for. - missing = object() - listname = msgdata.get('listname', missing) - mlist = (None - if listname is missing - else getUtility(IListManager).get(unicode(listname))) - if mlist is None: - elog.error( - '%s runner "%s" shunting message for missing list: %s', - msg['message-id'], self.name, - ('n/a' if listname is missing else listname)) - config.switchboards['shunt'].enqueue(msg, msgdata) - return - # Now process this message. We also want to set up the language - # context for this message. The context will be the preferred - # language for the user if the sender is a member of the list, or it - # will be the list's preferred language. However, we must take - # special care to reset the defaults, otherwise subsequent messages - # may be translated incorrectly. - if mlist is None: - language_manager = getUtility(ILanguageManager) - language = language_manager[config.mailman.default_language] - elif msg.sender: - member = mlist.members.get_member(msg.sender) - language = (member.preferred_language - if member is not None - else mlist.preferred_language) - else: - language = mlist.preferred_language - with _.using(language.code): - msgdata['lang'] = language.code - keepqueued = self._dispose(mlist, msg, msgdata) - if keepqueued: - self.switchboard.enqueue(msg, msgdata) - - def _log(self, exc): - elog.error('Uncaught runner exception: %s', exc) - s = StringIO() - traceback.print_exc(file=s) - elog.error('%s', s.getvalue()) - - def _clean_up(self): - """See `IRunner`.""" - pass - - def _dispose(self, mlist, msg, msgdata): - """See `IRunner`.""" - raise NotImplementedError - - def _do_periodic(self): - """See `IRunner`.""" - pass - - def _snooze(self, filecnt): - """See `IRunner`.""" - if filecnt or self.sleep_float <= 0: - return - time.sleep(self.sleep_float) - - def _short_circuit(self): - """See `IRunner`.""" - return self._stop diff --git a/src/mailman/queue/archive.py b/src/mailman/queue/archive.py deleted file mode 100644 index 99682f310..000000000 --- a/src/mailman/queue/archive.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright (C) 2000-2011 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Archive queue runner.""" - -__metaclass__ = type -__all__ = [ - 'ArchiveRunner', - ] - - -import os -import logging - -from datetime import datetime -from email.utils import parsedate_tz, mktime_tz, formatdate -from flufl.lock import Lock -from lazr.config import as_timedelta - -from mailman.config import config -from mailman.queue import Runner - -log = logging.getLogger('mailman.error') - - - -class ArchiveRunner(Runner): - """The archive runner.""" - - def _dispose(self, mlist, msg, msgdata): - # Support clobber_date, i.e. setting the date in the archive to the - # received date, not the (potentially bogus) Date: header of the - # original message. - clobber = False - original_date = msg.get('date') - received_time = formatdate(msgdata['received_time']) - if not original_date: - clobber = True - elif int(config.archiver.pipermail.clobber_date_policy) == 1: - clobber = True - elif int(config.archiver.pipermail.clobber_date_policy) == 2: - # What's the timestamp on the original message? - timetup = parsedate_tz(original_date) - now = datetime.now() - try: - if not timetup: - clobber = True - else: - utc_timestamp = datetime.fromtimestamp(mktime_tz(timetup)) - date_skew = as_timedelta( - config.archiver.pipermail.allowable_sane_date_skew) - clobber = (abs(now - utc_timestamp) > date_skew) - except (ValueError, OverflowError): - # The likely cause of this is that the year in the Date: field - # is horribly incorrect, e.g. (from SF bug # 571634): - # Date: Tue, 18 Jun 0102 05:12:09 +0500 - # Obviously clobber such dates. - clobber = True - if clobber: - del msg['date'] - del msg['x-original-date'] - msg['Date'] = received_time - if original_date: - msg['X-Original-Date'] = original_date - # Always put an indication of when we received the message. - msg['X-List-Received-Date'] = received_time - # While a list archiving lock is acquired, archive the message. - with Lock(os.path.join(mlist.data_path, 'archive.lck')): - for archiver in config.archivers: - # A problem in one archiver should not prevent other archivers - # from running. - try: - archiver.archive_message(mlist, msg) - except Exception: - log.exception('Broken archiver: %s' % archiver.name) diff --git a/src/mailman/queue/bounce.py b/src/mailman/queue/bounce.py deleted file mode 100644 index a714f2669..000000000 --- a/src/mailman/queue/bounce.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright (C) 2001-2011 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Bounce queue runner.""" - -import logging - -from zope.component import getUtility - -from mailman.app.bounces import ( - ProbeVERP, StandardVERP, maybe_forward, scan_message) -from mailman.interfaces.bounce import BounceContext, IBounceProcessor, Stop -from mailman.queue import Runner - - -COMMASPACE = ', ' - -log = logging.getLogger('mailman.bounce') -elog = logging.getLogger('mailman.error') - - - -class BounceRunner(Runner): - """The bounce runner.""" - - def __init__(self, name, slice=None): - super(BounceRunner, self).__init__(name, slice) - self._processor = getUtility(IBounceProcessor) - - def _dispose(self, mlist, msg, msgdata): - # List isn't doing bounce processing? - if not mlist.bounce_processing: - return False - # Try VERP detection first, since it's quick and easy - context = BounceContext.normal - addresses = StandardVERP().get_verp(mlist, msg) - if addresses: - # We have an address, but check if the message is non-fatal. It - # will be non-fatal if the bounce scanner returns Stop. It will - # return a set of addresses when the bounce is fatal, but we don't - # care about those addresses, since we got it out of the VERP. - if scan_message(mlist, msg) is Stop: - return False - else: - # See if this was a probe message. - addresses = ProbeVERP().get_verp(mlist, msg) - if addresses: - context = BounceContext.probe - else: - # That didn't give us anything useful, so try the old fashion - # bounce matching modules. - addresses = scan_message(mlist, msg) - if addresses is Stop: - # This is a recognized, non-fatal notice. Ignore it. - return False - # If that still didn't return us any useful addresses, then send it on - # or discard it. - if len(addresses) > 0: - for address in addresses: - self._processor.register(mlist, address, msg, context) - else: - log.info('Bounce message w/no discernable addresses: %s', - msg.get('message-id', 'n/a')) - maybe_forward(mlist, msg) - # Dequeue this message. - return False diff --git a/src/mailman/queue/command.py b/src/mailman/queue/command.py deleted file mode 100644 index ec3ec0089..000000000 --- a/src/mailman/queue/command.py +++ /dev/null @@ -1,219 +0,0 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""-request robot command queue runner.""" - -__metaclass__ = type -__all__ = [ - 'CommandRunner', - 'Results', - ] - -# See the delivery diagram in IncomingRunner.py. This module handles all -# email destined for mylist-request, -join, and -leave. It no longer handles -# bounce messages (i.e. -admin or -bounces), nor does it handle mail to -# -owner. - -import re -import logging - -from StringIO import StringIO -from email.errors import HeaderParseError -from email.header import decode_header, make_header -from email.iterators import typed_subpart_iterator -from zope.component import getUtility -from zope.interface import implements - -from mailman.config import config -from mailman.core.i18n import _ -from mailman.email.message import UserNotification -from mailman.interfaces.command import ContinueProcessing, IEmailResults -from mailman.interfaces.languages import ILanguageManager -from mailman.queue import Runner - - -NL = '\n' -log = logging.getLogger('mailman.vette') - - - -class CommandFinder: - """Generate commands from the content of a message.""" - - def __init__(self, msg, msgdata, results): - self.command_lines = [] - self.ignored_lines = [] - self.processed_lines = [] - # Depending on where the message was destined to, add some implicit - # commands. For example, if this was sent to the -join or -leave - # addresses, it's the same as if 'join' or 'leave' commands were sent - # to the -request address. - subaddress = msgdata.get('subaddress') - if subaddress == 'join': - self.command_lines.append('join') - elif subaddress == 'leave': - self.command_lines.append('leave') - elif subaddress == 'confirm': - mo = re.match(config.mta.verp_confirm_regexp, msg.get('to', '')) - if mo: - self.command_lines.append('confirm ' + mo.group('cookie')) - # Extract the subject header and do RFC 2047 decoding. - raw_subject = msg.get('subject', '') - try: - subject = unicode(make_header(decode_header(raw_subject))) - # Mail commands must be ASCII. - self.command_lines.append(subject.encode('us-ascii')) - except (HeaderParseError, UnicodeError, LookupError): - # The Subject header was unparseable or not ASCII, so just ignore - # it. - pass - # Find the first text/plain part of the message. - part = None - for part in typed_subpart_iterator(msg, 'text', 'plain'): - break - if part is None or part is not msg: - # Either there was no text/plain part or we ignored some - # non-text/plain parts. - print >> results, _('Ignoring non-text/plain MIME parts') - if part is None: - # There was no text/plain part to be found. - return - body = part.get_payload(decode=True) - # text/plain parts better have string payloads. - assert isinstance(body, basestring), 'Non-string decoded payload' - lines = body.splitlines() - # Use no more lines than specified - max_lines = int(config.mailman.email_commands_max_lines) - self.command_lines.extend(lines[:max_lines]) - self.ignored_lines.extend(lines[max_lines:]) - - def __iter__(self): - """Return each command line, split into commands and arguments. - - :return: 2-tuples where the first element is the command and the - second element is a tuple of the arguments. - """ - while self.command_lines: - line = self.command_lines.pop(0) - self.processed_lines.append(line) - parts = line.strip().split() - if len(parts) == 0: - continue - command = parts.pop(0) - yield command, tuple(parts) - - - -class Results: - """The email command results.""" - - implements(IEmailResults) - - def __init__(self): - self._output = StringIO() - print >> self._output, _("""\ -The results of your email command are provided below. -""") - - def write(self, text): - self._output.write(text) - - def __unicode__(self): - value = self._output.getvalue() - assert isinstance(value, unicode), 'Not a unicode: %r' % value - return value - - - -class CommandRunner(Runner): - """The email command runner.""" - - def _dispose(self, mlist, msg, msgdata): - message_id = msg.get('message-id', 'n/a') - # The policy here is similar to the Replybot policy. If a message has - # "Precedence: bulk|junk|list" and no "X-Ack: yes" header, we discard - # the command message. - precedence = msg.get('precedence', '').lower() - ack = msg.get('x-ack', '').lower() - if ack <> 'yes' and precedence in ('bulk', 'junk', 'list'): - log.info('%s Precedence: %s message discarded by: %s', - message_id, precedence, mlist.request_address) - return False - # Do replybot for commands. - replybot = config.handlers['replybot'] - replybot.process(mlist, msg, msgdata) - if mlist.autorespond_requests == 1: - # Respond and discard. - log.info('%s -request message replied and discard', message_id) - return False - # Now craft the response and process the command lines. - results = Results() - # Include just a few key pieces of information from the original: the - # sender, date, and message id. - print >> results, _('- Original message details:') - subject = msg.get('subject', 'n/a') - date = msg.get('date', 'n/a') - from_ = msg.get('from', 'n/a') - print >> results, _(' From: $from_') - print >> results, _(' Subject: $subject') - print >> results, _(' Date: $date') - print >> results, _(' Message-ID: $message_id') - print >> results, _('\n- Results:') - finder = CommandFinder(msg, msgdata, results) - for command_name, arguments in finder: - command = config.commands.get(command_name) - if command is None: - print >> results, _('No such command: $command_name') - else: - status = command.process( - mlist, msg, msgdata, arguments, results) - assert status in ContinueProcessing, ( - 'Invalid status: %s' % status) - if status == ContinueProcessing.no: - break - # All done. Strip blank lines and send the response. - lines = filter(None, (line.strip() for line in finder.command_lines)) - if len(lines) > 0: - print >> results, _('\n- Unprocessed:') - for line in lines: - print >> results, line - lines = filter(None, (line.strip() for line in finder.ignored_lines)) - if len(lines) > 0: - print >> results, _('\n- Ignored:') - for line in lines: - print >> results, line - print >> results, _('\n- Done.') - # Send a reply, but do not attach the original message. This is a - # compromise because the original message is often helpful in tracking - # down problems, but it's also a vector for backscatter spam. - language = getUtility(ILanguageManager)[msgdata['lang']] - reply = UserNotification(msg.sender, mlist.bounces_address, - _('The results of your email commands'), - lang=language) - # Find a charset for the response body. Try ascii first, then - # latin-1 and finally falling back to utf-8. - reply_body = unicode(results) - for charset in ('us-ascii', 'latin-1'): - try: - reply_body.encode(charset) - break - except UnicodeError: - pass - else: - charset = 'utf-8' - reply.set_payload(reply_body, charset=charset) - reply.send(mlist) diff --git a/src/mailman/queue/digest.py b/src/mailman/queue/digest.py deleted file mode 100644 index 075335158..000000000 --- a/src/mailman/queue/digest.py +++ /dev/null @@ -1,368 +0,0 @@ -# Copyright (C) 2009-2011 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Digest queue runner.""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'DigestRunner', - ] - - -import re - -# cStringIO doesn't support unicode. -from StringIO import StringIO -from contextlib import nested -from copy import deepcopy -from email.header import Header -from email.message import Message -from email.mime.message import MIMEMessage -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText -from email.utils import formatdate, getaddresses, make_msgid - -from mailman.config import config -from mailman.core.errors import DiscardMessage -from mailman.core.i18n import _ -from mailman.interfaces.member import DeliveryMode, DeliveryStatus -from mailman.pipeline.decorate import decorate -from mailman.pipeline.scrubber import process as scrubber -from mailman.queue import Runner -from mailman.utilities.i18n import make -from mailman.utilities.mailbox import Mailbox -from mailman.utilities.string import oneline, wrap - - - -class Digester: - """Base digester class.""" - - def __init__(self, mlist, volume, digest_number): - self._mlist = mlist - self._charset = mlist.preferred_language.charset - # This will be used in the Subject, so use $-strings. - realname = mlist.real_name - issue = digest_number - self._digest_id = _('$realname Digest, Vol $volume, Issue $issue') - self._subject = Header(self._digest_id, - self._charset, - header_name='Subject') - self._message = self._make_message() - self._message['From'] = mlist.request_address - self._message['Subject'] = self._subject - self._message['To'] = mlist.posting_address - self._message['Reply-To'] = mlist.posting_address - self._message['Date'] = formatdate(localtime=True) - self._message['Message-ID'] = make_msgid() - # In the rfc1153 digest, the masthead contains the digest boilerplate - # plus any digest header. In the MIME digests, the masthead and - # digest header are separate MIME subobjects. In either case, it's - # the first thing in the digest, and we can calculate it now, so go - # ahead and add it now. - self._masthead = make('masthead.txt', - mailing_list=mlist, - real_name=mlist.real_name, - got_list_email=mlist.posting_address, - got_listinfo_url=mlist.script_url('listinfo'), - got_request_email=mlist.request_address, - got_owner_email=mlist.owner_address, - ) - # Set things up for the table of contents. - self._header = decorate(mlist, mlist.digest_header) - self._toc = StringIO() - print >> self._toc, _("Today's Topics:\n") - - def add_to_toc(self, msg, count): - """Add a message to the table of contents.""" - subject = msg.get('subject', _('(no subject)')) - subject = oneline(subject, in_unicode=True) - # Don't include the redundant subject prefix in the toc - mo = re.match('(re:? *)?({0})'.format( - re.escape(self._mlist.subject_prefix)), - subject, re.IGNORECASE) - if mo: - subject = subject[:mo.start(2)] + subject[mo.end(2):] - # Take only the first author we find. - username = '' - addresses = getaddresses( - [oneline(msg.get('from', ''), in_unicode=True)]) - if addresses: - username = addresses[0][0] - if not username: - username = addresses[0][1] - if username: - username = ' ({0})'.format(username) - lines = wrap('{0:2}. {1}'. format(count, subject), 65).split('\n') - # See if the user's name can fit on the last line - if len(lines[-1]) + len(username) > 70: - lines.append(username) - else: - lines[-1] += username - # Add this subject to the accumulating topics - first = True - for line in lines: - if first: - print >> self._toc, ' ', line - first = False - else: - print >> self._toc, ' ', line.lstrip() - - def add_message(self, msg, count): - """Add the message to the digest.""" - # We do not want all the headers of the original message to leak - # through in the digest messages. - keepers = {} - for header in self._keepers: - keepers[header] = msg.get_all(header, []) - # Remove all the unkempt <wink> headers. Use .keys() to allow for - # destructive iteration... - for header in msg.keys(): - del msg[header] - # ... and add them in the designated order. - for header in self._keepers: - for value in keepers[header]: - msg[header] = value - # Add some useful extra stuff. - msg['Message'] = unicode(count) - - - -class MIMEDigester(Digester): - """A MIME digester.""" - - def __init__(self, mlist, volume, digest_number): - super(MIMEDigester, self).__init__(mlist, volume, digest_number) - masthead = MIMEText(self._masthead.encode(self._charset), - _charset=self._charset) - masthead['Content-Description'] = self._subject - self._message.attach(masthead) - # Add the optional digest header. - if mlist.digest_header: - header = MIMEText(self._header.encode(self._charset), - _charset=self._charset) - header['Content-Description'] = _('Digest Header') - self._message.attach(header) - # Calculate the set of headers we're to keep in the MIME digest. - self._keepers = set(config.digests.mime_digest_keep_headers.split()) - - def _make_message(self): - return MIMEMultipart('mixed') - - def add_toc(self, count): - """Add the table of contents.""" - toc_text = self._toc.getvalue() - try: - toc_part = MIMEText(toc_text.encode(self._charset), - _charset=self._charset) - except UnicodeError: - toc_part = MIMEText(toc_text.encode('utf-8'), _charset='utf-8') - toc_part['Content-Description']= _("Today's Topics ($count messages)") - self._message.attach(toc_part) - - def add_message(self, msg, count): - """Add the message to the digest.""" - # Make a copy of the message object, since the RFC 1153 processing - # scrubs out attachments. - self._message.attach(MIMEMessage(deepcopy(msg))) - - def finish(self): - """Finish up the digest, producing the email-ready copy.""" - if self._mlist.digest_footer: - footer_text = decorate(self._mlist, self._mlist.digest_footer) - footer = MIMEText(footer_text.encode(self._charset), - _charset=self._charset) - footer['Content-Description'] = _('Digest Footer') - self._message.attach(footer) - # This stuff is outside the normal MIME goo, and it's what the old - # MIME digester did. No one seemed to complain, probably because you - # won't see it in an MUA that can't display the raw message. We've - # never got complaints before, but if we do, just wax this. It's - # primarily included for (marginally useful) backwards compatibility. - self._message.postamble = _('End of ') + self._digest_id - return self._message - - - -class RFC1153Digester(Digester): - """A digester of the format specified by RFC 1153.""" - - def __init__(self, mlist, volume, digest_number): - super(RFC1153Digester, self).__init__(mlist, volume, digest_number) - self._separator70 = '-' * 70 - self._separator30 = '-' * 30 - self._text = StringIO() - print >> self._text, self._masthead - print >> self._text - # Add the optional digest header. - if mlist.digest_header: - print >> self._text, self._header - print >> self._text - # Calculate the set of headers we're to keep in the RFC1153 digest. - self._keepers = set(config.digests.plain_digest_keep_headers.split()) - - def _make_message(self): - return Message() - - def add_toc(self, count): - """Add the table of contents.""" - print >> self._text, self._toc.getvalue() - print >> self._text - print >> self._text, self._separator70 - print >> self._text - - def add_message(self, msg, count): - """Add the message to the digest.""" - if count > 1: - print >> self._text, self._separator30 - print >> self._text - # Scrub attachements. - try: - msg = scrubber(self._mlist, msg) - except DiscardMessage: - print >> self._text, _('[Message discarded by content filter]') - return - # Each message section contains a few headers. - for header in config.digests.plain_digest_keep_headers.split(): - if header in msg: - value = oneline(msg[header], in_unicode=True) - value = wrap('{0}: {1}'.format(header, value)) - value = '\n\t'.join(value.split('\n')) - print >> self._text, value - print >> self._text - # Add the payload. If the decoded payload is empty, this may be a - # multipart message. In that case, just stringify it. - payload = msg.get_payload(decode=True) - payload = (payload if payload else msg.as_string().split('\n\n', 1)[1]) - try: - charset = msg.get_content_charset('us-ascii') - payload = unicode(payload, charset, 'replace') - except (LookupError, TypeError): - # Unknown or empty charset. - payload = unicode(payload, 'us-ascii', 'replace') - print >> self._text, payload - if not payload.endswith('\n'): - print >> self._text - - def finish(self): - """Finish up the digest, producing the email-ready copy.""" - if self._mlist.digest_footer: - footer_text = decorate(self._mlist, self._mlist.digest_footer) - # This is not strictly conformant RFC 1153. The trailer is only - # supposed to contain two lines, i.e. the "End of ... Digest" line - # and the row of asterisks. If this screws up MUAs, the solution - # is to add the footer as the last message in the RFC 1153 digest. - # I just hate the way that VM does that and I think it's confusing - # to users, so don't do it unless there's a clamor. - print >> self._text, self._separator30 - print >> self._text - print >> self._text, footer_text - print >> self._text - # Add the sign-off. - sign_off = _('End of ') + self._digest_id - print >> self._text, sign_off - print >> self._text, '*' * len(sign_off) - # If the digest message can't be encoded by the list character set, - # fall back to utf-8. - text = self._text.getvalue() - try: - self._message.set_payload(text.encode(self._charset), - charset=self._charset) - except UnicodeError: - self._message.set_payload(text.encode('utf-8'), charset='utf-8') - return self._message - - - -class DigestRunner(Runner): - """The digest queue runner.""" - - def _dispose(self, mlist, msg, msgdata): - """See `IRunner`.""" - volume = msgdata['volume'] - digest_number = msgdata['digest_number'] - with nested(Mailbox(msgdata['digest_path']), - _.using(mlist.preferred_language.code)) as (mailbox, - language_code): - # Create the digesters. - mime_digest = MIMEDigester(mlist, volume, digest_number) - rfc1153_digest = RFC1153Digester(mlist, volume, digest_number) - # Cruise through all the messages in the mailbox, first building - # the table of contents and accumulating Subject: headers and - # authors. The question really is whether it's better from a - # performance and memory footprint to go through the mailbox once - # and cache the messages in a list, or to cruise through the - # mailbox twice. We'll do the latter, but it's a complete guess. - count = None - for count, (key, message) in enumerate(mailbox.iteritems(), 1): - mime_digest.add_to_toc(message, count) - rfc1153_digest.add_to_toc(message, count) - assert count is not None, 'No digest messages?' - # Add the table of contents. - mime_digest.add_toc(count) - rfc1153_digest.add_toc(count) - # Cruise through the set of messages a second time, adding them to - # the actual digest. - for count, (key, message) in enumerate(mailbox.iteritems(), 1): - mime_digest.add_message(message, count) - rfc1153_digest.add_message(message, count) - # Finish up the digests. - mime = mime_digest.finish() - rfc1153 = rfc1153_digest.finish() - # Calculate the recipients lists - mime_recipients = set() - rfc1153_recipients = set() - # When someone turns off digest delivery, they will get one last - # digest to ensure that there will be no gaps in the messages they - # receive. - digest_members = set(mlist.digest_members.members) - for member in digest_members: - if member.delivery_status <> DeliveryStatus.enabled: - continue - # Send the digest to the case-preserved address of the digest - # members. - email_address = member.address.original_email - if member.delivery_mode == DeliveryMode.plaintext_digests: - rfc1153_recipients.add(email_address) - elif member.delivery_mode == DeliveryMode.mime_digests: - mime_recipients.add(email_address) - else: - raise AssertionError( - 'Digest member "{0}" unexpected delivery mode: {1}'.format( - email_address, member.delivery_mode)) - # Add also the folks who are receiving one last digest. - for address, delivery_mode in mlist.last_digest_recipients: - if delivery_mode == DeliveryMode.plaintext_digests: - rfc1153_recipients.add(address.original_email) - elif delivery_mode == DeliveryMode.mime_digests: - mime_recipients.add(address.original_email) - else: - raise AssertionError( - 'OLD recipient "{0}" unexpected delivery mode: {1}'.format( - address, delivery_mode)) - # Send the digests to the virgin queue for final delivery. - queue = config.switchboards['virgin'] - queue.enqueue(mime, - recipients=mime_recipients, - listname=mlist.fqdn_listname, - isdigest=True) - queue.enqueue(rfc1153, - recipients=rfc1153_recipients, - listname=mlist.fqdn_listname, - isdigest=True) diff --git a/src/mailman/queue/docs/OVERVIEW.txt b/src/mailman/queue/docs/OVERVIEW.txt deleted file mode 100644 index 41ccc18c9..000000000 --- a/src/mailman/queue/docs/OVERVIEW.txt +++ /dev/null @@ -1,80 +0,0 @@ -============== -Alias Overview -============== - -A typical Mailman list exposes nine aliases which point to seven different -wrapped scripts. E.g. for a list named ``mylist``, you'd have:: - - mylist-bounces -> bounces - mylist-confirm -> confirm - mylist-join -> join (-subscribe is an alias) - mylist-leave -> leave (-unsubscribe is an alias) - mylist-owner -> owner - mylist -> post - mylist-request -> request - -``-request``, ``-join``, and ``-leave`` are a robot addresses; their sole -purpose is to process emailed commands, although the latter two are hardcoded -to subscription and unsubscription requests. ``-bounces`` is the automated -bounce processor, and all messages to list members have their return address -set to ``-bounces``. If the bounce processor fails to extract a bouncing -member address, it can optionally forward the message on to the list owners. - -``-owner`` is for reaching a human operator with minimal list interaction -(i.e. no bounce processing). ``-confirm`` is another robot address which -processes replies to VERP-like confirmation notices. - -So delivery flow of messages look like this:: - - joerandom ---> mylist ---> list members - | | - | |[bounces] - | mylist-bounces <---+ <-------------------------------+ - | | | - | +--->[internal bounce processing] | - | ^ | | - | | | [bounce found] | - | [bounces *] +--->[register and discard] | - | | | | | - | | | |[*] | - | [list owners] |[no bounce found] | | - | ^ | | | - | | | | | - +-------> mylist-owner <--------+ | | - | | | - | data/owner-bounces.mbox <--[site list] <---+ | - | | - +-------> mylist-join--+ | - | | | - +------> mylist-leave--+ | - | | | - | v | - +-------> mylist-request | - | | | - | +---> [command processor] | - | | | - +-----> mylist-confirm ----> +---> joerandom | - | | - |[bounces] | - +----------------------+ - -A person can send an email to the list address (for posting), the ``-owner`` -address (to reach the human operator), or the ``-confirm``, ``-join``, -``-leave``, and ``-request`` mailbots. Message to the list address are then -forwarded on to the list membership, with bounces directed to the -bounces -address. - -[*] Messages sent to the ``-owner`` address are forwarded on to the list -owner/moderators. All ``-owner`` destined messages have their bounces -directed to the site list ``-bounces`` address, regardless of whether a human -sent the message or the message was crafted internally. The intention here is -that the site owners want to be notified when one of their list owners' -addresses starts bouncing (yes, the will be automated in a future release). - -Any messages to site owners has their bounces directed to a special *loop -killer* address, which just dumps the message into -``data/owners-bounces.mbox``. - -Finally, message to any of the mailbots causes the requested action to be -performed. Results notifications are sent to the author of the message, which -all bounces pointing back to the -bounces address. diff --git a/src/mailman/queue/docs/archiver.txt b/src/mailman/queue/docs/archiver.txt deleted file mode 100644 index cdee449e1..000000000 --- a/src/mailman/queue/docs/archiver.txt +++ /dev/null @@ -1,35 +0,0 @@ -========= -Archiving -========= - -Mailman can archive to any number of archivers that adhere to the -``IArchiver`` interface. By default, there's a Pipermail archiver. -:: - - >>> mlist = create_list('test@example.com') - >>> transaction.commit() - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: test@example.com - ... Subject: My first post - ... Message-ID: <first> - ... - ... First post! - ... """) - - >>> archiver_queue = config.switchboards['archive'] - >>> ignore = archiver_queue.enqueue(msg, {}, listname=mlist.fqdn_listname) - - >>> from mailman.queue.archive import ArchiveRunner - >>> from mailman.testing.helpers import make_testable_runner - >>> runner = make_testable_runner(ArchiveRunner) - >>> runner.run() - - # The best we can do is verify some landmark exists. Let's use the - # Pipermail pickle file exists. - >>> listname = mlist.fqdn_listname - >>> import os - >>> os.path.exists(os.path.join( - ... config.PUBLIC_ARCHIVE_FILE_DIR, listname, 'pipermail.pck')) - True diff --git a/src/mailman/queue/docs/command.txt b/src/mailman/queue/docs/command.txt deleted file mode 100644 index c767e6f5f..000000000 --- a/src/mailman/queue/docs/command.txt +++ /dev/null @@ -1,289 +0,0 @@ -======================== -The command queue runner -======================== - -This queue runner's purpose is to process and respond to email commands. -Commands are extensible using the Mailman plug-in system, but Mailman comes -with a number of email commands out of the box. These are processed when a -message is sent to the list's ``-request`` address. - - >>> mlist = create_list('test@example.com') - - -A command in the Subject -======================== - -For example, the ``echo`` command simply echoes the original command back to -the sender. The command can be in the ``Subject`` header. -:: - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: test-request@example.com - ... Subject: echo hello - ... Message-ID: <aardvark> - ... - ... """) - - >>> from mailman.inject import inject_message - >>> inject_message(mlist, msg, switchboard='command') - >>> from mailman.queue.command import CommandRunner - >>> from mailman.testing.helpers import make_testable_runner - >>> command = make_testable_runner(CommandRunner) - >>> command.run() - -And now the response is in the ``virgin`` queue. - - >>> from mailman.queue import Switchboard - >>> virgin_queue = config.switchboards['virgin'] - >>> len(virgin_queue.files) - 1 - >>> from mailman.testing.helpers import get_queue_messages - >>> item = get_queue_messages('virgin')[0] - >>> print item.msg.as_string() - Subject: The results of your email commands - From: test-bounces@example.com - To: aperson@example.com - ... - <BLANKLINE> - The results of your email command are provided below. - <BLANKLINE> - - Original message details: - From: aperson@example.com - Subject: echo hello - Date: ... - Message-ID: <aardvark> - <BLANKLINE> - - Results: - echo hello - <BLANKLINE> - - Done. - <BLANKLINE> - >>> dump_msgdata(item.msgdata) - _parsemsg : False - listname : test@example.com - nodecorate : True - recipients : set([u'aperson@example.com']) - reduced_list_headers: True - version : ... - - -A command in the body -===================== - -The command can also be found in the body of the message, as long as the -message is plain text. -:: - - >>> msg = message_from_string("""\ - ... From: bperson@example.com - ... To: test-request@example.com - ... Message-ID: <bobcat> - ... - ... echo foo bar - ... """) - - >>> inject_message(mlist, msg, switchboard='command') - >>> command.run() - >>> len(virgin_queue.files) - 1 - >>> item = get_queue_messages('virgin')[0] - >>> print item.msg.as_string() - Subject: The results of your email commands - From: test-bounces@example.com - To: bperson@example.com - ... - Precedence: bulk - <BLANKLINE> - The results of your email command are provided below. - <BLANKLINE> - - Original message details: - From: bperson@example.com - Subject: n/a - Date: ... - Message-ID: <bobcat> - <BLANKLINE> - - Results: - echo foo bar - <BLANKLINE> - - Done. - <BLANKLINE> - - -Implicit commands -================= - -For some commands, specifically for joining and leaving a mailing list, there -are email aliases that act like commands, even when there's nothing else in -the ``Subject`` or body. For example, to join a mailing list, a user need -only email the ``-join`` address or ``-subscribe`` address (the latter is -deprecated). - -Because Dirk has never registered with Mailman before, he gets two responses. -The first is a confirmation message so that Dirk can validate his email -address, and the other is the results of his email command. -:: - - >>> msg = message_from_string("""\ - ... From: Dirk Person <dperson@example.com> - ... To: test-join@example.com - ... - ... """) - - >>> inject_message(mlist, msg, switchboard='command', subaddress='join') - >>> command.run() - >>> len(virgin_queue.files) - 2 - - >>> def sortkey(item): - ... return str(item.msg['subject']) - >>> messages = sorted(get_queue_messages('virgin'), key=sortkey) - - >>> from mailman.interfaces.registrar import IRegistrar - >>> from zope.component import getUtility - >>> registrar = getUtility(IRegistrar) - >>> for item in messages: - ... subject = item.msg['subject'] - ... print 'Subject:', subject - ... if 'confirm' in str(subject): - ... token = str(subject).split()[1].strip() - ... status = registrar.confirm(token) - ... assert status, 'Confirmation failed' - Subject: The results of your email commands - Subject: confirm ... - -Similarly, to leave a mailing list, the user need only email the ``-leave`` or -``-unsubscribe`` address (the latter is deprecated). -:: - - >>> msg = message_from_string("""\ - ... From: dperson@example.com - ... To: test-leave@example.com - ... - ... """) - - >>> inject_message(mlist, msg, switchboard='command', subaddress='leave') - >>> command.run() - >>> len(virgin_queue.files) - 1 - >>> item = get_queue_messages('virgin')[0] - >>> print item.msg.as_string() - Subject: The results of your email commands - From: test-bounces@example.com - To: dperson@example.com - ... - <BLANKLINE> - The results of your email command are provided below. - <BLANKLINE> - - Original message details: - From: dperson@example.com - Subject: n/a - Date: ... - Message-ID: ... - <BLANKLINE> - - Results: - Dirk Person <dperson@example.com> left test@example.com - <BLANKLINE> - - Done. - <BLANKLINE> - -The ``-confirm`` address is also available as an implicit command. -:: - - >>> msg = message_from_string("""\ - ... From: dperson@example.com - ... To: test-confirm+123@example.com - ... - ... """) - - >>> inject_message(mlist, msg, switchboard='command', subaddress='confirm') - >>> command.run() - >>> len(virgin_queue.files) - 1 - >>> item = get_queue_messages('virgin')[0] - >>> print item.msg.as_string() - Subject: The results of your email commands - From: test-bounces@example.com - To: dperson@example.com - ... - <BLANKLINE> - The results of your email command are provided below. - <BLANKLINE> - - Original message details: - From: dperson@example.com - Subject: n/a - Date: ... - Message-ID: ... - <BLANKLINE> - - Results: - Confirmation token did not match - <BLANKLINE> - - Done. - <BLANKLINE> - - -Stopping command processing -=========================== - -The ``end`` command stops email processing, so that nothing following is -looked at by the command queue. -:: - - >>> msg = message_from_string("""\ - ... From: cperson@example.com - ... To: test-request@example.com - ... Message-ID: <caribou> - ... - ... echo foo bar - ... end ignored - ... echo baz qux - ... """) - - >>> inject_message(mlist, msg, switchboard='command') - >>> command.run() - >>> len(virgin_queue.files) - 1 - >>> item = get_queue_messages('virgin')[0] - >>> print item.msg.as_string() - Subject: The results of your email commands - ... - <BLANKLINE> - - Results: - echo foo bar - <BLANKLINE> - - Unprocessed: - echo baz qux - <BLANKLINE> - - Done. - <BLANKLINE> - -The ``stop`` command is an alias for ``end``. -:: - - >>> msg = message_from_string("""\ - ... From: cperson@example.com - ... To: test-request@example.com - ... Message-ID: <caribou> - ... - ... echo foo bar - ... stop ignored - ... echo baz qux - ... """) - - >>> inject_message(mlist, msg, switchboard='command') - >>> command.run() - >>> len(virgin_queue.files) - 1 - >>> item = get_queue_messages('virgin')[0] - >>> print item.msg.as_string() - Subject: The results of your email commands - ... - <BLANKLINE> - - Results: - echo foo bar - <BLANKLINE> - - Unprocessed: - echo baz qux - <BLANKLINE> - - Done. - <BLANKLINE> diff --git a/src/mailman/queue/docs/digester.txt b/src/mailman/queue/docs/digester.txt deleted file mode 100644 index 285b2072a..000000000 --- a/src/mailman/queue/docs/digester.txt +++ /dev/null @@ -1,604 +0,0 @@ -========= -Digesting -========= - -Mailman crafts and sends digests by a separate digest queue runner process. -This starts by a number of messages being posted to the mailing list. -:: - - >>> mlist = create_list('test@example.com') - >>> mlist.digest_size_threshold = 0.6 - >>> mlist.volume = 1 - >>> mlist.next_digest_number = 1 - - >>> from string import Template - >>> process = config.handlers['to-digest'].process - - >>> def fill_digest(): - ... size = 0 - ... for i in range(1, 5): - ... text = Template("""\ - ... From: aperson@example.com - ... To: xtest@example.com - ... Subject: Test message $i - ... List-Post: <test@example.com> - ... - ... Here is message $i - ... """).substitute(i=i) - ... msg = message_from_string(text) - ... process(mlist, msg, {}) - ... size += len(text) - ... if size >= mlist.digest_size_threshold * 1024: - ... break - - >>> fill_digest() - -The queue runner gets kicked off when a marker message gets dropped into the -digest queue. The message metadata points to the mailbox file containing the -messages to put in the digest. -:: - - >>> digestq = config.switchboards['digest'] - >>> len(digestq.files) - 1 - - >>> from mailman.testing.helpers import get_queue_messages - >>> entry = get_queue_messages('digest')[0] - -The marker message is empty. - - >>> print entry.msg.as_string() - -But the message metadata has a reference to the digest file. -:: - - >>> dump_msgdata(entry.msgdata) - _parsemsg : False - digest_number: 1 - digest_path : .../lists/test@example.com/digest.1.1.mmdf - listname : test@example.com - version : 3 - volume : 1 - - # Put the messages back in the queue for the runner to handle. - >>> filebase = digestq.enqueue(entry.msg, entry.msgdata) - -There are 4 messages in the digest. - - >>> from mailman.utilities.mailbox import Mailbox - >>> sum(1 for item in Mailbox(entry.msgdata['digest_path'])) - 4 - -When the queue runner runs, it processes the digest mailbox, crafting both the -plain text (RFC 1153) digest and the MIME digest. - - >>> from mailman.queue.digest import DigestRunner - >>> from mailman.testing.helpers import make_testable_runner - >>> runner = make_testable_runner(DigestRunner) - >>> runner.run() - -The digest runner places both digests into the virgin queue for final -delivery. - - >>> messages = get_queue_messages('virgin') - >>> len(messages) - 2 - -The MIME digest is a multipart, and the RFC 1153 digest is the other one. -:: - - >>> def mime_rfc1153(messages): - ... if messages[0].msg.is_multipart(): - ... return messages[0], messages[1] - ... return messages[1], messages[0] - - >>> mime, rfc1153 = mime_rfc1153(messages) - -The MIME digest has lots of good stuff, all contained in the multipart. - - >>> print mime.msg.as_string() - Content-Type: multipart/mixed; boundary="===============...==" - MIME-Version: 1.0 - From: test-request@example.com - Subject: Test Digest, Vol 1, Issue 1 - To: test@example.com - Reply-To: test@example.com - Date: ... - Message-ID: ... - <BLANKLINE> - --===============...== - Content-Type: text/plain; charset="us-ascii" - MIME-Version: 1.0 - Content-Transfer-Encoding: 7bit - Content-Description: Test Digest, Vol 1, Issue 1 - <BLANKLINE> - Send Test mailing list submissions to - test@example.com - <BLANKLINE> - To subscribe or unsubscribe via the World Wide Web, visit - http://lists.example.com/listinfo/test@example.com - or, via email, send a message with subject or body 'help' to - test-request@example.com - <BLANKLINE> - You can reach the person managing the list at - test-owner@example.com - <BLANKLINE> - When replying, please edit your Subject line so it is more specific - than "Re: Contents of Test digest..." - <BLANKLINE> - --===============...== - Content-Type: text/plain; charset="us-ascii" - MIME-Version: 1.0 - Content-Transfer-Encoding: 7bit - Content-Description: Today's Topics (4 messages) - <BLANKLINE> - Today's Topics: - <BLANKLINE> - 1. Test message 1 (aperson@example.com) - 2. Test message 2 (aperson@example.com) - 3. Test message 3 (aperson@example.com) - 4. Test message 4 (aperson@example.com) - <BLANKLINE> - --===============...== - Content-Type: message/rfc822 - MIME-Version: 1.0 - <BLANKLINE> - From: aperson@example.com - To: xtest@example.com - Subject: Test message 1 - List-Post: <test@example.com> - <BLANKLINE> - Here is message 1 - <BLANKLINE> - --===============...== - Content-Type: message/rfc822 - MIME-Version: 1.0 - <BLANKLINE> - From: aperson@example.com - To: xtest@example.com - Subject: Test message 2 - List-Post: <test@example.com> - <BLANKLINE> - Here is message 2 - <BLANKLINE> - --===============...== - Content-Type: message/rfc822 - MIME-Version: 1.0 - <BLANKLINE> - From: aperson@example.com - To: xtest@example.com - Subject: Test message 3 - List-Post: <test@example.com> - <BLANKLINE> - Here is message 3 - <BLANKLINE> - --===============...== - Content-Type: message/rfc822 - MIME-Version: 1.0 - <BLANKLINE> - From: aperson@example.com - To: xtest@example.com - Subject: Test message 4 - List-Post: <test@example.com> - <BLANKLINE> - Here is message 4 - <BLANKLINE> - --===============...== - Content-Type: text/plain; charset="us-ascii" - MIME-Version: 1.0 - Content-Transfer-Encoding: 7bit - Content-Description: Digest Footer - <BLANKLINE> - _______________________________________________ - Test mailing list - test@example.com - http://lists.example.com/listinfo/test@example.com - <BLANKLINE> - --===============...==-- - -The RFC 1153 contains the digest in a single plain text message. - - >>> print rfc1153.msg.as_string() - From: test-request@example.com - Subject: Test Digest, Vol 1, Issue 1 - To: test@example.com - Reply-To: test@example.com - Date: ... - Message-ID: ... - MIME-Version: 1.0 - Content-Type: text/plain; charset="us-ascii" - Content-Transfer-Encoding: 7bit - <BLANKLINE> - Send Test mailing list submissions to - test@example.com - <BLANKLINE> - To subscribe or unsubscribe via the World Wide Web, visit - http://lists.example.com/listinfo/test@example.com - or, via email, send a message with subject or body 'help' to - test-request@example.com - <BLANKLINE> - You can reach the person managing the list at - test-owner@example.com - <BLANKLINE> - When replying, please edit your Subject line so it is more specific - than "Re: Contents of Test digest..." - <BLANKLINE> - <BLANKLINE> - Today's Topics: - <BLANKLINE> - 1. Test message 1 (aperson@example.com) - 2. Test message 2 (aperson@example.com) - 3. Test message 3 (aperson@example.com) - 4. Test message 4 (aperson@example.com) - <BLANKLINE> - <BLANKLINE> - ---------------------------------------------------------------------- - <BLANKLINE> - From: aperson@example.com - Subject: Test message 1 - To: xtest@example.com - Message-ID: ... - <BLANKLINE> - Here is message 1 - <BLANKLINE> - ------------------------------ - <BLANKLINE> - From: aperson@example.com - Subject: Test message 2 - To: xtest@example.com - Message-ID: ... - <BLANKLINE> - Here is message 2 - <BLANKLINE> - ------------------------------ - <BLANKLINE> - From: aperson@example.com - Subject: Test message 3 - To: xtest@example.com - Message-ID: ... - <BLANKLINE> - Here is message 3 - <BLANKLINE> - ------------------------------ - <BLANKLINE> - From: aperson@example.com - Subject: Test message 4 - To: xtest@example.com - Message-ID: ... - <BLANKLINE> - Here is message 4 - <BLANKLINE> - ------------------------------ - <BLANKLINE> - _______________________________________________ - Test mailing list - test@example.com - http://lists.example.com/listinfo/test@example.com - <BLANKLINE> - <BLANKLINE> - End of Test Digest, Vol 1, Issue 1 - ********************************** - <BLANKLINE> - - -Internationalized digests -========================= - -When messages come in with a content-type character set different than that of -the list's preferred language, recipients will get an internationalized -digest. French is not enabled by default site-wide, so enable that now. -:: - - # Simulate the site administrator setting the default server language to - # French in the configuration file. Without this, the English template - # will be found and the masthead won't be translated. - >>> config.push('french', """ - ... [mailman] - ... default_language: fr - ... """) - - >>> mlist.preferred_language = 'fr' - >>> msg = message_from_string("""\ - ... From: aperson@example.org - ... To: test@example.com - ... Subject: =?iso-2022-jp?b?GyRCMGxIVhsoQg==?= - ... MIME-Version: 1.0 - ... Content-Type: text/plain; charset=iso-2022-jp - ... Content-Transfer-Encoding: 7bit - ... - ... \x1b$B0lHV\x1b(B - ... """) - -Set the digest threshold to zero so that the digests will be sent immediately. - - >>> mlist.digest_size_threshold = 0 - >>> process(mlist, msg, {}) - -The marker message is sitting in the digest queue. - - >>> len(digestq.files) - 1 - >>> entry = get_queue_messages('digest')[0] - >>> dump_msgdata(entry.msgdata) - _parsemsg : False - digest_number: 2 - digest_path : .../lists/test@example.com/digest.1.2.mmdf - listname : test@example.com - version : 3 - volume : 1 - -The digest queue runner runs a loop, placing the two digests into the virgin -queue. -:: - - # Put the messages back in the queue for the runner to handle. - >>> filebase = digestq.enqueue(entry.msg, entry.msgdata) - >>> runner.run() - >>> messages = get_queue_messages('virgin') - >>> len(messages) - 2 - -One of which is the MIME digest and the other of which is the RFC 1153 digest. - - >>> mime, rfc1153 = mime_rfc1153(messages) - -You can see that the digests contain a mix of French and Japanese. - - >>> print mime.msg.as_string() - Content-Type: multipart/mixed; boundary="===============...==" - MIME-Version: 1.0 - From: test-request@example.com - Subject: Groupe Test, Vol. 1, Parution 2 - To: test@example.com - Reply-To: test@example.com - Date: ... - Message-ID: ... - <BLANKLINE> - --===============...== - Content-Type: text/plain; charset="iso-8859-1" - MIME-Version: 1.0 - Content-Transfer-Encoding: quoted-printable - Content-Description: Groupe Test, Vol. 1, Parution 2 - <BLANKLINE> - Envoyez vos messages pour la liste Test =E0 - test@example.com - <BLANKLINE> - Pour vous (d=E9s)abonner par le web, consultez - http://lists.example.com/listinfo/test@example.com - <BLANKLINE> - ou, par courriel, envoyez un message avec =AB=A0help=A0=BB dans le corps ou - dans le sujet =E0 - test-request@example.com - <BLANKLINE> - Vous pouvez contacter l'administrateur de la liste =E0 l'adresse - test-owner@example.com - <BLANKLINE> - Si vous r=E9pondez, n'oubliez pas de changer l'objet du message afin - qu'il soit plus sp=E9cifique que =AB=A0Re: Contenu du groupe de Test...=A0= - =BB - --===============...== - Content-Type: text/plain; charset="utf-8" - MIME-Version: 1.0 - Content-Transfer-Encoding: base64 - Content-Description: Today's Topics (1 messages) - <BLANKLINE> - VGjDqG1lcyBkdSBqb3VyIDoKCiAgIDEuIOS4gOeVqiAoYXBlcnNvbkBleGFtcGxlLm9yZykK - <BLANKLINE> - --===============...== - Content-Type: message/rfc822 - MIME-Version: 1.0 - <BLANKLINE> - From: aperson@example.org - To: test@example.com - Subject: =?iso-2022-jp?b?GyRCMGxIVhsoQg==?= - MIME-Version: 1.0 - Content-Type: text/plain; charset=iso-2022-jp - Content-Transfer-Encoding: 7bit - <BLANKLINE> - $B0lHV(B - <BLANKLINE> - --===============...== - Content-Type: text/plain; charset="iso-8859-1" - MIME-Version: 1.0 - Content-Transfer-Encoding: quoted-printable - Content-Description: =?utf-8?q?Pied_de_page_des_remises_group=C3=A9es?= - <BLANKLINE> - _______________________________________________ - Test mailing list - test@example.com - http://lists.example.com/listinfo/test@example.com - <BLANKLINE> - --===============...==-- - -The RFC 1153 digest will be encoded in UTF-8 since it contains a mixture of -French and Japanese characters. - - >>> print rfc1153.msg.as_string() - From: test-request@example.com - Subject: Groupe Test, Vol. 1, Parution 2 - To: test@example.com - Reply-To: test@example.com - Date: ... - Message-ID: ... - MIME-Version: 1.0 - Content-Type: text/plain; charset="utf-8" - Content-Transfer-Encoding: base64 - <BLANKLINE> - RW52b... - <BLANKLINE> - -The content can be decoded to see the actual digest text. -:: - - # We must display the repr of the decoded value because doctests cannot - # handle the non-ascii characters. - >>> [repr(line) - ... for line in rfc1153.msg.get_payload(decode=True).splitlines()] - ["'Envoyez vos messages pour la liste Test \\xc3\\xa0'", - "'\\ttest@example.com'", - "''", - "'Pour vous (d\\xc3\\xa9s)abonner par le web, consultez'", - "'\\thttp://lists.example.com/listinfo/test@example.com'", - "''", - "'ou, par courriel, envoyez un message avec \\xc2\\xab\\xc2\\xa0... - "'dans le sujet \\xc3\\xa0'", - "'\\ttest-request@example.com'", - "''", - '"Vous pouvez contacter l\'administrateur de la liste \\xc3\\xa0 ... - "'\\ttest-owner@example.com'", - "''", - '"Si vous r\\xc3\\xa9pondez, n\'oubliez pas de changer l\'objet du ... - '"qu\'il soit plus sp\\xc3\\xa9cifique que \\xc2\\xab\\xc2\\xa0Re: ... - "''", - "'Th\\xc3\\xa8mes du jour :'", - "''", - "' 1. \\xe4\\xb8\\x80\\xe7\\x95\\xaa (aperson@example.org)'", - "''", - "''", - "'---------------------------------------------------------------------... - "''", - "'From: aperson@example.org'", - "'Subject: \\xe4\\xb8\\x80\\xe7\\x95\\xaa'", - "'To: test@example.com'", - "'Message-ID: ... - "'Content-Type: text/plain; charset=iso-2022-jp'", - "''", - "'\\xe4\\xb8\\x80\\xe7\\x95\\xaa'", - "''", - "'------------------------------'", - "''", - "'_______________________________________________'", - "'Test mailing list'", - "'test@example.com'", - "'http://lists.example.com/listinfo/test@example.com'", - "''", - "''", - "'Fin de Groupe Test, Vol. 1, Parution 2'", - "'**************************************'"] - - >>> config.pop('french') - - -Digest delivery -=============== - -A mailing list's members can choose to receive normal delivery, plain text -digests, or MIME digests. -:: - - >>> len(get_queue_messages('virgin')) - 0 - - >>> from mailman.interfaces.usermanager import IUserManager - >>> from zope.component import getUtility - >>> user_manager = getUtility(IUserManager) - - >>> from mailman.interfaces.member import DeliveryMode, MemberRole - >>> def subscribe(email, mode): - ... address = user_manager.create_address(email) - ... member = mlist.subscribe(address, MemberRole.member) - ... member.preferences.delivery_mode = mode - ... return member - -Two regular delivery members subscribe to the mailing list. - - >>> member_1 = subscribe('uperson@example.com', DeliveryMode.regular) - >>> member_2 = subscribe('vperson@example.com', DeliveryMode.regular) - -Two MIME digest members subscribe to the mailing list. - - >>> member_3 = subscribe('wperson@example.com', DeliveryMode.mime_digests) - >>> member_4 = subscribe('xperson@example.com', DeliveryMode.mime_digests) - -One RFC 1153 digest member subscribes to the mailing list. - - >>> member_5 = subscribe( - ... 'yperson@example.com', DeliveryMode.plaintext_digests) - >>> member_6 = subscribe( - ... 'zperson@example.com', DeliveryMode.plaintext_digests) - -When a digest gets sent, the appropriate recipient list is chosen. - - >>> mlist.preferred_language = 'en' - >>> mlist.digest_size_threshold = 0.5 - >>> fill_digest() - >>> runner.run() - -The digests are sitting in the virgin queue. One of them is the MIME digest -and the other is the RFC 1153 digest. -:: - - >>> messages = get_queue_messages('virgin') - >>> len(messages) - 2 - - >>> mime, rfc1153 = mime_rfc1153(messages) - -Only wperson and xperson get the MIME digests. - - >>> sorted(mime.msgdata['recipients']) - [u'wperson@example.com', u'xperson@example.com'] - -Only yperson and zperson get the RFC 1153 digests. - - >>> sorted(rfc1153.msgdata['recipients']) - [u'yperson@example.com', u'zperson@example.com'] - -Now uperson decides that they would like to start receiving digests too. -:: - - >>> member_1.preferences.delivery_mode = DeliveryMode.mime_digests - >>> fill_digest() - >>> runner.run() - - >>> messages = get_queue_messages('virgin') - >>> len(messages) - 2 - - >>> mime, rfc1153 = mime_rfc1153(messages) - >>> sorted(mime.msgdata['recipients']) - [u'uperson@example.com', u'wperson@example.com', u'xperson@example.com'] - - >>> sorted(rfc1153.msgdata['recipients']) - [u'yperson@example.com', u'zperson@example.com'] - -At this point, both uperson and wperson decide that they'd rather receive -regular deliveries instead of digests. uperson would like to get any last -digest that may be sent so that she doesn't miss anything. wperson does care -as much and does not want to receive one last digest. -:: - - >>> mlist.send_one_last_digest_to( - ... member_1.address, member_1.preferences.delivery_mode) - - >>> member_1.preferences.delivery_mode = DeliveryMode.regular - >>> member_3.preferences.delivery_mode = DeliveryMode.regular - - >>> fill_digest() - >>> runner.run() - - >>> messages = get_queue_messages('virgin') - >>> mime, rfc1153 = mime_rfc1153(messages) - >>> sorted(mime.msgdata['recipients']) - [u'uperson@example.com', u'xperson@example.com'] - - >>> sorted(rfc1153.msgdata['recipients']) - [u'yperson@example.com', u'zperson@example.com'] - -Since uperson has received their last digest, they will not get any more of -them. -:: - - >>> fill_digest() - >>> runner.run() - - >>> messages = get_queue_messages('virgin') - >>> len(messages) - 2 - - >>> mime, rfc1153 = mime_rfc1153(messages) - >>> sorted(mime.msgdata['recipients']) - [u'xperson@example.com'] - - >>> sorted(rfc1153.msgdata['recipients']) - [u'yperson@example.com', u'zperson@example.com'] diff --git a/src/mailman/queue/docs/incoming.txt b/src/mailman/queue/docs/incoming.txt deleted file mode 100644 index 6455db20b..000000000 --- a/src/mailman/queue/docs/incoming.txt +++ /dev/null @@ -1,263 +0,0 @@ -========================= -The incoming queue runner -========================= - -This runner's sole purpose in life is to decide the disposition of the -message. It can either be accepted for delivery, rejected (i.e. bounced), -held for moderator approval, or discarded. - -The runner operates by processing chains on a message/metadata pair in the -context of a mailing list. Each mailing list may have a 'start chain' where -processing begins, with a global default. This chain is processed with the -message eventually ending up in one of the four disposition states described -above. - - >>> mlist = create_list('test@example.com') - >>> print mlist.start_chain - built-in - - -Sender addresses -================ - -The incoming queue runner ensures that the sender addresses on the message are -registered with the system. This is used for determining nonmember posting -privileges. The addresses will not be linked to a user and will be -unverified, so if the real user comes along later and claims the address, it -will be linked to their user account (and must be verified). - -While configurable, the *sender addresses* by default are those named in the -`From:`, `Sender:`, and `Reply-To:` headers, as well as the envelope sender -(though we won't worry about the latter). -:: - - >>> msg = message_from_string("""\ - ... From: zperson@example.com - ... Reply-To: yperson@example.com - ... Sender: xperson@example.com - ... To: test@example.com - ... Subject: This is spiced ham - ... Message-ID: <bogus> - ... - ... """) - - >>> from zope.component import getUtility - >>> from mailman.interfaces.usermanager import IUserManager - >>> user_manager = getUtility(IUserManager) - >>> print user_manager.get_address('xperson@example.com') - None - >>> print user_manager.get_address('yperson@example.com') - None - >>> print user_manager.get_address('zperson@example.com') - None - -Inject the message into the incoming queue, similar to the way the upstream -mail server normally would. - - >>> from mailman.inject import inject_message - >>> inject_message(mlist, msg) - -The incoming queue runner runs until it is empty. - - >>> from mailman.queue.incoming import IncomingRunner - >>> from mailman.testing.helpers import make_testable_runner - >>> incoming = make_testable_runner(IncomingRunner, 'in') - >>> incoming.run() - -And now the addresses are known to the system. As mentioned above, they are -not linked to a user and are unverified. - - >>> for localpart in ('xperson', 'yperson', 'zperson'): - ... email = '{0}@example.com'.format(localpart) - ... address = user_manager.get_address(email) - ... print '{0}; verified? {1}; user? {2}'.format( - ... address.email, - ... ('No' if address.verified_on is None else 'Yes'), - ... user_manager.get_user(email)) - xperson@example.com; verified? No; user? None - yperson@example.com; verified? No; user? None - zperson@example.com; verified? No; user? None - -.. - Clear the pipeline queue of artifacts that affect the following tests. - >>> from mailman.testing.helpers import get_queue_messages - >>> ignore = get_queue_messages('pipeline') - - -Accepted messages -================= - -We have a message that is going to be sent to the mailing list. Once Anne is -a member of the mailing list, this message is so perfectly fine for posting -that it will be accepted and forward to the pipeline queue. - - >>> from mailman.testing.helpers import subscribe - >>> subscribe(mlist, 'Anne') - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: test@example.com - ... Subject: My first post - ... Message-ID: <first> - ... - ... First post! - ... """) - -Inject the message into the incoming queue and run until the queue is empty. - - >>> inject_message(mlist, msg) - >>> incoming.run() - -Now the message is in the pipeline queue. - - >>> pipeline_queue = config.switchboards['pipeline'] - >>> len(pipeline_queue.files) - 1 - >>> incoming_queue = config.switchboards['in'] - >>> len(incoming_queue.files) - 0 - >>> item = get_queue_messages('pipeline')[0] - >>> print item.msg.as_string() - From: aperson@example.com - To: test@example.com - Subject: My first post - Message-ID: <first> - Date: ... - X-Mailman-Rule-Misses: approved; emergency; loop; member-moderation; - administrivia; implicit-dest; max-recipients; max-size; - news-moderation; no-subject; suspicious-header; nonmember-moderation - <BLANKLINE> - First post! - <BLANKLINE> - >>> dump_msgdata(item.msgdata) - _parsemsg : False - envsender : noreply@example.com - ... - - -Held messages -============= - -The list moderator sets the emergency flag on the mailing list. The built-in -chain will now hold all posted messages, so nothing will show up in the -pipeline queue. -:: - - >>> from mailman.chains.base import ChainNotification - >>> def on_chain(event): - ... if isinstance(event, ChainNotification): - ... print event - ... print event.chain - ... print 'From: {0}\nTo: {1}\nMessage-ID: {2}'.format( - ... event.msg['from'], event.msg['to'], - ... event.msg['message-id']) - - >>> mlist.emergency = True - - >>> from mailman.testing.helpers import event_subscribers - >>> with event_subscribers(on_chain): - ... inject_message(mlist, msg) - ... incoming.run() - <mailman.chains.hold.HoldNotification ...> - <mailman.chains.hold.HoldChain ...> - From: aperson@example.com - To: test@example.com - Message-ID: <first> - - >>> mlist.emergency = False - - -Discarded messages -================== - -Another possibility is that the message would get immediately discarded. The -built-in chain does not have such a disposition by default, so let's craft a -new chain and set it as the mailing list's start chain. -:: - - >>> from mailman.chains.base import Chain, Link - >>> from mailman.interfaces.chain import LinkAction - >>> def make_chain(name, target_chain): - ... truth_rule = config.rules['truth'] - ... target_chain = config.chains[target_chain] - ... test_chain = Chain(name, 'Testing {0}'.format(target_chain)) - ... config.chains[test_chain.name] = test_chain - ... link = Link(truth_rule, LinkAction.jump, target_chain) - ... test_chain.append_link(link) - ... return test_chain - - >>> test_chain = make_chain('always-discard', 'discard') - >>> mlist.start_chain = test_chain.name - - >>> msg.replace_header('message-id', '<second>') - >>> with event_subscribers(on_chain): - ... inject_message(mlist, msg) - ... incoming.run() - <mailman.chains.discard.DiscardNotification ...> - <mailman.chains.discard.DiscardChain ...> - From: aperson@example.com - To: test@example.com - Message-ID: <second> - - >>> del config.chains[test_chain.name] - -.. - The virgin queue needs to be cleared out due to artifacts from the - previous tests above. - - >>> virgin_queue = config.switchboards['virgin'] - >>> ignore = get_queue_messages('virgin') - - -Rejected messages -================= - -Similar to discarded messages, a message can be rejected, or bounced back to -the original sender. Again, the built-in chain doesn't support this so we'll -just create a new chain that does. - - >>> test_chain = make_chain('always-reject', 'reject') - >>> mlist.start_chain = test_chain.name - - >>> msg.replace_header('message-id', '<third>') - >>> with event_subscribers(on_chain): - ... inject_message(mlist, msg) - ... incoming.run() - <mailman.chains.reject.RejectNotification ...> - <mailman.chains.reject.RejectChain ...> - From: aperson@example.com - To: test@example.com - Message-ID: <third> - -The rejection message is sitting in the virgin queue waiting to be delivered -to the original sender. - - >>> len(virgin_queue.files) - 1 - >>> item = get_queue_messages('virgin')[0] - >>> print item.msg.as_string() - Subject: My first post - From: test-owner@example.com - To: aperson@example.com - ... - <BLANKLINE> - --===============... - Content-Type: text/plain; charset="us-ascii" - MIME-Version: 1.0 - Content-Transfer-Encoding: 7bit - <BLANKLINE> - [No bounce details are available] - --===============... - Content-Type: message/rfc822 - MIME-Version: 1.0 - <BLANKLINE> - From: aperson@example.com - To: test@example.com - Subject: My first post - Message-ID: <third> - Date: ... - <BLANKLINE> - First post! - <BLANKLINE> - --===============... - - >>> del config.chains['always-reject'] diff --git a/src/mailman/queue/docs/lmtp.txt b/src/mailman/queue/docs/lmtp.txt deleted file mode 100644 index c95c6aa2b..000000000 --- a/src/mailman/queue/docs/lmtp.txt +++ /dev/null @@ -1,313 +0,0 @@ -=========== -LTMP server -=========== - -Mailman can accept messages via LMTP (RFC 2033). Most modern mail servers -support LMTP local delivery, so this is a very portable way to connect Mailman -with your mail server. - -Our LMTP server is fairly simple though; all it does is make sure that the -message is destined for a valid endpoint, e.g. ``mylist-join@example.com``. - -Let's start a testable LMTP queue runner. - - >>> from mailman.testing import helpers - >>> master = helpers.TestableMaster() - >>> master.start('lmtp') - -It also helps to have a nice LMTP client. - - >>> lmtp = helpers.get_lmtp_client() - (220, '... Python LMTP queue runner 1.0') - >>> lmtp.lhlo('remote.example.org') - (250, ...) - - -Posting address -=============== - -If the mail server tries to send a message to a nonexistent mailing list, it -will get a 550 error. - - >>> lmtp.sendmail( - ... 'anne.person@example.com', - ... ['mylist@example.com'], """\ - ... From: anne.person@example.com - ... To: mylist@example.com - ... Subject: An interesting message - ... Message-ID: <aardvark> - ... - ... This is an interesting message. - ... """) - Traceback (most recent call last): - ... - SMTPDataError: (550, 'Requested action not taken: mailbox unavailable') - -Once the mailing list is created, the posting address is valid. -:: - - >>> create_list('mylist@example.com') - <mailing list "mylist@example.com" at ...> - - >>> transaction.commit() - >>> lmtp.sendmail( - ... 'anne.person@example.com', - ... ['mylist@example.com'], """\ - ... From: anne.person@example.com - ... To: mylist@example.com - ... Subject: An interesting message - ... Message-ID: <badger> - ... - ... This is an interesting message. - ... """) - {} - - >>> from mailman.testing.helpers import get_queue_messages - >>> messages = get_queue_messages('in') - >>> len(messages) - 1 - >>> print messages[0].msg.as_string() - From: anne.person@example.com - To: mylist@example.com - Subject: An interesting message - Message-ID: <badger> - X-MailFrom: anne.person@example.com - <BLANKLINE> - This is an interesting message. - >>> dump_msgdata(messages[0].msgdata) - _parsemsg : False - listname : mylist@example.com - original_size: ... - to_list : True - version : ... - - -Sub-addresses -============= - -The LMTP server understands each of the list's sub-addreses, such as -join, --leave, -request and so on. If the message is posted to an invalid -sub-address though, it is rejected. - - >>> lmtp.sendmail( - ... 'anne.person@example.com', - ... ['mylist-bogus@example.com'], """\ - ... From: anne.person@example.com - ... To: mylist-bogus@example.com - ... Subject: Help - ... Message-ID: <cow> - ... - ... Please help me. - ... """) - Traceback (most recent call last): - ... - SMTPDataError: (550, 'Requested action not taken: mailbox unavailable') - -But the message is accepted if posted to a valid sub-address. - - >>> lmtp.sendmail( - ... 'anne.person@example.com', - ... ['mylist-request@example.com'], """\ - ... From: anne.person@example.com - ... To: mylist-request@example.com - ... Subject: Help - ... Message-ID: <dog> - ... - ... Please help me. - ... """) - {} - - -Request subaddress ------------------- - -Depending on the subaddress, there is a message in the appropriate queue for -later processing. For example, all -request messages are put into the command -queue for processing. - - >>> messages = get_queue_messages('command') - >>> len(messages) - 1 - >>> print messages[0].msg.as_string() - From: anne.person@example.com - To: mylist-request@example.com - Subject: Help - Message-ID: <dog> - X-MailFrom: anne.person@example.com - <BLANKLINE> - Please help me. - >>> dump_msgdata(messages[0].msgdata) - _parsemsg : False - listname : mylist@example.com - original_size: ... - subaddress : request - version : ... - - -Bounce processor ----------------- - -A message to the -bounces address goes to the bounce processor. - - >>> lmtp.sendmail( - ... 'mail-daemon@example.com', - ... ['mylist-bounces@example.com'], """\ - ... From: mail-daemon@example.com - ... To: mylist-bounces@example.com - ... Subject: A bounce - ... Message-ID: <elephant> - ... - ... Bouncy bouncy. - ... """) - {} - >>> messages = get_queue_messages('bounces') - >>> len(messages) - 1 - >>> dump_msgdata(messages[0].msgdata) - _parsemsg : False - listname : mylist@example.com - original_size: ... - subaddress : bounces - version : ... - - -Command processor ------------------ - -Confirmation messages go to the command processor... - - >>> lmtp.sendmail( - ... 'anne.person@example.com', - ... ['mylist-confirm@example.com'], """\ - ... From: anne.person@example.com - ... To: mylist-confirm@example.com - ... Subject: A bounce - ... Message-ID: <falcon> - ... - ... confirm 123 - ... """) - {} - >>> messages = get_queue_messages('command') - >>> len(messages) - 1 - >>> dump_msgdata(messages[0].msgdata) - _parsemsg : False - listname : mylist@example.com - original_size: ... - subaddress : confirm - version : ... - -...as do join messages... -:: - - >>> lmtp.sendmail( - ... 'anne.person@example.com', - ... ['mylist-join@example.com'], """\ - ... From: anne.person@example.com - ... To: mylist-join@example.com - ... Message-ID: <giraffe> - ... - ... """) - {} - >>> messages = get_queue_messages('command') - >>> len(messages) - 1 - >>> dump_msgdata(messages[0].msgdata) - _parsemsg : False - listname : mylist@example.com - original_size: ... - subaddress : join - version : ... - - >>> lmtp.sendmail( - ... 'anne.person@example.com', - ... ['mylist-subscribe@example.com'], """\ - ... From: anne.person@example.com - ... To: mylist-subscribe@example.com - ... Message-ID: <hippopotamus> - ... - ... """) - {} - >>> messages = get_queue_messages('command') - >>> len(messages) - 1 - >>> dump_msgdata(messages[0].msgdata) - _parsemsg : False - listname : mylist@example.com - original_size: ... - subaddress : join - version : ... - -...and leave messages. -:: - - >>> lmtp.sendmail( - ... 'anne.person@example.com', - ... ['mylist-leave@example.com'], """\ - ... From: anne.person@example.com - ... To: mylist-leave@example.com - ... Message-ID: <iguana> - ... - ... """) - {} - >>> messages = get_queue_messages('command') - >>> len(messages) - 1 - >>> dump_msgdata(messages[0].msgdata) - _parsemsg : False - listname : mylist@example.com - original_size: ... - subaddress : leave - version : ... - - >>> lmtp.sendmail( - ... 'anne.person@example.com', - ... ['mylist-unsubscribe@example.com'], """\ - ... From: anne.person@example.com - ... To: mylist-unsubscribe@example.com - ... Message-ID: <jackal> - ... - ... """) - {} - >>> messages = get_queue_messages('command') - >>> len(messages) - 1 - >>> dump_msgdata(messages[0].msgdata) - _parsemsg : False - listname : mylist@example.com - original_size: ... - subaddress : leave - version : ... - - -Incoming processor ------------------- - -Messages to the -owner address go to the incoming processor. - - >>> lmtp.sendmail( - ... 'anne.person@example.com', - ... ['mylist-owner@example.com'], """\ - ... From: anne.person@example.com - ... To: mylist-owner@example.com - ... Message-ID: <kangaroo> - ... - ... """) - {} - >>> messages = get_queue_messages('in') - >>> len(messages) - 1 - >>> dump_msgdata(messages[0].msgdata) - _parsemsg : False - envsender : changeme@example.com - listname : mylist@example.com - original_size: ... - subaddress : owner - to_owner : True - version : ... - - -Clean up -======== - - >>> master.stop() diff --git a/src/mailman/queue/docs/news.txt b/src/mailman/queue/docs/news.txt deleted file mode 100644 index 7261aa333..000000000 --- a/src/mailman/queue/docs/news.txt +++ /dev/null @@ -1,161 +0,0 @@ -=============== -The news runner -=============== - -The news runner is the queue runner that gateways mailing list messages to an -NNTP newsgroup. One of the most important things this runner does is prepare -the message for Usenet (yes, I know that NNTP is not Usenet, but this runner -was originally written to gate to Usenet, which has its own rules). - - >>> mlist = create_list('_xtest@example.com') - >>> mlist.linked_newsgroup = 'comp.lang.python' - -Some NNTP servers such as INN reject messages containing a set of prohibited -headers, so one of the things that the news runner does is remove these -prohibited headers. -:: - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: _xtest@example.com - ... NNTP-Posting-Host: news.example.com - ... NNTP-Posting-Date: today - ... X-Trace: blah blah - ... X-Complaints-To: abuse@dom.ain - ... Xref: blah blah - ... Xref: blah blah - ... Date-Received: yesterday - ... Posted: tomorrow - ... Posting-Version: 99.99 - ... Relay-Version: 88.88 - ... Received: blah blah - ... - ... A message - ... """) - >>> msgdata = {} - - >>> from mailman.queue.news import prepare_message - >>> prepare_message(mlist, msg, msgdata) - >>> msgdata['prepped'] - True - >>> print msg.as_string() - From: aperson@example.com - To: _xtest@example.com - Newsgroups: comp.lang.python - Message-ID: ... - Lines: 1 - <BLANKLINE> - A message - <BLANKLINE> - -Some NNTP servers will reject messages where certain headers are duplicated, -so the news runner must collapse or move these duplicate headers to an -``X-Original-*`` header that the news server doesn't care about. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: _xtest@example.com - ... To: two@example.com - ... Cc: three@example.com - ... Cc: four@example.com - ... Cc: five@example.com - ... Content-Transfer-Encoding: yes - ... Content-Transfer-Encoding: no - ... Content-Transfer-Encoding: maybe - ... - ... A message - ... """) - >>> msgdata = {} - >>> prepare_message(mlist, msg, msgdata) - >>> msgdata['prepped'] - True - >>> print msg.as_string() - From: aperson@example.com - Newsgroups: comp.lang.python - Message-ID: ... - Lines: 1 - To: _xtest@example.com - X-Original-To: two@example.com - CC: three@example.com - X-Original-CC: four@example.com - X-Original-CC: five@example.com - Content-Transfer-Encoding: yes - X-Original-Content-Transfer-Encoding: no - X-Original-Content-Transfer-Encoding: maybe - <BLANKLINE> - A message - <BLANKLINE> - -But if no headers are duplicated, then the news runner doesn't need to modify -the message. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: _xtest@example.com - ... Cc: someother@example.com - ... Content-Transfer-Encoding: yes - ... - ... A message - ... """) - >>> msgdata = {} - >>> prepare_message(mlist, msg, msgdata) - >>> msgdata['prepped'] - True - >>> print msg.as_string() - From: aperson@example.com - To: _xtest@example.com - Cc: someother@example.com - Content-Transfer-Encoding: yes - Newsgroups: comp.lang.python - Message-ID: ... - Lines: 1 - <BLANKLINE> - A message - <BLANKLINE> - - -Newsgroup moderation -==================== - -When the newsgroup is moderated, an ``Approved:`` header with the list's -posting address is added for the benefit of the Usenet system. -:: - - >>> from mailman.interfaces.nntp import NewsModeration - >>> mlist.news_moderation = NewsModeration.open_moderated - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: _xtest@example.com - ... Approved: this gets deleted - ... - ... """) - >>> prepare_message(mlist, msg, {}) - >>> print msg['approved'] - _xtest@example.com - - >>> mlist.news_moderation = NewsModeration.moderated - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: _xtest@example.com - ... Approved: this gets deleted - ... - ... """) - >>> prepare_message(mlist, msg, {}) - >>> print msg['approved'] - _xtest@example.com - -But if the newsgroup is not moderated, the ``Approved:`` header is not changed. - - >>> mlist.news_moderation = NewsModeration.none - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: _xtest@example.com - ... Approved: this doesn't get deleted - ... - ... """) - >>> prepare_message(mlist, msg, {}) - >>> msg['approved'] - u"this doesn't get deleted" - - -XXX More of the NewsRunner should be tested. diff --git a/src/mailman/queue/docs/outgoing.txt b/src/mailman/queue/docs/outgoing.txt deleted file mode 100644 index 0af22b808..000000000 --- a/src/mailman/queue/docs/outgoing.txt +++ /dev/null @@ -1,413 +0,0 @@ -===================== -Outgoing queue runner -===================== - -The outgoing queue runner is the process that delivers messages to the -directly upstream SMTP server. It is this upstream SMTP server that performs -final delivery to the intended recipients. - -Messages that appear in the outgoing queue are processed individually through -a *delivery module*, essentially a pluggable interface for determining how the -recipient set will be batched, whether messages will be personalized and -VERP'd, etc. The outgoing runner doesn't itself support retrying but it can -move messages to the 'retry queue' for handling delivery failures. -:: - - >>> mlist = create_list('test@example.com') - - >>> from mailman.app.membership import add_member - >>> from mailman.interfaces.member import DeliveryMode - >>> add_member(mlist, 'aperson@example.com', 'Anne Person', - ... 'password', DeliveryMode.regular, 'en') - <Member: Anne Person <aperson@example.com> - on test@example.com as MemberRole.member> - >>> add_member(mlist, 'bperson@example.com', 'Bart Person', - ... 'password', DeliveryMode.regular, 'en') - <Member: Bart Person <bperson@example.com> - on test@example.com as MemberRole.member> - >>> add_member(mlist, 'cperson@example.com', 'Cris Person', - ... 'password', DeliveryMode.regular, 'en') - <Member: Cris Person <cperson@example.com> - on test@example.com as MemberRole.member> - -Normally, messages would show up in the outgoing queue after the message has -been processed by the rule set and pipeline. But we can simulate that here by -injecting a message directly into the outgoing queue. First though, we must -call the ``calculate-recipients`` handler so that the message metadata will be -populated with the list of addresses to deliver the message to. -:: - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: test@example.com - ... Subject: My first post - ... Message-ID: <first> - ... - ... First post! - ... """) - - >>> msgdata = {} - >>> handler = config.handlers['calculate-recipients'] - >>> handler.process(mlist, msg, msgdata) - >>> outgoing_queue = config.switchboards['out'] - -The ``to-outgoing`` handler populates the message metadata with the -destination mailing list name. Simulate that here too. - - >>> ignore = outgoing_queue.enqueue( - ... msg, msgdata, - ... tolist=True, - ... listname=mlist.fqdn_listname) - -Running the outgoing queue runner processes the message, delivering it to the -upstream SMTP. - - >>> from mailman.queue.outgoing import OutgoingRunner - >>> from mailman.testing.helpers import make_testable_runner - >>> outgoing = make_testable_runner(OutgoingRunner, 'out') - >>> outgoing.run() - -Every recipient got the same copy of the message. -:: - - >>> messages = list(smtpd.messages) - >>> len(messages) - 1 - - >>> print messages[0].as_string() - From: aperson@example.com - To: test@example.com - Subject: My first post - Message-ID: <first> - X-Peer: ... - X-MailFrom: test-bounces@example.com - X-RcptTo: cperson@example.com, bperson@example.com, aperson@example.com - <BLANKLINE> - First post! - - -Personalization -=============== - -Mailman supports sending individual messages to each recipient by -personalizing delivery. This increases the bandwidth between Mailman and the -upstream mail server, and between the upstream mail server and the remote -recipient mail servers. The benefit is that personalization provides for a -much better user experience, because the messages can be tailored for each -recipient. - - >>> from mailman.interfaces.mailinglist import Personalization - >>> mlist.personalize = Personalization.individual - >>> transaction.commit() - -Now when we send the message, our mail server will get three copies instead of -just one. - - >>> ignore = outgoing_queue.enqueue( - ... msg, msgdata, - ... listname=mlist.fqdn_listname) - >>> outgoing.run() - >>> messages = list(smtpd.messages) - >>> len(messages) - 3 - -Since we've done no other configuration, the only difference in the messages -is the recipient address. Specifically, the Sender header is the same for all -recipients. -:: - - >>> from operator import itemgetter - >>> def show_headers(messages): - ... for message in sorted(messages, key=itemgetter('x-rcptto')): - ... print message['X-RcptTo'], message['X-MailFrom'] - - >>> show_headers(messages) - aperson@example.com test-bounces@example.com - bperson@example.com test-bounces@example.com - cperson@example.com test-bounces@example.com - - -VERP -==== - -An even more interesting personalization opportunity arises if VERP_ is -enabled. Here, Mailman takes advantage of the fact that it's sending -individualized messages anyway, so it also encodes the recipients address in -the Sender header. - -.. _VERP: ../../mta/docs/verp.html - - -Forcing VERP ------------- - -A handler can force VERP by setting the ``verp`` key in the message metadata. -:: - - >>> ignore = outgoing_queue.enqueue( - ... msg, msgdata, - ... verp=True, - ... listname=mlist.fqdn_listname) - >>> outgoing.run() - >>> messages = list(smtpd.messages) - >>> len(messages) - 3 - - >>> show_headers(messages) - aperson@example.com test-bounces+aperson=example.com@example.com - bperson@example.com test-bounces+bperson=example.com@example.com - cperson@example.com test-bounces+cperson=example.com@example.com - - -VERP personalized deliveries ----------------------------- - -The site administrator can enable VERP whenever messages are personalized. - - >>> config.push('verp', """ - ... [mta] - ... verp_personalized_deliveries: yes - ... """) - -Again, we get three individual messages, with VERP'd ``Sender`` headers. -:: - - >>> ignore = outgoing_queue.enqueue( - ... msg, msgdata, - ... listname=mlist.fqdn_listname) - >>> outgoing.run() - >>> messages = list(smtpd.messages) - >>> len(messages) - 3 - - >>> show_headers(messages) - aperson@example.com test-bounces+aperson=example.com@example.com - bperson@example.com test-bounces+bperson=example.com@example.com - cperson@example.com test-bounces+cperson=example.com@example.com - - >>> config.pop('verp') - >>> mlist.personalize = Personalization.none - >>> transaction.commit() - - -VERP every once in a while --------------------------- - -Perhaps personalization is too much of an overhead, but the list owners would -still like to occasionally get the benefits of VERP. The site administrator -can enable occasional VERPing of messages every so often, by setting a -delivery interval. Every N non-personalized deliveries turns on VERP for just -the next one. -:: - - >>> config.push('verp occasionally', """ - ... [mta] - ... verp_delivery_interval: 3 - ... """) - - # Reset the list's post_id, which is used to calculate the intervals. - >>> mlist.post_id = 1 - >>> transaction.commit() - -The first message is sent to the list, and it is neither personalized nor -VERP'd. -:: - - >>> ignore = outgoing_queue.enqueue( - ... msg, msgdata, - ... listname=mlist.fqdn_listname) - >>> outgoing.run() - >>> messages = list(smtpd.messages) - >>> len(messages) - 1 - - >>> show_headers(messages) - cperson@example.com, bperson@example.com, aperson@example.com - test-bounces@example.com - - # Perform post-delivery bookkeeping. - >>> after = config.handlers['after-delivery'] - >>> after.process(mlist, msg, msgdata) - >>> transaction.commit() - -The second message sent to the list is also not VERP'd. -:: - - >>> ignore = outgoing_queue.enqueue( - ... msg, msgdata, - ... listname=mlist.fqdn_listname) - >>> outgoing.run() - >>> messages = list(smtpd.messages) - >>> len(messages) - 1 - - >>> show_headers(messages) - cperson@example.com, bperson@example.com, aperson@example.com - test-bounces@example.com - - # Perform post-delivery bookkeeping. - >>> after.process(mlist, msg, msgdata) - >>> transaction.commit() - -The third message though is VERP'd. -:: - - >>> ignore = outgoing_queue.enqueue( - ... msg, msgdata, - ... listname=mlist.fqdn_listname) - >>> outgoing.run() - >>> messages = list(smtpd.messages) - >>> len(messages) - 3 - - >>> show_headers(messages) - aperson@example.com test-bounces+aperson=example.com@example.com - bperson@example.com test-bounces+bperson=example.com@example.com - cperson@example.com test-bounces+cperson=example.com@example.com - - # Perform post-delivery bookkeeping. - >>> after.process(mlist, msg, msgdata) - >>> transaction.commit() - -The next one is back to bulk delivery. -:: - - >>> ignore = outgoing_queue.enqueue( - ... msg, msgdata, - ... listname=mlist.fqdn_listname) - >>> outgoing.run() - >>> messages = list(smtpd.messages) - >>> len(messages) - 1 - - >>> show_headers(messages) - cperson@example.com, bperson@example.com, aperson@example.com - test-bounces@example.com - - >>> config.pop('verp occasionally') - - -VERP every time ---------------- - -If the site administrator wants to enable VERP for every delivery, even if no -personalization is going on, they can set the interval to 1. -:: - - >>> config.push('always verp', """ - ... [mta] - ... verp_delivery_interval: 1 - ... """) - - # Reset the list's post_id, which is used to calculate the intervals. - >>> mlist.post_id = 1 - >>> transaction.commit() - -The first message is VERP'd. -:: - - >>> ignore = outgoing_queue.enqueue( - ... msg, msgdata, - ... listname=mlist.fqdn_listname) - >>> outgoing.run() - >>> messages = list(smtpd.messages) - >>> len(messages) - 3 - - >>> show_headers(messages) - aperson@example.com test-bounces+aperson=example.com@example.com - bperson@example.com test-bounces+bperson=example.com@example.com - cperson@example.com test-bounces+cperson=example.com@example.com - - # Perform post-delivery bookkeeping. - >>> after.process(mlist, msg, msgdata) - >>> transaction.commit() - -As is the second message. -:: - - >>> ignore = outgoing_queue.enqueue( - ... msg, msgdata, - ... listname=mlist.fqdn_listname) - >>> outgoing.run() - >>> messages = list(smtpd.messages) - >>> len(messages) - 3 - - >>> show_headers(messages) - aperson@example.com test-bounces+aperson=example.com@example.com - bperson@example.com test-bounces+bperson=example.com@example.com - cperson@example.com test-bounces+cperson=example.com@example.com - - # Perform post-delivery bookkeeping. - >>> after.process(mlist, msg, msgdata) - >>> transaction.commit() - -And the third message. -:: - - >>> ignore = outgoing_queue.enqueue( - ... msg, msgdata, - ... listname=mlist.fqdn_listname) - >>> outgoing.run() - >>> messages = list(smtpd.messages) - >>> len(messages) - 3 - - >>> show_headers(messages) - aperson@example.com test-bounces+aperson=example.com@example.com - bperson@example.com test-bounces+bperson=example.com@example.com - cperson@example.com test-bounces+cperson=example.com@example.com - - # Perform post-delivery bookkeeping. - >>> after.process(mlist, msg, msgdata) - >>> transaction.commit() - - >>> config.pop('always verp') - - -Never VERP ----------- - -Similarly, the site administrator can disable occasional VERP'ing of -non-personalized messages by setting the interval to zero. -:: - - >>> config.push('never verp', """ - ... [mta] - ... verp_delivery_interval: 0 - ... """) - - # Reset the list's post_id, which is used to calculate the intervals. - >>> mlist.post_id = 1 - >>> transaction.commit() - -Neither the first message... -:: - - >>> ignore = outgoing_queue.enqueue( - ... msg, msgdata, - ... listname=mlist.fqdn_listname) - >>> outgoing.run() - >>> messages = list(smtpd.messages) - >>> len(messages) - 1 - - >>> show_headers(messages) - cperson@example.com, bperson@example.com, aperson@example.com - test-bounces@example.com - -...nor the second message is VERP'd. -:: - - >>> ignore = outgoing_queue.enqueue( - ... msg, msgdata, - ... listname=mlist.fqdn_listname) - >>> outgoing.run() - >>> messages = list(smtpd.messages) - >>> len(messages) - 1 - - >>> show_headers(messages) - cperson@example.com, bperson@example.com, aperson@example.com - test-bounces@example.com diff --git a/src/mailman/queue/docs/rest.txt b/src/mailman/queue/docs/rest.txt deleted file mode 100644 index 9e8851eca..000000000 --- a/src/mailman/queue/docs/rest.txt +++ /dev/null @@ -1,25 +0,0 @@ -=========== -REST server -=========== - -Mailman is controllable through an administrative `REST`_ HTTP server. - - >>> from mailman.testing import helpers - >>> master = helpers.TestableMaster(helpers.wait_for_webservice) - >>> master.start('rest') - -The RESTful server can be used to access basic version information. - - >>> dump_json('http://localhost:9001/3.0/system') - http_etag: "..." - mailman_version: GNU Mailman 3.0... (...) - python_version: ... - self_link: http://localhost:9001/3.0/system - - -Clean up -======== - - >>> master.stop() - -.. _REST: http://en.wikipedia.org/wiki/REST diff --git a/src/mailman/queue/docs/runner.txt b/src/mailman/queue/docs/runner.txt deleted file mode 100644 index 39e8fede2..000000000 --- a/src/mailman/queue/docs/runner.txt +++ /dev/null @@ -1,73 +0,0 @@ -============= -Queue runners -============= - -The queue runners (*qrunner*) are the processes that move messages around the -Mailman system. Each qrunner is responsible for a slice of the hash space in -a queue directory. It processes all the files in its slice, sleeps a little -while, then wakes up and runs through its queue files again. - - -Basic architecture -================== - -The basic architecture of qrunner is implemented in the base class that all -runners inherit from. This base class implements a ``.run()`` method that -runs continuously in a loop until the ``.stop()`` method is called. - - >>> mlist = create_list('_xtest@example.com') - -Here is a very simple derived qrunner class. Queue runners use a -configuration section in the configuration files to determine run -characteristics, such as the queue directory to use. Here we push a -configuration section for the test runner. -:: - - >>> config.push('test-runner', """ - ... [qrunner.test] - ... max_restarts: 1 - ... """) - - >>> from mailman.queue import Runner - >>> class TestableRunner(Runner): - ... def _dispose(self, mlist, msg, msgdata): - ... self.msg = msg - ... self.msgdata = msgdata - ... return False - ... - ... def _do_periodic(self): - ... self.stop() - ... - ... def _snooze(self, filecnt): - ... return - - >>> runner = TestableRunner('test') - -This qrunner doesn't do much except run once, storing the message and metadata -on instance variables. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: _xtest@example.com - ... - ... A test message. - ... """) - >>> switchboard = config.switchboards['test'] - >>> filebase = switchboard.enqueue(msg, listname=mlist.fqdn_listname, - ... foo='yes', bar='no') - >>> runner.run() - >>> print runner.msg.as_string() - From: aperson@example.com - To: _xtest@example.com - <BLANKLINE> - A test message. - <BLANKLINE> - >>> dump_msgdata(runner.msgdata) - _parsemsg: False - bar : no - foo : yes - lang : en - listname : _xtest@example.com - version : 3 - -XXX More of the Runner API should be tested. diff --git a/src/mailman/queue/docs/switchboard.txt b/src/mailman/queue/docs/switchboard.txt deleted file mode 100644 index d89aa3693..000000000 --- a/src/mailman/queue/docs/switchboard.txt +++ /dev/null @@ -1,187 +0,0 @@ -The switchboard -=============== - -The switchboard is subsystem that moves messages between queues. Each -instance of a switchboard is responsible for one queue directory. - - >>> msg = message_from_string("""\ - ... From: aperson@example.com - ... To: _xtest@example.com - ... - ... A test message. - ... """) - -Create a switchboard by giving its queue name and directory. - - >>> import os - >>> queue_directory = os.path.join(config.QUEUE_DIR, 'test') - >>> from mailman.queue import Switchboard - >>> switchboard = Switchboard('test', queue_directory) - >>> print switchboard.name - test - >>> switchboard.queue_directory == queue_directory - True - -Here's a helper function for ensuring things work correctly. - - >>> def check_qfiles(directory=None): - ... if directory is None: - ... directory = queue_directory - ... files = {} - ... for qfile in os.listdir(directory): - ... root, ext = os.path.splitext(qfile) - ... files[ext] = files.get(ext, 0) + 1 - ... if len(files) == 0: - ... print 'empty' - ... for ext in sorted(files): - ... print '{0}: {1}'.format(ext, files[ext]) - - -Enqueing and dequeing ---------------------- - -The message can be enqueued with metadata specified in the passed in -dictionary. - - >>> filebase = switchboard.enqueue(msg) - >>> check_qfiles() - .pck: 1 - -To read the contents of a queue file, dequeue it. - - >>> msg, msgdata = switchboard.dequeue(filebase) - >>> print msg.as_string() - From: aperson@example.com - To: _xtest@example.com - <BLANKLINE> - A test message. - <BLANKLINE> - >>> dump_msgdata(msgdata) - _parsemsg: False - version : 3 - >>> check_qfiles() - .bak: 1 - -To complete the dequeing process, removing all traces of the message file, -finish it (without preservation). - - >>> switchboard.finish(filebase) - >>> check_qfiles() - empty - -When enqueing a file, you can provide additional metadata keys by using -keyword arguments. - - >>> filebase = switchboard.enqueue(msg, {'foo': 1}, bar=2) - >>> msg, msgdata = switchboard.dequeue(filebase) - >>> switchboard.finish(filebase) - >>> dump_msgdata(msgdata) - _parsemsg: False - bar : 2 - foo : 1 - version : 3 - -Keyword arguments override keys from the metadata dictionary. - - >>> filebase = switchboard.enqueue(msg, {'foo': 1}, foo=2) - >>> msg, msgdata = switchboard.dequeue(filebase) - >>> switchboard.finish(filebase) - >>> dump_msgdata(msgdata) - _parsemsg: False - foo : 2 - version : 3 - - -Iterating over files --------------------- - -There are two ways to iterate over all the files in a switchboard's queue. -Normally, queue files end in .pck (for 'pickle') and the easiest way to -iterate over just these files is to use the .files attribute. - - >>> filebase_1 = switchboard.enqueue(msg, foo=1) - >>> filebase_2 = switchboard.enqueue(msg, foo=2) - >>> filebase_3 = switchboard.enqueue(msg, foo=3) - >>> filebases = sorted((filebase_1, filebase_2, filebase_3)) - >>> sorted(switchboard.files) == filebases - True - >>> check_qfiles() - .pck: 3 - -You can also use the .get_files() method if you want to iterate over all the -file bases for some other extension. - - >>> for filebase in switchboard.get_files(): - ... msg, msgdata = switchboard.dequeue(filebase) - >>> bakfiles = sorted(switchboard.get_files('.bak')) - >>> bakfiles == filebases - True - >>> check_qfiles() - .bak: 3 - >>> for filebase in switchboard.get_files('.bak'): - ... switchboard.finish(filebase) - >>> check_qfiles() - empty - - -Recovering files ----------------- - -Calling .dequeue() without calling .finish() leaves .bak backup files in -place. These can be recovered when the switchboard is instantiated. - - >>> filebase_1 = switchboard.enqueue(msg, foo=1) - >>> filebase_2 = switchboard.enqueue(msg, foo=2) - >>> filebase_3 = switchboard.enqueue(msg, foo=3) - >>> for filebase in switchboard.files: - ... msg, msgdata = switchboard.dequeue(filebase) - ... # Don't call .finish() - >>> check_qfiles() - .bak: 3 - >>> switchboard_2 = Switchboard('test', queue_directory, recover=True) - >>> check_qfiles() - .pck: 3 - -The files can be recovered explicitly. - - >>> for filebase in switchboard.files: - ... msg, msgdata = switchboard.dequeue(filebase) - ... # Don't call .finish() - >>> check_qfiles() - .bak: 3 - >>> switchboard.recover_backup_files() - >>> check_qfiles() - .pck: 3 - -But the files will only be recovered at most three times before they are -considered defective. In order to prevent mail bombs and loops, once this -maximum is reached, the files will be preserved in the 'bad' queue. -:: - - >>> for filebase in switchboard.files: - ... msg, msgdata = switchboard.dequeue(filebase) - ... # Don't call .finish() - >>> check_qfiles() - .bak: 3 - >>> switchboard.recover_backup_files() - >>> check_qfiles() - empty - - >>> bad = config.switchboards['bad'] - >>> check_qfiles(bad.queue_directory) - .psv: 3 - - -Clean up --------- - - >>> for file in os.listdir(bad.queue_directory): - ... os.remove(os.path.join(bad.queue_directory, file)) - >>> check_qfiles(bad.queue_directory) - empty - - -Queue slices ------------- - -XXX Add tests for queue slices. diff --git a/src/mailman/queue/incoming.py b/src/mailman/queue/incoming.py deleted file mode 100644 index f8d671177..000000000 --- a/src/mailman/queue/incoming.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Incoming queue runner. - -This runner's sole purpose in life is to decide the disposition of the -message. It can either be accepted for delivery, rejected (i.e. bounced), -held for moderator approval, or discarded. - -When accepted, the message is forwarded on to the `prep queue` where it is -prepared for delivery. Rejections, discards, and holds are processed -immediately. -""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'IncomingRunner', - ] - - -from zope.component import getUtility - -from mailman.config import config -from mailman.core.chains import process -from mailman.interfaces.address import ExistingAddressError -from mailman.interfaces.usermanager import IUserManager -from mailman.queue import Runner - - - -class IncomingRunner(Runner): - """The incoming queue runner.""" - - def _dispose(self, mlist, msg, msgdata): - """See `IRunner`.""" - if msgdata.get('envsender') is None: - msgdata['envsender'] = mlist.no_reply_address - # Ensure that the email addresses of the message's senders are known - # to Mailman. This will be used in nonmember posting dispositions. - user_manager = getUtility(IUserManager) - for sender in msg.senders: - try: - user_manager.create_address(sender) - except ExistingAddressError: - pass - config.db.commit() - # Process the message through the mailing list's start chain. - process(mlist, msg, msgdata, mlist.start_chain) - # Do not keep this message queued. - return False diff --git a/src/mailman/queue/lmtp.py b/src/mailman/queue/lmtp.py deleted file mode 100644 index 9163a88e6..000000000 --- a/src/mailman/queue/lmtp.py +++ /dev/null @@ -1,236 +0,0 @@ -# Copyright (C) 2006-2011 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Mailman LMTP runner (server). - -Most mail servers can be configured to deliver local messages via 'LMTP'[1]. -This module is actually an LMTP server rather than a standard queue runner. - -The LMTP runner opens a local TCP port and waits for the mail server to -connect to it. The messages it receives over LMTP are very minimally parsed -for sanity and if they look okay, they are accepted and injected into -Mailman's incoming queue for normal processing. If they don't look good, or -are destined for a bogus sub-address, they are rejected right away, hopefully -so that the peer mail server can provide better diagnostics. - -[1] RFC 2033 Local Mail Transport Protocol - http://www.faqs.org/rfcs/rfc2033.html -""" - -import email -import smtpd -import logging -import asyncore - -from email.utils import parseaddr -from zope.component import getUtility - -from mailman.config import config -from mailman.database.transaction import txn -from mailman.email.message import Message -from mailman.interfaces.listmanager import IListManager -from mailman.queue import Runner - -elog = logging.getLogger('mailman.error') -qlog = logging.getLogger('mailman.qrunner') - - -# We only care about the listname and the sub-addresses as in listname@ or -# listname-request@. This maps user visible subaddress names (which may -# include aliases) to the internal canonical subaddress name. -SUBADDRESS_NAMES = dict( - admin='bounces', - bounces='bounces', - confirm='confirm', - join='join', - leave='leave', - owner='owner', - request='request', - subscribe='join', - unsubscribe='leave', - ) - -# This maps subaddress canonical name to the destination queue that handles -# messages sent to that subaddress. -SUBADDRESS_QUEUES = dict( - bounces='bounces', - confirm='command', - join='command', - leave='command', - owner='in', - request='command', - ) - -DASH = '-' -CRLF = '\r\n' -ERR_451 = '451 Requested action aborted: error in processing' -ERR_501 = '501 Message has defects' -ERR_502 = '502 Error: command HELO not implemented' -ERR_550 = '550 Requested action not taken: mailbox unavailable' - -# XXX Blech -smtpd.__version__ = 'Python LMTP queue runner 1.0' - - - -def split_recipient(address): - """Split an address into listname, subaddress and domain parts. - - For example: - - >>> split_recipient('mylist@example.com') - ('mylist', None, 'example.com') - - >>> split_recipient('mylist-request@example.com') - ('mylist', 'request', 'example.com') - - :param address: The destination address. - :return: A 3-tuple of the form (list-shortname, subaddress, domain). - subaddress may be None if this is the list's posting address. - """ - localpart, domain = address.split('@', 1) - localpart = localpart.split(config.mta.verp_delimiter, 1)[0] - parts = localpart.split(DASH) - if parts[-1] in SUBADDRESS_NAMES: - listname = DASH.join(parts[:-1]) - subaddress = parts[-1] - else: - listname = localpart - subaddress = None - return listname, subaddress, domain - - - -class Channel(smtpd.SMTPChannel): - """An LMTP channel.""" - - def __init__(self, server, conn, addr): - smtpd.SMTPChannel.__init__(self, server, conn, addr) - # Stash this here since the subclass uses private attributes. :( - self._server = server - - def smtp_LHLO(self, arg): - """The LMTP greeting, used instead of HELO/EHLO.""" - smtpd.SMTPChannel.smtp_HELO(self, arg) - - def smtp_HELO(self, arg): - """HELO is not a valid LMTP command.""" - self.push(ERR_502) - - - -class LMTPRunner(Runner, smtpd.SMTPServer): - # Only __init__ is called on startup. Asyncore is responsible for later - # connections from the MTA. slice and numslices are ignored and are - # necessary only to satisfy the API. - def __init__(self, slice=None, numslices=1): - localaddr = config.mta.lmtp_host, int(config.mta.lmtp_port) - # Do not call Runner's constructor because there's no QDIR to create - qlog.debug('LMTP server listening on %s:%s', - localaddr[0], localaddr[1]) - smtpd.SMTPServer.__init__(self, localaddr, remoteaddr=None) - - def handle_accept(self): - conn, addr = self.accept() - Channel(self, conn, addr) - qlog.debug('LMTP accept from %s', addr) - - @txn - def process_message(self, peer, mailfrom, rcpttos, data): - try: - # Refresh the list of list names every time we process a message - # since the set of mailing lists could have changed. - listnames = set(getUtility(IListManager).names) - # Parse the message data. If there are any defects in the - # message, reject it right away; it's probably spam. - msg = email.message_from_string(data, Message) - msg.original_size = len(data) - if msg.defects: - return ERR_501 - msg['X-MailFrom'] = mailfrom - message_id = msg['message-id'] - except Exception: - elog.exception('LMTP message parsing') - config.db.abort() - return CRLF.join(ERR_451 for to in rcpttos) - # RFC 2033 requires us to return a status code for every recipient. - status = [] - # Now for each address in the recipients, parse the address to first - # see if it's destined for a valid mailing list. If so, then queue - # the message to the appropriate place and record a 250 status for - # that recipient. If not, record a failure status for that recipient. - for to in rcpttos: - try: - to = parseaddr(to)[1].lower() - listname, subaddress, domain = split_recipient(to) - qlog.debug('%s to: %s, list: %s, sub: %s, dom: %s', - message_id, to, listname, subaddress, domain) - listname += '@' + domain - if listname not in listnames: - status.append(ERR_550) - continue - # The recipient is a valid mailing list. Find the subaddress - # if there is one, and set things up to enqueue to the proper - # queue runner. - queue = None - msgdata = dict(listname=listname, - original_size=msg.original_size) - canonical_subaddress = SUBADDRESS_NAMES.get(subaddress) - queue = SUBADDRESS_QUEUES.get(canonical_subaddress) - if subaddress is None: - # The message is destined for the mailing list. - msgdata['to_list'] = True - queue = 'in' - elif canonical_subaddress is None: - # The subaddress was bogus. - elog.error('%s unknown sub-address: %s', - message_id, subaddress) - status.append(ERR_550) - continue - else: - # A valid subaddress. - msgdata['subaddress'] = canonical_subaddress - if canonical_subaddress == 'owner': - msgdata.update(dict( - to_owner=True, - envsender=config.mailman.site_owner, - )) - queue = 'in' - # If we found a valid destination, enqueue the message and add - # a success status for this recipient. - if queue is not None: - config.switchboards[queue].enqueue(msg, msgdata) - qlog.debug('%s subaddress: %s, queue: %s', - message_id, canonical_subaddress, queue) - status.append('250 Ok') - except Exception: - elog.exception('Queue detection: %s', msg['message-id']) - config.db.abort() - status.append(ERR_550) - # All done; returning this big status string should give the expected - # response to the LMTP client. - return CRLF.join(status) - - def run(self): - """See `IRunner`.""" - asyncore.loop() - - def stop(self): - """See `IRunner`.""" - asyncore.socket_map.clear() - asyncore.close_all() - self.close() diff --git a/src/mailman/queue/maildir.py b/src/mailman/queue/maildir.py deleted file mode 100644 index 07a89903c..000000000 --- a/src/mailman/queue/maildir.py +++ /dev/null @@ -1,193 +0,0 @@ -# Copyright (C) 2002-2011 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Maildir pre-queue runner. - -Most MTAs can be configured to deliver messages to a `Maildir'[1]. This -runner will read messages from a maildir's new/ directory and inject them into -Mailman's qfiles/in directory for processing in the normal pipeline. This -delivery mechanism contrasts with mail program delivery, where incoming -messages end up in qfiles/in via the MTA executing the scripts/post script -(and likewise for the other -aliases for each mailing list). - -The advantage to Maildir delivery is that it is more efficient; there's no -need to fork an intervening program just to take the message from the MTA's -standard output, to the qfiles/in directory. - -[1] http://cr.yp.to/proto/maildir.html - -We're going to use the :info flag == 1, experimental status flag for our own -purposes. The :1 can be followed by one of these letters: - -- P means that MaildirRunner's in the process of parsing and enqueuing the - message. If successful, it will delete the file. - -- X means something failed during the parse/enqueue phase. An error message - will be logged to log/error and the file will be renamed <filename>:1,X. - MaildirRunner will never automatically return to this file, but once the - problem is fixed, you can manually move the file back to the new/ directory - and MaildirRunner will attempt to re-process it. At some point we may do - this automatically. - -See the variable USE_MAILDIR in Defaults.py.in for enabling this delivery -mechanism. -""" - -# NOTE: Maildir delivery is experimental in Mailman 2.1. - -import os -import errno -import logging - -from email.parser import Parser -from email.utils import parseaddr - -from mailman.config import config -from mailman.message import Message -from mailman.queue import Runner, Switchboard - - -log = logging.getLogger('mailman.error') - -# We only care about the listname and the subq as in listname@ or -# listname-request@ -subqnames = ('admin', 'bounces', 'confirm', 'join', 'leave', - 'owner', 'request', 'subscribe', 'unsubscribe') - - -def getlistq(address): - localpart, domain = address.split('@', 1) - # TK: FIXME I only know configs of Postfix. - if config.POSTFIX_STYLE_VIRTUAL_DOMAINS: - p = localpart.split(config.POSTFIX_VIRTUAL_SEPARATOR, 1) - if len(p) == 2: - localpart, domain = p - l = localpart.split('-') - if l[-1] in subqnames: - listname = '-'.join(l[:-1]) - subq = l[-1] - else: - listname = localpart - subq = None - return listname, subq, domain - - - -class MaildirRunner(Runner): - # This class is much different than most runners because it pulls files - # of a different format than what scripts/post and friends leaves. The - # files this runner reads are just single message files as dropped into - # the directory by the MTA. This runner will read the file, and enqueue - # it in the expected qfiles directory for normal processing. - def __init__(self, slice=None, numslices=1): - # Don't call the base class constructor, but build enough of the - # underlying attributes to use the base class's implementation. - self._stop = 0 - self._dir = os.path.join(config.MAILDIR_DIR, 'new') - self._cur = os.path.join(config.MAILDIR_DIR, 'cur') - self._parser = Parser(Message) - - def _one_iteration(self): - # Refresh this each time through the list. - listnames = list(config.list_manager.names) - # Cruise through all the files currently in the new/ directory - try: - files = os.listdir(self._dir) - except OSError, e: - if e.errno <> errno.ENOENT: - raise - # Nothing's been delivered yet - return 0 - for file in files: - srcname = os.path.join(self._dir, file) - dstname = os.path.join(self._cur, file + ':1,P') - xdstname = os.path.join(self._cur, file + ':1,X') - try: - os.rename(srcname, dstname) - except OSError, e: - if e.errno == errno.ENOENT: - # Some other MaildirRunner beat us to it - continue - log.error('Could not rename maildir file: %s', srcname) - raise - # Now open, read, parse, and enqueue this message - try: - fp = open(dstname) - try: - msg = self._parser.parse(fp) - finally: - fp.close() - # Now we need to figure out which queue of which list this - # message was destined for. See get_verp() in - # mailman.app.bounces for why we do things this way. - vals = [] - for header in ('delivered-to', 'envelope-to', 'apparently-to'): - vals.extend(msg.get_all(header, [])) - for field in vals: - to = parseaddr(field)[1].lower() - if not to: - continue - listname, subq, domain = getlistq(to) - listname = listname + '@' + domain - if listname in listnames: - break - else: - # As far as we can tell, this message isn't destined for - # any list on the system. What to do? - log.error('Message apparently not for any list: %s', - xdstname) - os.rename(dstname, xdstname) - continue - # BAW: blech, hardcoded - msgdata = {'listname': listname} - # -admin is deprecated - if subq in ('bounces', 'admin'): - queue = Switchboard('bounces', config.BOUNCEQUEUE_DIR) - elif subq == 'confirm': - msgdata['toconfirm'] = 1 - queue = Switchboard('command', config.CMDQUEUE_DIR) - elif subq in ('join', 'subscribe'): - msgdata['tojoin'] = 1 - queue = Switchboard('command', config.CMDQUEUE_DIR) - elif subq in ('leave', 'unsubscribe'): - msgdata['toleave'] = 1 - queue = Switchboard('command', config.CMDQUEUE_DIR) - elif subq == 'owner': - msgdata.update({ - 'toowner': True, - 'envsender': config.SITE_OWNER_ADDRESS, - 'pipeline': config.OWNER_PIPELINE, - }) - queue = Switchboard('in', config.INQUEUE_DIR) - elif subq is None: - msgdata['tolist'] = 1 - queue = Switchboard('in', config.INQUEUE_DIR) - elif subq == 'request': - msgdata['torequest'] = 1 - queue = Switchboard('command', config.CMDQUEUE_DIR) - else: - log.error('Unknown sub-queue: %s', subq) - os.rename(dstname, xdstname) - continue - queue.enqueue(msg, msgdata) - os.unlink(dstname) - except Exception, e: - os.rename(dstname, xdstname) - log.error('%s', e) - - def _clean_up(self): - pass diff --git a/src/mailman/queue/news.py b/src/mailman/queue/news.py deleted file mode 100644 index a3d915244..000000000 --- a/src/mailman/queue/news.py +++ /dev/null @@ -1,167 +0,0 @@ -# Copyright (C) 2000-2011 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""NNTP queue runner.""" - -import re -import email -import socket -import logging -import nntplib - -from cStringIO import StringIO -from lazr.config import as_host_port - -from mailman.config import config -from mailman.interfaces.nntp import NewsModeration -from mailman.queue import Runner - -COMMASPACE = ', ' -log = logging.getLogger('mailman.error') - - -# Matches our Mailman crafted Message-IDs. See Utils.unique_message_id() -# XXX The move to email.utils.make_msgid() breaks this. -mcre = re.compile(r""" - <mailman. # match the prefix - \d+. # serial number - \d+. # time in seconds since epoch - \d+. # pid - (?P<listname>[^@]+) # list's internal_name() - @ # localpart@dom.ain - (?P<hostname>[^>]+) # list's host_name - > # trailer - """, re.VERBOSE) - - - -class NewsRunner(Runner): - def _dispose(self, mlist, msg, msgdata): - # Make sure we have the most up-to-date state - mlist.Load() - if not msgdata.get('prepped'): - prepare_message(mlist, msg, msgdata) - try: - # Flatten the message object, sticking it in a StringIO object - fp = StringIO(msg.as_string()) - conn = None - try: - try: - nntp_host, nntp_port = as_host_port( - mlist.nntp_host, default_port=119) - conn = nntplib.NNTP(nntp_host, nntp_port, - readermode=True, - user=config.nntp.username, - password=config.nntp.password) - conn.post(fp) - except nntplib.error_temp, e: - log.error('(NNTPDirect) NNTP error for list "%s": %s', - mlist.internal_name(), e) - except socket.error, e: - log.error('(NNTPDirect) socket error for list "%s": %s', - mlist.internal_name(), e) - finally: - if conn: - conn.quit() - except Exception, e: - # Some other exception occurred, which we definitely did not - # expect, so set this message up for requeuing. - self._log(e) - return True - return False - - - -def prepare_message(mlist, msg, msgdata): - # If the newsgroup is moderated, we need to add this header for the Usenet - # software to accept the posting, and not forward it on to the n.g.'s - # moderation address. The posting would not have gotten here if it hadn't - # already been approved. 1 == open list, mod n.g., 2 == moderated - if mlist.news_moderation in (NewsModeration.open_moderated, - NewsModeration.moderated): - del msg['approved'] - msg['Approved'] = mlist.posting_address - # Should we restore the original, non-prefixed subject for gatewayed - # messages? TK: We use stripped_subject (prefix stripped) which was - # crafted in CookHeaders.py to ensure prefix was stripped from the subject - # came from mailing list user. - stripped_subject = msgdata.get('stripped_subject') \ - or msgdata.get('origsubj') - if not mlist.news_prefix_subject_too and stripped_subject is not None: - del msg['subject'] - msg['subject'] = stripped_subject - # Add the appropriate Newsgroups: header - ngheader = msg['newsgroups'] - if ngheader is not None: - # See if the Newsgroups: header already contains our linked_newsgroup. - # If so, don't add it again. If not, append our linked_newsgroup to - # the end of the header list - ngroups = [s.strip() for s in ngheader.split(',')] - if mlist.linked_newsgroup not in ngroups: - ngroups.append(mlist.linked_newsgroup) - # Subtitute our new header for the old one. - del msg['newsgroups'] - msg['Newsgroups'] = COMMASPACE.join(ngroups) - else: - # Newsgroups: isn't in the message - msg['Newsgroups'] = mlist.linked_newsgroup - # Note: We need to be sure two messages aren't ever sent to the same list - # in the same process, since message ids need to be unique. Further, if - # messages are crossposted to two Usenet-gated mailing lists, they each - # need to have unique message ids or the nntpd will only accept one of - # them. The solution here is to substitute any existing message-id that - # isn't ours with one of ours, so we need to parse it to be sure we're not - # looping. - # - # Our Message-ID format is <mailman.secs.pid.listname@hostname> - msgid = msg['message-id'] - hackmsgid = True - if msgid: - mo = mcre.search(msgid) - if mo: - lname, hname = mo.group('listname', 'hostname') - if lname == mlist.internal_name() and hname == mlist.host_name: - hackmsgid = False - if hackmsgid: - del msg['message-id'] - msg['Message-ID'] = email.utils.make_msgid() - # Lines: is useful - if msg['Lines'] is None: - # BAW: is there a better way? - count = len(list(email.Iterators.body_line_iterator(msg))) - msg['Lines'] = str(count) - # Massage the message headers by remove some and rewriting others. This - # woon't completely sanitize the message, but it will eliminate the bulk - # of the rejections based on message headers. The NNTP server may still - # reject the message because of other problems. - for header in config.nntp.remove_headers.split(): - del msg[header] - for rewrite_pairs in config.nntp.rewrite_duplicate_headers.splitlines(): - if len(rewrite_pairs.strip()) == 0: - continue - header, rewrite = rewrite_pairs.split() - values = msg.get_all(header, []) - if len(values) < 2: - # We only care about duplicates - continue - del msg[header] - # But keep the first one... - msg[header] = values[0] - for v in values[1:]: - msg[rewrite] = v - # Mark this message as prepared in case it has to be requeued - msgdata['prepped'] = True diff --git a/src/mailman/queue/outgoing.py b/src/mailman/queue/outgoing.py deleted file mode 100644 index ed27f014c..000000000 --- a/src/mailman/queue/outgoing.py +++ /dev/null @@ -1,160 +0,0 @@ -# Copyright (C) 2000-2011 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Outgoing queue runner.""" - -import socket -import logging - -from datetime import datetime -from lazr.config import as_boolean, as_timedelta -from zope.component import getUtility - -from mailman.config import config -from mailman.interfaces.bounce import BounceContext, IBounceProcessor -from mailman.interfaces.mailinglist import Personalization -from mailman.interfaces.membership import ISubscriptionService -from mailman.interfaces.mta import SomeRecipientsFailed -from mailman.interfaces.pending import IPendings -from mailman.queue import Runner -from mailman.utilities.datetime import now -from mailman.utilities.modules import find_name - - -# This controls how often _do_periodic() will try to deal with deferred -# permanent failures. It is a count of calls to _do_periodic() -DEAL_WITH_PERMFAILURES_EVERY = 10 - -log = logging.getLogger('mailman.error') -smtp_log = logging.getLogger('mailman.smtp') - - - -class OutgoingRunner(Runner): - """The outgoing queue runner.""" - - def __init__(self, slice=None, numslices=1): - super(OutgoingRunner, self).__init__(slice, numslices) - # We look this function up only at startup time. - self._func = find_name(config.mta.outgoing) - # This prevents smtp server connection problems from filling up the - # error log. It gets reset if the message was successfully sent, and - # set if there was a socket.error. - self._logged = False - self._retryq = config.switchboards['retry'] - - def _dispose(self, mlist, msg, msgdata): - # See if we should retry delivery of this message again. - deliver_after = msgdata.get('deliver_after', datetime.fromtimestamp(0)) - if now() < deliver_after: - return True - # Calculate whether we should VERP this message or not. The results of - # this set the 'verp' key in the message metadata. - interval = int(config.mta.verp_delivery_interval) - if 'verp' in msgdata: - # Honor existing settings. - pass - # If personalization is enabled for this list and we've configured - # Mailman to always VERP personalized deliveries, then yes we VERP it. - # Also, if personalization is /not/ enabled, but - # verp_delivery_interval is set (and we've hit this interval), then - # again, this message should be VERP'd. Otherwise, no. - elif mlist.personalize != Personalization.none: - if as_boolean(config.mta.verp_personalized_deliveries): - msgdata['verp'] = True - elif interval == 0: - # Never VERP. - msgdata['verp'] = False - elif interval == 1: - # VERP every time. - msgdata['verp'] = True - else: - # VERP every 'interval' number of times. - msgdata['verp'] = (mlist.post_id % interval == 0) - try: - self._func(mlist, msg, msgdata) - self._logged = False - except socket.error: - # There was a problem connecting to the SMTP server. Log this - # once, but crank up our sleep time so we don't fill the error - # log. - port = int(config.mta.smtp_port) - if port == 0: - port = 'smtp' # Log this just once. - if not self._logged: - log.error('Cannot connect to SMTP server %s on port %s', - config.mta.smtp_host, port) - self._logged = True - return True - except SomeRecipientsFailed as error: - processor = getUtility(IBounceProcessor) - # BAW: msg is the original message that failed delivery, not a - # bounce message. This may be confusing if this is what's sent to - # the user in the probe message. Maybe we should craft a - # bounce-like message containing information about the permanent - # SMTP failure? - if 'probe_token' in msgdata: - # This is a failure of our local MTA to deliver to a probe - # message recipient. Register the bounce event for permanent - # failures. Start by grabbing and confirming (i.e. removing) - # the pendable record associated with this bounce token, - # regardless of what address was actually failing. - if len(error.permanent_failures) > 0: - pended = getUtility(IPendings).confirm( - msgdata['probe_token']) - # It's possible the token has been confirmed out of the - # database. Just ignore that. - if pended is not None: - member = getUtility(ISubscriptionService).get_member( - pended['member_id']) - processor.register( - mlist, member.address.email, msg, - BounceContext.probe) - else: - # Delivery failed at SMTP time for some or all of the - # recipients. Permanent failures are registered as bounces, - # but temporary failures are retried for later. - for email in error.permanent_failures: - processor.register(mlist, email, msg, BounceContext.normal) - # Move temporary failures to the qfiles/retry queue which will - # occasionally move them back here for another shot at - # delivery. - if error.temporary_failures: - current_time = now() - recipients = error.temporary_failures - last_recip_count = msgdata.get('last_recip_count', 0) - deliver_until = msgdata.get('deliver_until', current_time) - if len(recipients) == last_recip_count: - # We didn't make any progress. If we've exceeded the - # configured retry period, log this failure and - # discard the message. - if current_time > deliver_until: - smtp_log.error('Discarding message with ' - 'persistent temporary failures: ' - '{0}'.format(msg['message-id'])) - return False - else: - # We made some progress, so keep trying to delivery - # this message for a while longer. - deliver_until = current_time + as_timedelta( - config.mta.delivery_retry_period) - msgdata['last_recip_count'] = len(recipients) - msgdata['deliver_until'] = deliver_until - msgdata['recipients'] = recipients - self._retryq.enqueue(msg, msgdata) - # We've successfully completed handling of this message. - return False diff --git a/src/mailman/queue/pipeline.py b/src/mailman/queue/pipeline.py deleted file mode 100644 index 099ebd032..000000000 --- a/src/mailman/queue/pipeline.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (C) 2008-2011 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""The pipeline queue runner. - -This runner's purpose is to take messages that have been approved for posting -through the 'preparation pipeline'. This pipeline adds, deletes and modifies -headers, calculates message recipients, and more. -""" - -from mailman.core.pipelines import process -from mailman.queue import Runner - - - -class PipelineRunner(Runner): - def _dispose(self, mlist, msg, msgdata): - # Process the message through the mailing list's pipeline. - process(mlist, msg, msgdata, mlist.pipeline) - # Do not keep this message queued. - return False diff --git a/src/mailman/queue/rest.py b/src/mailman/queue/rest.py deleted file mode 100644 index 31e840a51..000000000 --- a/src/mailman/queue/rest.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (C) 2009-2011 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Start the administrative HTTP server.""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'RESTRunner', - ] - - -import sys -import errno -import select -import signal -import logging - -from mailman.queue import Runner -from mailman.rest.wsgiapp import make_server - - -log = logging.getLogger('mailman.http') - - - -class RESTRunner(Runner): - intercept_signals = False - - def run(self): - log.info('Starting REST server') - try: - make_server().serve_forever() - except KeyboardInterrupt: - log.info('REST server interrupted') - sys.exit(signal.SIGTERM) - except select.error as (errcode, message): - if errcode == errno.EINTR: - log.info('REST server exiting') - sys.exit(signal.SIGTERM) - raise - except: - raise diff --git a/src/mailman/queue/retry.py b/src/mailman/queue/retry.py deleted file mode 100644 index 24aa7f82b..000000000 --- a/src/mailman/queue/retry.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (C) 2003-2011 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Retry delivery.""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'RetryRunner', - ] - - -import time - -from mailman.config import config -from mailman.queue import Runner - - - -class RetryRunner(Runner): - """Retry delivery.""" - - def _dispose(self, mlist, msg, msgdata): - # Move the message to the out queue for another try. - config.switchboards['outgoing'].enqueue(msg, msgdata) - return False - - def _snooze(self, filecnt): - # We always want to snooze. - time.sleep(self.sleep_float) diff --git a/src/mailman/queue/tests/__init__.py b/src/mailman/queue/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/src/mailman/queue/tests/__init__.py +++ /dev/null diff --git a/src/mailman/queue/tests/test_bounce.py b/src/mailman/queue/tests/test_bounce.py deleted file mode 100644 index 1946df50c..000000000 --- a/src/mailman/queue/tests/test_bounce.py +++ /dev/null @@ -1,236 +0,0 @@ -# Copyright (C) 2011 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Test the bounce queue runner.""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'test_suite', - ] - - -import unittest - -from zope.component import getUtility - -from mailman.app.bounces import send_probe -from mailman.app.lifecycle import create_list -from mailman.config import config -from mailman.interfaces.bounce import ( - BounceContext, IBounceProcessor, UnrecognizedBounceDisposition) -from mailman.interfaces.member import MemberRole -from mailman.interfaces.usermanager import IUserManager -from mailman.queue.bounce import BounceRunner -from mailman.testing.helpers import ( - LogFileMark, - get_queue_messages, - make_testable_runner, - specialized_message_from_string as message_from_string) -from mailman.testing.layers import ConfigLayer - - - -class TestBounceQueue(unittest.TestCase): - """Test the bounce queue runner.""" - - layer = ConfigLayer - - def setUp(self): - self._mlist = create_list('test@example.com') - self._bounceq = config.switchboards['bounces'] - self._runner = make_testable_runner(BounceRunner, 'bounces') - self._anne = getUtility(IUserManager).create_address( - 'anne@example.com') - self._member = self._mlist.subscribe(self._anne, MemberRole.member) - self._msg = message_from_string("""\ -From: mail-daemon@example.com -To: test-bounces+anne=example.com@example.com -Message-Id: <first> - -""") - self._msgdata = dict(listname='test@example.com') - self._processor = getUtility(IBounceProcessor) - config.push('site owner', """ - [mailman] - site_owner: postmaster@example.com - """) - - def tearDown(self): - config.pop('site owner') - - def test_does_no_processing(self): - # If the mailing list does no bounce processing, the messages are - # simply discarded. - self._mlist.bounce_processing = False - self._bounceq.enqueue(self._msg, self._msgdata) - self._runner.run() - self.assertEqual(len(get_queue_messages('bounces')), 0) - self.assertEqual(len(list(self._processor.events)), 0) - - def test_verp_detection(self): - # When we get a VERPd bounce, and we're doing processing, a bounce - # event will be registered. - self._bounceq.enqueue(self._msg, self._msgdata) - self._runner.run() - self.assertEqual(len(get_queue_messages('bounces')), 0) - events = list(self._processor.events) - self.assertEqual(len(events), 1) - self.assertEqual(events[0].email, 'anne@example.com') - self.assertEqual(events[0].list_name, 'test@example.com') - self.assertEqual(events[0].message_id, '<first>') - self.assertEqual(events[0].context, BounceContext.normal) - self.assertEqual(events[0].processed, False) - - def test_nonfatal_verp_detection(self): - # A VERPd bounce was received, but the error was nonfatal. - nonfatal = message_from_string("""\ -From: mail-daemon@example.com -To: test-bounces+anne=example.com@example.com -Message-Id: <first> -Content-Type: multipart/report; report-type=delivery-status; boundary=AAA -MIME-Version: 1.0 - ---AAA -Content-Type: message/delivery-status - -Action: delayed -Original-Recipient: rfc822; somebody@example.com - ---AAA-- -""") - self._bounceq.enqueue(nonfatal, self._msgdata) - self._runner.run() - self.assertEqual(len(get_queue_messages('bounces')), 0) - events = list(self._processor.events) - self.assertEqual(len(events), 0) - - def test_verp_probe_bounce(self): - # A VERP probe bounced. The primary difference here is that the - # registered bounce event will have a different context. The - # Message-Id will be different too, because of the way we're - # simulating the probe bounce. - # - # Start be simulating a probe bounce. - send_probe(self._member, self._msg) - message = get_queue_messages('virgin')[0].msg - bounce = message_from_string("""\ -To: {0} -From: mail-daemon@example.com -Message-Id: <second> - -""".format(message['From'])) - self._bounceq.enqueue(bounce, self._msgdata) - self._runner.run() - self.assertEqual(len(get_queue_messages('bounces')), 0) - events = list(self._processor.events) - self.assertEqual(len(events), 1) - self.assertEqual(events[0].email, 'anne@example.com') - self.assertEqual(events[0].list_name, 'test@example.com') - self.assertEqual(events[0].message_id, '<second>') - self.assertEqual(events[0].context, BounceContext.probe) - self.assertEqual(events[0].processed, False) - - def test_nonverp_detectable_fatal_bounce(self): - # Here's a bounce that is not VERPd, but which has a bouncing address - # that can be parsed from a known bounce format. DSN is as good as - # any, but we'll make the parsed address different for the fun of it. - dsn = message_from_string("""\ -From: mail-daemon@example.com -To: test-bounces@example.com -Message-Id: <first> -Content-Type: multipart/report; report-type=delivery-status; boundary=AAA -MIME-Version: 1.0 - ---AAA -Content-Type: message/delivery-status - -Action: fail -Original-Recipient: rfc822; bart@example.com - ---AAA-- -""") - self._bounceq.enqueue(dsn, self._msgdata) - self._runner.run() - self.assertEqual(len(get_queue_messages('bounces')), 0) - events = list(self._processor.events) - self.assertEqual(len(events), 1) - self.assertEqual(events[0].email, 'bart@example.com') - self.assertEqual(events[0].list_name, 'test@example.com') - self.assertEqual(events[0].message_id, '<first>') - self.assertEqual(events[0].context, BounceContext.normal) - self.assertEqual(events[0].processed, False) - - def test_nonverp_detectable_nonfatal_bounce(self): - # Here's a bounce that is not VERPd, but which has a bouncing address - # that can be parsed from a known bounce format. The bounce is - # non-fatal so no bounce event is registered. - dsn = message_from_string("""\ -From: mail-daemon@example.com -To: test-bounces@example.com -Message-Id: <first> -Content-Type: multipart/report; report-type=delivery-status; boundary=AAA -MIME-Version: 1.0 - ---AAA -Content-Type: message/delivery-status - -Action: delayed -Original-Recipient: rfc822; bart@example.com - ---AAA-- -""") - self._bounceq.enqueue(dsn, self._msgdata) - self._runner.run() - self.assertEqual(len(get_queue_messages('bounces')), 0) - events = list(self._processor.events) - self.assertEqual(len(events), 0) - - def test_no_detectable_bounce_addresses(self): - # A bounce message was received, but no addresses could be detected. - # A message will be logged in the bounce log though, and the message - # can be forwarded to someone who can do something about it. - self._mlist.forward_unrecognized_bounces_to = ( - UnrecognizedBounceDisposition.site_owner) - bogus = message_from_string("""\ -From: mail-daemon@example.com -To: test-bounces@example.com -Message-Id: <third> - -""") - self._bounceq.enqueue(bogus, self._msgdata) - mark = LogFileMark('mailman.bounce') - self._runner.run() - self.assertEqual(len(get_queue_messages('bounces')), 0) - events = list(self._processor.events) - self.assertEqual(len(events), 0) - line = mark.readline() - self.assertEqual( - line[-51:-1], - 'Bounce message w/no discernable addresses: <third>') - # Here's the forwarded message to the site owners. - forwards = get_queue_messages('virgin') - self.assertEqual(len(forwards), 1) - self.assertEqual(forwards[0].msg['to'], 'postmaster@example.com') - - - -def test_suite(): - suite = unittest.TestSuite() - suite.addTest(unittest.makeSuite(TestBounceQueue)) - return suite diff --git a/src/mailman/queue/tests/test_outgoing.py b/src/mailman/queue/tests/test_outgoing.py deleted file mode 100644 index a0fe407c8..000000000 --- a/src/mailman/queue/tests/test_outgoing.py +++ /dev/null @@ -1,549 +0,0 @@ -# Copyright (C) 2011 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Test the outgoing queue runner.""" - -from __future__ import absolute_import, unicode_literals - -__metaclass__ = type -__all__ = [ - 'test_suite', - ] - - -import os -import socket -import logging -import unittest - -from contextlib import contextmanager -from datetime import datetime, timedelta -from lazr.config import as_timedelta -from zope.component import getUtility - -from mailman.app.bounces import send_probe -from mailman.app.lifecycle import create_list -from mailman.config import config -from mailman.interfaces.bounce import BounceContext, IBounceProcessor -from mailman.interfaces.mailinglist import Personalization -from mailman.interfaces.member import MemberRole -from mailman.interfaces.mta import SomeRecipientsFailed -from mailman.interfaces.pending import IPendings -from mailman.interfaces.usermanager import IUserManager -from mailman.queue.outgoing import OutgoingRunner -from mailman.testing.helpers import ( - LogFileMark, - get_queue_messages, - make_testable_runner, - specialized_message_from_string as message_from_string) -from mailman.testing.layers import ConfigLayer, SMTPLayer -from mailman.utilities.datetime import factory, now - - - -def run_once(qrunner): - """Predicate for make_testable_runner(). - - Ensures that the queue runner only runs once. - """ - return True - - -@contextmanager -def temporary_config(name, settings): - """Temporarily set a configuration (use in a with-statement).""" - config.push(name, settings) - try: - yield - finally: - config.pop(name) - - - -class TestOnce(unittest.TestCase): - """Test outgoing runner message disposition.""" - - layer = SMTPLayer - - def setUp(self): - self._mlist = create_list('test@example.com') - self._outq = config.switchboards['out'] - self._runner = make_testable_runner(OutgoingRunner, 'out', run_once) - self._msg = message_from_string("""\ -From: anne@example.com -To: test@example.com -Message-Id: <first> - -""") - self._msgdata = {} - - def test_deliver_after(self): - # When the metadata has a deliver_after key in the future, the queue - # runner will re-enqueue the message rather than delivering it. - deliver_after = now() + timedelta(days=10) - self._msgdata['deliver_after'] = deliver_after - self._outq.enqueue(self._msg, self._msgdata, - tolist=True, listname='test@example.com') - self._runner.run() - items = get_queue_messages('out') - self.assertEqual(len(items), 1) - self.assertEqual(items[0].msgdata['deliver_after'], deliver_after) - self.assertEqual(items[0].msg['message-id'], '<first>') - - - -captured_mlist = None -captured_msg = None -captured_msgdata = None - -def capture(mlist, msg, msgdata): - global captured_mlist, captured_msg, captured_msgdata - captured_mlist = mlist - captured_msg = msg - captured_msgdata = msgdata - - -class TestVERPSettings(unittest.TestCase): - """Test the selection of VERP based on various criteria.""" - - layer = ConfigLayer - - def setUp(self): - global captured_mlist, captured_msg, captured_msgdata - # Push a config where actual delivery is handled by a dummy function. - # We generally don't care what this does, since we're just testing the - # setting of the 'verp' key in the metadata. - config.push('fake outgoing', """ - [mta] - outgoing: mailman.queue.tests.test_outgoing.capture - """) - # Reset the captured data. - captured_mlist = None - captured_msg = None - captured_msgdata = None - self._mlist = create_list('test@example.com') - self._outq = config.switchboards['out'] - self._runner = make_testable_runner(OutgoingRunner, 'out') - self._msg = message_from_string("""\ -From: anne@example.com -To: test@example.com -Message-Id: <first> - -""") - - def tearDown(self): - config.pop('fake outgoing') - - def test_delivery_callback(self): - # Test that the configuration variable calls the appropriate callback. - self._outq.enqueue(self._msg, {}, listname='test@example.com') - self._runner.run() - self.assertEqual(captured_mlist, self._mlist) - self.assertEqual(captured_msg.as_string(), self._msg.as_string()) - # Of course, the message metadata will contain a bunch of keys added - # by the processing. We don't really care about the details, so this - # test is a good enough stand-in. - self.assertEqual(captured_msgdata['listname'], 'test@example.com') - - def test_verp_in_metadata(self): - # Test that if the metadata has a 'verp' key, it is unchanged. - marker = 'yepper' - msgdata = dict(verp=marker) - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') - self._runner.run() - self.assertEqual(captured_msgdata['verp'], marker) - - def test_personalized_individual_deliveries_verp(self): - # When deliveries are personalized, and the configuration setting - # indicates, messages will be VERP'd. - msgdata = {} - self._mlist.personalize = Personalization.individual - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') - with temporary_config('personalize', """ - [mta] - verp_personalized_deliveries: yes - """): - self._runner.run() - self.assertTrue(captured_msgdata['verp']) - - def test_personalized_full_deliveries_verp(self): - # When deliveries are personalized, and the configuration setting - # indicates, messages will be VERP'd. - msgdata = {} - self._mlist.personalize = Personalization.full - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') - with temporary_config('personalize', """ - [mta] - verp_personalized_deliveries: yes - """): - self._runner.run() - self.assertTrue(captured_msgdata['verp']) - - def test_personalized_deliveries_no_verp(self): - # When deliveries are personalized, but the configuration setting - # does not indicate, messages will not be VERP'd. - msgdata = {} - self._mlist.personalize = Personalization.full - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') - self._runner.run() - self.assertFalse('verp' in captured_msgdata) - - def test_verp_never(self): - # Never VERP when the interval is zero. - msgdata = {} - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') - with temporary_config('personalize', """ - [mta] - verp_delivery_interval: 0 - """): - self._runner.run() - self.assertEqual(captured_msgdata['verp'], False) - - def test_verp_always(self): - # Always VERP when the interval is one. - msgdata = {} - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') - with temporary_config('personalize', """ - [mta] - verp_delivery_interval: 1 - """): - self._runner.run() - self.assertEqual(captured_msgdata['verp'], True) - - def test_verp_on_interval_match(self): - # VERP every so often, when the post_id matches. - self._mlist.post_id = 5 - msgdata = {} - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') - with temporary_config('personalize', """ - [mta] - verp_delivery_interval: 5 - """): - self._runner.run() - self.assertEqual(captured_msgdata['verp'], True) - - def test_no_verp_on_interval_miss(self): - # VERP every so often, when the post_id matches. - self._mlist.post_id = 4 - msgdata = {} - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') - with temporary_config('personalize', """ - [mta] - verp_delivery_interval: 5 - """): - self._runner.run() - self.assertEqual(captured_msgdata['verp'], False) - - - -def raise_socket_error(mlist, msg, msgdata): - raise socket.error - - -class TestSocketError(unittest.TestCase): - """Test socket.error occurring in the delivery function.""" - - layer = ConfigLayer - - def setUp(self): - # Push a config where actual delivery is handled by a dummy function. - # We generally don't care what this does, since we're just testing the - # setting of the 'verp' key in the metadata. - config.push('fake outgoing', """ - [mta] - outgoing: mailman.queue.tests.test_outgoing.raise_socket_error - """) - self._mlist = create_list('test@example.com') - self._outq = config.switchboards['out'] - self._runner = make_testable_runner(OutgoingRunner, 'out', run_once) - self._msg = message_from_string("""\ -From: anne@example.com -To: test@example.com -Message-Id: <first> - -""") - - def tearDown(self): - config.pop('fake outgoing') - - def test_error_with_port_0(self): - # Test the code path where a socket.error is raised in the delivery - # function, and the MTA port is set to zero. The only real effect of - # that is a log message. Start by opening the error log and reading - # the current file position. - error_log = logging.getLogger('mailman.error') - filename = error_log.handlers[0].filename - filepos = os.stat(filename).st_size - self._outq.enqueue(self._msg, {}, listname='test@example.com') - with temporary_config('port 0', """ - [mta] - smtp_port: 0 - """): - self._runner.run() - with open(filename) as fp: - fp.seek(filepos) - line = fp.readline() - # The log line will contain a variable timestamp, the PID, and a - # trailing newline. Ignore these. - self.assertEqual( - line[-53:-1], - 'Cannot connect to SMTP server localhost on port smtp') - - def test_error_with_numeric_port(self): - # Test the code path where a socket.error is raised in the delivery - # function, and the MTA port is set to zero. The only real effect of - # that is a log message. Start by opening the error log and reading - # the current file position. - mark = LogFileMark('mailman.error') - self._outq.enqueue(self._msg, {}, listname='test@example.com') - with temporary_config('port 0', """ - [mta] - smtp_port: 2112 - """): - self._runner.run() - line = mark.readline() - # The log line will contain a variable timestamp, the PID, and a - # trailing newline. Ignore these. - self.assertEqual( - line[-53:-1], - 'Cannot connect to SMTP server localhost on port 2112') - - - -temporary_failures = [] -permanent_failures = [] - - -def raise_SomeRecipientsFailed(mlist, msg, msgdata): - raise SomeRecipientsFailed(temporary_failures, permanent_failures) - - -class TestSomeRecipientsFailed(unittest.TestCase): - """Test socket.error occurring in the delivery function.""" - - layer = ConfigLayer - - def setUp(self): - global temporary_failures, permanent_failures - del temporary_failures[:] - del permanent_failures[:] - self._processor = getUtility(IBounceProcessor) - # Push a config where actual delivery is handled by a dummy function. - # We generally don't care what this does, since we're just testing the - # setting of the 'verp' key in the metadata. - config.push('fake outgoing', """ - [mta] - outgoing: mailman.queue.tests.test_outgoing.raise_SomeRecipientsFailed - """) - self._mlist = create_list('test@example.com') - self._outq = config.switchboards['out'] - self._runner = make_testable_runner(OutgoingRunner, 'out', run_once) - self._msg = message_from_string("""\ -From: anne@example.com -To: test@example.com -Message-Id: <first> - -""") - - def tearDown(self): - config.pop('fake outgoing') - - def test_probe_failure(self): - # When a probe message fails during SMTP, a bounce event is recorded - # with the proper bounce context. - anne = getUtility(IUserManager).create_address('anne@example.com') - member = self._mlist.subscribe(anne, MemberRole.member) - token = send_probe(member, self._msg) - msgdata = dict(probe_token=token) - permanent_failures.append('anne@example.com') - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') - self._runner.run() - events = list(self._processor.unprocessed) - self.assertEqual(len(events), 1) - event = events[0] - self.assertEqual(event.list_name, 'test@example.com') - self.assertEqual(event.email, 'anne@example.com') - self.assertEqual(event.timestamp, datetime(2005, 8, 1, 7, 49, 23)) - self.assertEqual(event.message_id, '<first>') - self.assertEqual(event.context, BounceContext.probe) - self.assertEqual(event.processed, False) - - def test_confirmed_probe_failure(self): - # This time, a probe also fails, but for some reason the probe token - # has already been confirmed and no longer exists in the database. - anne = getUtility(IUserManager).create_address('anne@example.com') - member = self._mlist.subscribe(anne, MemberRole.member) - token = send_probe(member, self._msg) - getUtility(IPendings).confirm(token) - msgdata = dict(probe_token=token) - permanent_failures.append('anne@example.com') - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') - self._runner.run() - events = list(self._processor.unprocessed) - self.assertEqual(len(events), 0) - - def test_probe_temporary_failure(self): - # This time, a probe also fails, but the failures are temporary so - # they are not registered. - anne = getUtility(IUserManager).create_address('anne@example.com') - member = self._mlist.subscribe(anne, MemberRole.member) - token = send_probe(member, self._msg) - getUtility(IPendings).confirm(token) - msgdata = dict(probe_token=token) - temporary_failures.append('anne@example.com') - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') - self._runner.run() - events = list(self._processor.unprocessed) - self.assertEqual(len(events), 0) - - def test_one_permanent_failure(self): - # Normal (i.e. non-probe) permanent failures just get registered. - permanent_failures.append('anne@example.com') - self._outq.enqueue(self._msg, {}, listname='test@example.com') - self._runner.run() - events = list(self._processor.unprocessed) - self.assertEqual(len(events), 1) - self.assertEqual(events[0].email, 'anne@example.com') - self.assertEqual(events[0].context, BounceContext.normal) - - def test_two_permanent_failures(self): - # Two normal (i.e. non-probe) permanent failures just get registered. - permanent_failures.append('anne@example.com') - permanent_failures.append('bart@example.com') - self._outq.enqueue(self._msg, {}, listname='test@example.com') - self._runner.run() - events = list(self._processor.unprocessed) - self.assertEqual(len(events), 2) - self.assertEqual(events[0].email, 'anne@example.com') - self.assertEqual(events[0].context, BounceContext.normal) - self.assertEqual(events[1].email, 'bart@example.com') - self.assertEqual(events[1].context, BounceContext.normal) - - def test_one_temporary_failure(self): - # The first time there are temporary failures, the message just gets - # put in the retry queue, but with some metadata to prevent infinite - # retries. - temporary_failures.append('cris@example.com') - self._outq.enqueue(self._msg, {}, listname='test@example.com') - self._runner.run() - events = list(self._processor.unprocessed) - self.assertEqual(len(events), 0) - items = get_queue_messages('retry') - self.assertEqual(len(items), 1) - self.assertEqual(self._msg.as_string(), items[0].msg.as_string()) - # The metadata has three keys which are used two decide whether the - # next temporary failure should be retried. - self.assertEqual(items[0].msgdata['last_recip_count'], 1) - deliver_until = (datetime(2005, 8, 1, 7, 49, 23) + - as_timedelta(config.mta.delivery_retry_period)) - self.assertEqual(items[0].msgdata['deliver_until'], deliver_until) - self.assertEqual(items[0].msgdata['recipients'], ['cris@example.com']) - - def test_two_temporary_failures(self): - # The first time there are temporary failures, the message just gets - # put in the retry queue, but with some metadata to prevent infinite - # retries. - temporary_failures.append('cris@example.com') - temporary_failures.append('dave@example.com') - self._outq.enqueue(self._msg, {}, listname='test@example.com') - self._runner.run() - events = list(self._processor.unprocessed) - self.assertEqual(len(events), 0) - items = get_queue_messages('retry') - # There's still only one item in the retry queue, but the metadata - # contains both temporary failures. - self.assertEqual(len(items), 1) - self.assertEqual(items[0].msgdata['last_recip_count'], 2) - self.assertEqual(items[0].msgdata['recipients'], - ['cris@example.com', 'dave@example.com']) - - def test_mixed_failures(self): - # Some temporary and some permanent failures. - permanent_failures.append('elle@example.com') - permanent_failures.append('fred@example.com') - temporary_failures.append('gwen@example.com') - temporary_failures.append('herb@example.com') - self._outq.enqueue(self._msg, {}, listname='test@example.com') - self._runner.run() - # Let's look at the permanent failures. - events = list(self._processor.unprocessed) - self.assertEqual(len(events), 2) - self.assertEqual(events[0].email, 'elle@example.com') - self.assertEqual(events[0].context, BounceContext.normal) - self.assertEqual(events[1].email, 'fred@example.com') - self.assertEqual(events[1].context, BounceContext.normal) - # Let's look at the temporary failures. - items = get_queue_messages('retry') - self.assertEqual(len(items), 1) - self.assertEqual(items[0].msgdata['recipients'], - ['gwen@example.com', 'herb@example.com']) - - def test_no_progress_on_retries_within_retry_period(self): - # Temporary failures cause queuing for a retry later on, unless no - # progress is being made on the retries and we've tried for the - # specified delivery retry period. This test ensures that even if no - # progress is made, if the retry period hasn't expired, the message - # will be requeued. - temporary_failures.append('iona@example.com') - temporary_failures.append('jeff@example.com') - deliver_until = (datetime(2005, 8, 1, 7, 49, 23) + - as_timedelta(config.mta.delivery_retry_period)) - msgdata = dict(last_recip_count=2, - deliver_until=deliver_until) - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') - self._runner.run() - # The retry queue should have our message waiting to be retried. - items = get_queue_messages('retry') - self.assertEqual(len(items), 1) - self.assertEqual(items[0].msgdata['deliver_until'], deliver_until) - self.assertEqual(items[0].msgdata['recipients'], - ['iona@example.com', 'jeff@example.com']) - - def test_no_progress_on_retries_with_expired_retry_period(self): - # We've had temporary failures with no progress, and the retry period - # has expired. In that case, a log entry is written and message is - # discarded. There's nothing more that can be done. - temporary_failures.append('kira@example.com') - temporary_failures.append('lonn@example.com') - retry_period = as_timedelta(config.mta.delivery_retry_period) - deliver_until = datetime(2005, 8, 1, 7, 49, 23) + retry_period - msgdata = dict(last_recip_count=2, - deliver_until=deliver_until) - self._outq.enqueue(self._msg, msgdata, listname='test@example.com') - # Before the queue runner runs, several days pass. - factory.fast_forward(retry_period.days + 1) - mark = LogFileMark('mailman.smtp') - self._runner.run() - # There should be no message in the retry or outgoing queues. - self.assertEqual(len(get_queue_messages('retry')), 0) - self.assertEqual(len(get_queue_messages('out')), 0) - # There should be a log message in the smtp log indicating that the - # message has been discarded. - line = mark.readline() - self.assertEqual( - line[-63:-1], - 'Discarding message with persistent temporary failures: <first>') - - - -def test_suite(): - suite = unittest.TestSuite() - suite.addTest(unittest.makeSuite(TestOnce)) - suite.addTest(unittest.makeSuite(TestVERPSettings)) - suite.addTest(unittest.makeSuite(TestSocketError)) - suite.addTest(unittest.makeSuite(TestSomeRecipientsFailed)) - return suite diff --git a/src/mailman/queue/virgin.py b/src/mailman/queue/virgin.py deleted file mode 100644 index 2dcdca910..000000000 --- a/src/mailman/queue/virgin.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (C) 1998-2011 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Virgin message queue runner. - -This qrunner handles messages that the Mailman system gives virgin birth to. -E.g. acknowledgement responses to user posts or Replybot messages. They need -to go through some minimal processing before they can be sent out to the -recipient. -""" - -from mailman.core.pipelines import process -from mailman.queue import Runner - - - -class VirginRunner(Runner): - def _dispose(self, mlist, msg, msgdata): - # We need to fast track this message through any pipeline handlers - # that touch it, e.g. especially cook-headers. - msgdata['_fasttrack'] = True - # Use the 'virgin' pipeline. - process(mlist, msg, msgdata, 'virgin') - # Do not keep this message queued. - return False |