diff options
Diffstat (limited to '')
| -rw-r--r-- | mailman/Message.py | 7 | ||||
| -rw-r--r-- | mailman/app/chains.py | 2 | ||||
| -rw-r--r-- | mailman/app/pipelines.py | 45 | ||||
| -rw-r--r-- | mailman/bin/dumpdb.py | 134 | ||||
| -rw-r--r-- | mailman/bin/inject.py | 7 | ||||
| -rw-r--r-- | mailman/inject.py | 50 | ||||
| -rw-r--r-- | mailman/queue/__init__.py | 9 | ||||
| -rw-r--r-- | mailman/queue/archive.py | 38 | ||||
| -rw-r--r-- | mailman/queue/outgoing.py | 9 | ||||
| -rw-r--r-- | mailman/queue/virgin.py | 20 |
10 files changed, 178 insertions, 143 deletions
diff --git a/mailman/Message.py b/mailman/Message.py index 5d20fb897..e822dd0ff 100644 --- a/mailman/Message.py +++ b/mailman/Message.py @@ -248,11 +248,10 @@ class UserNotification(Message): This is used for all internally crafted messages. """ # Since we're crafting the message from whole cloth, let's make sure - # this message has a Message-ID. Yes, the MTA would give us one, but - # this is useful for logging to logs/smtp. + # this message has a Message-ID. if 'message-id' not in self: self['Message-ID'] = email.utils.make_msgid() - # Ditto for Date: which is required by RFC 2822 + # Ditto for Date: as required by RFC 2822. if 'date' not in self: self['Date'] = email.utils.formatdate(localtime=True) # UserNotifications are typically for admin messages, and for messages @@ -271,6 +270,7 @@ class UserNotification(Message): recips=self.recips, nodecorate=True, reduced_list_headers=True, + pipeline='virgin', ) if mlist is not None: enqueue_kws['listname'] = mlist.fqdn_listname @@ -307,4 +307,5 @@ class OwnerNotification(UserNotification): nodecorate=True, reduced_list_headers=True, envsender=self._sender, + pipeline='virgin', **_kws) diff --git a/mailman/app/chains.py b/mailman/app/chains.py index 58f940a83..d2f9eab86 100644 --- a/mailman/app/chains.py +++ b/mailman/app/chains.py @@ -25,11 +25,11 @@ __metaclass__ = type from mailman.chains.accept import AcceptChain +from mailman.chains.builtin import BuiltInChain from mailman.chains.discard import DiscardChain from mailman.chains.headers import HeaderMatchChain from mailman.chains.hold import HoldChain from mailman.chains.reject import RejectChain -from mailman.chains.builtin import BuiltInChain from mailman.configuration import config from mailman.interfaces import LinkAction diff --git a/mailman/app/pipelines.py b/mailman/app/pipelines.py index ab128a2b1..c2b0a6890 100644 --- a/mailman/app/pipelines.py +++ b/mailman/app/pipelines.py @@ -48,11 +48,27 @@ def process(mlist, msg, msgdata, pipeline_name='built-in'): -class BuiltInPipeline: - """The built-in pipeline.""" +class BasePipeline: + """Base pipeline implementation.""" implements(IPipeline) + _default_handlers = () + + def __init__(self): + self._handlers = [] + for handler_name in self._default_handlers: + self._handlers.append(config.handlers[handler_name]) + + def __iter__(self): + """See `IPipeline`.""" + for handler in self._handlers: + yield handler + + +class BuiltInPipeline(BasePipeline): + """The built-in pipeline.""" + name = 'built-in' description = _('The built-in pipeline.') @@ -73,15 +89,19 @@ class BuiltInPipeline: 'to-outgoing', ) - def __init__(self): - self._handlers = [] - for handler_name in self._default_handlers: - self._handlers.append(config.handlers[handler_name]) - def __iter__(self): - """See `IPipeline`.""" - for handler in self._handlers: - yield handler +class VirginPipeline(BasePipeline): + """The processing pipeline for virgin messages. + + Virgin messages are those that are crafted internally by Mailman. + """ + name = 'virgin' + description = _('The virgin queue pipeline.') + + _default_handlers = ( + 'cook-headers', + 'to-outgoing', + ) @@ -97,5 +117,6 @@ def initialize(): (handler.name, handler_finder)) config.handlers[handler.name] = handler # Set up some pipelines. - pipeline = BuiltInPipeline() - config.pipelines[pipeline.name] = pipeline + for pipeline_class in (BuiltInPipeline, VirginPipeline): + pipeline = pipeline_class() + config.pipelines[pipeline.name] = pipeline diff --git a/mailman/bin/dumpdb.py b/mailman/bin/dumpdb.py index 5ee7ff9dd..508b9d571 100644 --- a/mailman/bin/dumpdb.py +++ b/mailman/bin/dumpdb.py @@ -1,5 +1,3 @@ -#! @PYTHON@ -# # Copyright (C) 1998-2008 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or @@ -17,110 +15,76 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, # USA. -import sys +from __future__ import with_statement + import pprint import cPickle -import marshal -import optparse -from mailman import Version from mailman.configuration import config from mailman.i18n import _ +from mailman.interact import interact +from mailman.options import Options COMMASPACE = ', ' +m = [] -def parseargs(): - parser = optparse.OptionParser(version=Version.MAILMAN_VERSION, - usage=_("""\ -%%prog [options] filename +class ScriptOptions(Options): + usage=_("""\ +%prog [options] filename + +Dump the contents of any Mailman queue file. The queue file is a data file +with multiple Python pickles in it.""") -Dump the contents of any Mailman `database' file. + def add_options(self): + super(ScriptOptions, self).add_options() + self.parser.add_option( + '-n', '--noprint', + dest='doprint', default=True, action='store_false', + help=_("""\ +Don't attempt to pretty print the object. This is useful if there is some +problem with the object and you just want to get an unpickled representation. +Useful with 'bin/dumpdb -i <file>'. In that case, the list of +unpickled objects will be left in a variable called 'm'.""")) + self.parser.add_option( + '-i', '--interact', + default=False, action='store_true', + help=_("""\ +Start an interactive Python session, with a variable called 'm' containing the +list of unpickled objects.""")) -If the filename ends with `.db', then it is assumed that the file contains a -Python marshal. If the file ends with `.pck' then it is assumed to contain a -Python pickle. In either case, if you want to override the default assumption --- or if the file ends in neither suffix -- use the -p or -m flags.""")) - parser.add_option('-m', '--marshal', - default=False, action='store_true', - help=_("""\ -Assume the file contains a Python marshal, -overridding any automatic guessing.""")) - parser.add_option('-p', '--pickle', - default=False, action='store_true', - help=_("""\ -Assume the file contains a Python pickle, -overridding any automatic guessing.""")) - parser.add_option('-n', '--noprint', - default=False, action='store_true', - help=_("""\ -Don't attempt to pretty print the object. This is useful if there's -some problem with the object and you just want to get an unpickled -representation. Useful with `python -i bin/dumpdb <file>'. In that -case, the root of the tree will be left in a global called "msg".""")) - parser.add_option('-C', '--config', - help=_('Alternative configuration file to use')) - opts, args = parser.parse_args() - # Options. - # None == guess, 0 == pickle, 1 == marshal - opts.filetype = None - if opts.pickle: - opts.filetype = 0 - if opts.marshal: - opts.filetype = 1 - opts.doprint = not opts.noprint - if len(args) < 1: - parser.error(_('No filename given.')) - elif len(args) > 1: - pargs = COMMASPACE.join(args) - parser.error(_('Bad arguments: $pargs')) - else: - opts.filename = args[0] - if opts.filetype is None: - if opts.filename.endswith('.db'): - opts.filetype = 1 - elif opts.filename.endswith('.pck'): - opts.filetype = 0 + def sanity_check(self): + if len(self.arguments) < 1: + self.parser.error(_('No filename given.')) + elif len(self.arguments) > 1: + self.parser.error(_('Unexpected arguments')) else: - parser.error(_('Please specify either -p or -m.')) - return parser, opts, args + self.filename = self.arguments[0] def main(): - parser, opts, args = parseargs() - config.load(opts.config) + options = ScriptOptions() + options.initialize() - # Handle dbs pp = pprint.PrettyPrinter(indent=4) - if opts.filetype == 1: - load = marshal.load - typename = 'marshal' - else: - load = cPickle.load - typename = 'pickle' - fp = open(opts.filename) - m = [] - try: - cnt = 1 - if opts.doprint: - print _('[----- start $typename file -----]') + with open(options.filename) as fp: while True: try: - obj = load(fp) + m.append(cPickle.load(fp)) except EOFError: - if opts.doprint: - print _('[----- end $typename file -----]') break - if opts.doprint: - print _('<----- start object $cnt ----->') - if isinstance(obj, str): - print obj - else: - pp.pprint(obj) - cnt += 1 - m.append(obj) - finally: - fp.close() + if options.options.doprint: + print _('[----- start pickle -----]') + for i, obj in enumerate(m): + count = i + 1 + print _('<----- start object $count ----->') + if isinstance(obj, basestring): + print obj + else: + pp.pprint(obj) + print _('[----- end pickle -----]') + if options.options.interact: + interact() diff --git a/mailman/bin/inject.py b/mailman/bin/inject.py index 3e4012baf..8c4613756 100644 --- a/mailman/bin/inject.py +++ b/mailman/bin/inject.py @@ -20,11 +20,14 @@ from __future__ import with_statement import os import sys +from email import message_from_string + from mailman import Utils from mailman import Version +from mailman.Message import Message from mailman.configuration import config from mailman.i18n import _ -from mailman.inject import inject +from mailman.inject import inject_text from mailman.options import SingleMailingListOptions @@ -81,7 +84,7 @@ def main(): with open(options.filename) as fp: message_text = fp.read() - inject(fqdn_listname, message_text, qdir=qdir) + inject_text(mlist, message_text, qdir=qdir) diff --git a/mailman/inject.py b/mailman/inject.py index 5796eff5c..7e50513d8 100644 --- a/mailman/inject.py +++ b/mailman/inject.py @@ -15,20 +15,64 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, # USA. +__metaclass__ = type +__all__ = [ + 'inject_message', + 'inject_text', + ] + + +from email import message_from_string +from email.utils import formatdate, make_msgid + +from mailman.Message import Message from mailman.configuration import config from mailman.queue import Switchboard -def inject(listname, msg, recips=None, qdir=None): +def inject_message(mlist, msg, recips=None, qdir=None): + """Inject a message into a queue. + + :param mlist: The mailing list this message is destined for. + :param msg: The Message object to inject. + :param recips: Optional set of recipients to put into the message's + metadata. + :param qdir: Optional queue directory to inject this message into. If not + given, the incoming queue is used. + """ if qdir is None: qdir = config.INQUEUE_DIR + # Since we're crafting the message from whole cloth, let's make sure this + # message has a Message-ID. + if 'message-id' not in msg: + msg['Message-ID'] = make_msgid() + # Ditto for Date: as required by RFC 2822. + if 'date' not in msg: + msg['Date'] = formatdate(localtime=True) queue = Switchboard(qdir) kws = dict( - listname=listname, + listname=mlist.fqdn_listname, tolist=True, - _plaintext=True, + original_size=getattr(msg, 'original_size', len(msg.as_string())), ) if recips is not None: kws['recips'] = recips queue.enqueue(msg, **kws) + + + +def inject_text(mlist, text, recips=None, qdir=None): + """Inject a message into a queue. + + :param mlist: The mailing list this message is destined for. + :param text: The text of the message to inject. This will be parsed into + a Message object. + :param recips: Optional set of recipients to put into the message's + metadata. + :param qdir: Optional queue directory to inject this message into. If not + given, the incoming queue is used. + """ + message = message_from_string(text, Message) + message.original_size = len(text) + inject_message(mlist, message, recips, qdir) diff --git a/mailman/queue/__init__.py b/mailman/queue/__init__.py index 4175babaa..17386cda6 100644 --- a/mailman/queue/__init__.py +++ b/mailman/queue/__init__.py @@ -38,6 +38,7 @@ import sha import time import email import errno +import pickle import cPickle import logging import marshal @@ -95,12 +96,12 @@ class Switchboard: listname = data.get('listname', '--nolist--') # Get some data for the input to the sha hash. now = time.time() - if not data.get('_plaintext'): - protocol = 1 - msgsave = cPickle.dumps(_msg, protocol) - else: + if data.get('_plaintext'): protocol = 0 msgsave = cPickle.dumps(str(_msg), protocol) + else: + protocol = pickle.HIGHEST_PROTOCOL + msgsave = cPickle.dumps(_msg, protocol) # listname is unicode but the input to the hash function must be an # 8-bit string (eventually, a bytes object). hashfood = msgsave + listname.encode('utf-8') + repr(now) diff --git a/mailman/queue/archive.py b/mailman/queue/archive.py index 854e4340f..013f8c949 100644 --- a/mailman/queue/archive.py +++ b/mailman/queue/archive.py @@ -28,6 +28,7 @@ __all__ = [ import os import time +from datetime import datetime from email.Utils import parsedate_tz, mktime_tz, formatdate from locknix.lockfile import Lock @@ -44,37 +45,38 @@ class ArchiveRunner(Runner): # Support clobber_date, i.e. setting the date in the archive to the # received date, not the (potentially bogus) Date: header of the # original message. - clobber = 0 - originaldate = msg.get('date') - receivedtime = formatdate(msgdata['received_time']) - if not originaldate: - clobber = 1 + clobber = False + original_date = msg.get('date') + received_time = formatdate(msgdata['received_time']) + if not original_date: + clobber = True elif config.ARCHIVER_CLOBBER_DATE_POLICY == 1: - clobber = 1 + clobber = True elif config.ARCHIVER_CLOBBER_DATE_POLICY == 2: - # what's the timestamp on the original message? - tup = parsedate_tz(originaldate) - now = time.time() + # What's the timestamp on the original message? + timetup = parsedate_tz(original_date) + now = datetime.now() try: - if not tup: - clobber = 1 - elif abs(now - mktime_tz(tup)) > \ - config.ARCHIVER_ALLOWABLE_SANE_DATE_SKEW: - clobber = 1 + if not timetup: + clobber = True + else: + utc_timestamp = datetime.fromtimestamp(mktime_tz(timetup)) + clobber = (abs(now - utc_timestamp) > + config.ARCHIVER_ALLOWABLE_SANE_DATE_SKEW) except (ValueError, OverflowError): # The likely cause of this is that the year in the Date: field # is horribly incorrect, e.g. (from SF bug # 571634): # Date: Tue, 18 Jun 0102 05:12:09 +0500 # Obviously clobber such dates. - clobber = 1 + clobber = True if clobber: del msg['date'] del msg['x-original-date'] - msg['Date'] = receivedtime + msg['Date'] = received_time if originaldate: - msg['X-Original-Date'] = originaldate + msg['X-Original-Date'] = original_date # Always put an indication of when we received the message. - msg['X-List-Received-Date'] = receivedtime + msg['X-List-Received-Date'] = received_time # While a list archiving lock is acquired, archive the message. with Lock(os.path.join(mlist.data_path, 'archive.lck')): for archive_factory in get_plugins('mailman.archiver'): diff --git a/mailman/queue/outgoing.py b/mailman/queue/outgoing.py index 4c067b8c0..399432e5d 100644 --- a/mailman/queue/outgoing.py +++ b/mailman/queue/outgoing.py @@ -23,7 +23,8 @@ import copy import email import socket import logging -import datetime + +from datetime import datetime from mailman import Errors from mailman import Message @@ -56,8 +57,8 @@ class OutgoingRunner(Runner, BounceMixin): def _dispose(self, mlist, msg, msgdata): # See if we should retry delivery of this message again. - deliver_after = msgdata.get('deliver_after', 0) - if datetime.datetime.now() < deliver_after: + deliver_after = msgdata.get('deliver_after', datetime.fromtimestamp(0)) + if datetime.now() < deliver_after: return True # Make sure we have the most up-to-date state try: @@ -102,7 +103,7 @@ class OutgoingRunner(Runner, BounceMixin): # occasionally move them back here for another shot at # delivery. if e.tempfailures: - now = datetime.datetime.now() + now = datetime.now() recips = e.tempfailures last_recip_count = msgdata.get('last_recip_count', 0) deliver_until = msgdata.get('deliver_until', now) diff --git a/mailman/queue/virgin.py b/mailman/queue/virgin.py index 5534c95f0..0494700ce 100644 --- a/mailman/queue/virgin.py +++ b/mailman/queue/virgin.py @@ -23,22 +23,20 @@ to go through some minimal processing before they can be sent out to the recipient. """ +from mailman.app.pipelines import process from mailman.configuration import config from mailman.queue import Runner -from mailman.queue.incoming import IncomingRunner -class VirginRunner(IncomingRunner): +class VirginRunner(Runner): QDIR = config.VIRGINQUEUE_DIR def _dispose(self, mlist, msg, msgdata): - # We need to fasttrack this message through any handlers that touch - # it. E.g. especially CookHeaders. - msgdata['_fasttrack'] = 1 - return IncomingRunner._dispose(self, mlist, msg, msgdata) - - def _get_pipeline(self, mlist, msg, msgdata): - # It's okay to hardcode this, since it'll be the same for all - # internally crafted messages. - return ['CookHeaders', 'ToOutgoing'] + # We need to fast track this message through any pipeline handlers + # that touch it, e.g. especially cook-headers. + msgdata['_fasttrack'] = True + # Use the 'virgin' pipeline. + process(mlist, msg, msgdata, 'virgin') + # Do not keep this message queued. + return False |