diff options
| -rw-r--r-- | mailman/Defaults.py | 15 | ||||
| -rw-r--r-- | mailman/Message.py | 2 | ||||
| -rw-r--r-- | mailman/SafeDict.py | 31 | ||||
| -rw-r--r-- | mailman/Utils.py | 2 | ||||
| -rw-r--r-- | mailman/bin/withlist.py | 5 | ||||
| -rw-r--r-- | mailman/pipeline/smtp_direct.py | 62 | ||||
| -rw-r--r-- | mailman/queue/__init__.py | 38 | ||||
| -rw-r--r-- | mailman/tests/test_safedict.py | 50 |
8 files changed, 81 insertions, 124 deletions
diff --git a/mailman/Defaults.py b/mailman/Defaults.py index 04152dd33..43bf7a3c3 100644 --- a/mailman/Defaults.py +++ b/mailman/Defaults.py @@ -551,19 +551,21 @@ LOG_CONFIG_FILE = None # printing of this log message. SMTP_LOG_EVERY_MESSAGE = ( 'smtp', - '%(msg_message-id)s smtp to %(listname)s for %(#recips)d recips, completed in %(time).3f seconds') + ('${message-id} smtp to $listname for ${#recips} recips, ' + 'completed in $time seconds')) # This will only be printed if there were no immediate smtp failures. # Mutually exclusive with SMTP_LOG_REFUSED. SMTP_LOG_SUCCESS = ( 'post', - 'post to %(listname)s from %(sender)s, size=%(size)d, message-id=%(msg_message-id)s, success') + '${message-id} post to $listname from $sender, size=$size, success') # This will only be printed if there were any addresses which encountered an # immediate smtp failure. Mutually exclusive with SMTP_LOG_SUCCESS. SMTP_LOG_REFUSED = ( 'post', - 'post to %(listname)s from %(sender)s, size=%(size)d, message-id=%(msg_message-id)s, %(#refused)d failures') + ('${message-id} post to $listname from $sender, size=$size, ' + '${#refused} failures')) # This will be logged for each specific recipient failure. Additional %()s # keys are: @@ -573,7 +575,8 @@ SMTP_LOG_REFUSED = ( # failmsg -- the actual smtp message, if available SMTP_LOG_EACH_FAILURE = ( 'smtp-failure', - 'delivery to %(recipient)s failed with code %(failcode)d: %(failmsg)s') + ('${message-id} delivery to $recipient failed with code $failcode: ' + '$failmsg')) # These variables control the format and frequency of VERP-like delivery for # better bounce detection. VERP is Variable Envelope Return Path, defined @@ -878,8 +881,8 @@ DEFAULT_MSG_HEADER = u'' DEFAULT_MSG_FOOTER = u"""\ _______________________________________________ $real_name mailing list -$fqdn_realname -${web_page_url}listinfo${cgiext}/${list_name} +$fqdn_listname +${web_page_url}listinfo${cgiext}/${fqdn_listname} """ # Scrub regular delivery diff --git a/mailman/Message.py b/mailman/Message.py index e822dd0ff..42c778894 100644 --- a/mailman/Message.py +++ b/mailman/Message.py @@ -270,7 +270,6 @@ class UserNotification(Message): recips=self.recips, nodecorate=True, reduced_list_headers=True, - pipeline='virgin', ) if mlist is not None: enqueue_kws['listname'] = mlist.fqdn_listname @@ -307,5 +306,4 @@ class OwnerNotification(UserNotification): nodecorate=True, reduced_list_headers=True, envsender=self._sender, - pipeline='virgin', **_kws) diff --git a/mailman/SafeDict.py b/mailman/SafeDict.py index 1db43a760..1549777da 100644 --- a/mailman/SafeDict.py +++ b/mailman/SafeDict.py @@ -53,34 +53,3 @@ class SafeDict(dict): if isinstance(v, str): self.__setitem__(k, unicode(v, self.cset)) return template % self - - - -class MsgSafeDict(SafeDict): - def __init__(self, msg, d=None): - self.__msg = msg - if d is None: - d = {} - super(MsgSafeDict, self).__init__(d) - - def __getitem__(self, key): - if key.startswith('msg_'): - return self.__msg.get(key[4:], 'n/a') - elif key.startswith('allmsg_'): - missing = [] - all = self.__msg.get_all(key[7:], missing) - if all is missing: - return 'n/a' - return COMMASPACE.join(all) - else: - return super(MsgSafeDict, self).__getitem__(key) - - def copy(self): - d = super(MsgSafeDict, self).copy() - for k in self.__msg.keys(): - vals = self.__msg.get_all(k) - if len(vals) == 1: - d['msg_'+k.lower()] = vals[0] - else: - d['allmsg_'+k.lower()] = COMMASPACE.join(vals) - return d diff --git a/mailman/Utils.py b/mailman/Utils.py index 621a307d4..273d6af06 100644 --- a/mailman/Utils.py +++ b/mailman/Utils.py @@ -858,5 +858,3 @@ def get_pattern(email, pattern_list): matched = pattern break return matched - - diff --git a/mailman/bin/withlist.py b/mailman/bin/withlist.py index c6a352319..2b312a87c 100644 --- a/mailman/bin/withlist.py +++ b/mailman/bin/withlist.py @@ -210,5 +210,8 @@ def main(): "The variable 'm' is the $listname mailing list") else: banner = interact.DEFAULT_BANNER - overrides = dict(m=LAST_MLIST, r=r) + overrides = dict(m=LAST_MLIST, r=r, + commit=config.db.commit, + abort=config.db.abort, + config=config) interact.interact(upframe=False, banner=banner, overrides=overrides) diff --git a/mailman/pipeline/smtp_direct.py b/mailman/pipeline/smtp_direct.py index 74eaa5aad..e51a148c3 100644 --- a/mailman/pipeline/smtp_direct.py +++ b/mailman/pipeline/smtp_direct.py @@ -34,6 +34,7 @@ import copy import time import email import socket +import string import logging import smtplib @@ -44,7 +45,6 @@ from zope.interface import implements from mailman import Errors from mailman import Utils -from mailman.SafeDict import MsgSafeDict from mailman.configuration import config from mailman.i18n import _ from mailman.interfaces import IHandler, Personalization @@ -101,6 +101,20 @@ class Connection: +class MessageDict(dict): + def __init__(self, message, extras): + super(MessageDict, self).__init__() + for key, value in message.items(): + self[key.lower()] = value + self.update(extras) + + +class Template(string.Template): + # Allow dashes and # signs, in addition to the standard pattern. + idpattern = '[_a-z#-][_a-z0-9#-]*' + + + def process(mlist, msg, msgdata): recips = msgdata.get('recips') if not recips: @@ -178,23 +192,27 @@ def process(mlist, msg, msgdata): msgdata['recips'] = origrecips # Log the successful post t1 = time.time() - d = MsgSafeDict(msg, {'time' : t1-t0, - # BAW: Urg. This seems inefficient. - 'size' : len(msg.as_string()), - '#recips' : len(recips), - '#refused': len(refused), - 'listname': mlist.fqdn_listname, - 'sender' : origsender, - }) + substitutions = MessageDict(msg, { + 'time' : t1-t0, + 'size' : msg.original_size, + '#recips' : len(recips), + '#refused' : len(refused), + 'listname' : mlist.fqdn_listname, + 'sender' : origsender, + }) + if 'message-id' not in substitutions: + substitutions['message-id'] = 'n/a' # We have to use the copy() method because extended call syntax requires a # concrete dictionary object; it does not allow a generic mapping (XXX is # this still true in Python 2.3?). if config.SMTP_LOG_EVERY_MESSAGE: - every_log.info('%s', config.SMTP_LOG_EVERY_MESSAGE[1] % d) + template = Template(config.SMTP_LOG_EVERY_MESSAGE[1]) + every_log.info('%s', template.safe_substitute(substitutions)) if refused: if config.SMTP_LOG_REFUSED: - refused_log.info('%s', config.SMTP_LOG_REFUSED[1] % d) + template = Template(config.SMTP_LOG_REFUSED[1]) + refused_log.info('%s', template.safe_substitute(substitutions)) elif msgdata.get('tolist'): # Log the successful post, but only if it really was a post to the @@ -203,7 +221,8 @@ def process(mlist, msg, msgdata): # the other messages, but in that case, we should probably have a # separate configuration variable to control that. if config.SMTP_LOG_SUCCESS: - success_log.info('%s', config.SMTP_LOG_SUCCESS[1] % d) + template = Template(config.SMTP_LOG_SUCCESS[1]) + success_log.info('%s', template.safe_substitute(substitutions)) # Process any failed deliveries. tempfailures = [] @@ -226,10 +245,13 @@ def process(mlist, msg, msgdata): # future delivery. TBD: this could generate lots of log entries! tempfailures.append(recip) if config.SMTP_LOG_EACH_FAILURE: - d.update({'recipient': recip, - 'failcode' : code, - 'failmsg' : smtpmsg}) - failure_log.info('%s', config.SMTP_LOG_EACH_FAILURE[1] % d) + substitutions.update({ + 'recipient' : recip, + 'failcode' : code, + 'failmsg' : smtpmsg, + }) + template = Template(config.SMTP_LOG_EACH_FAILURE[1]) + failure_log.info('%s', template.safe_substitute(substitutions)) # Return the results if tempfailures or permfailures: raise Errors.SomeRecipientsFailed(tempfailures, permfailures) @@ -371,11 +393,11 @@ def bulkdeliver(mlist, msg, msgdata, envsender, failures, conn): # Send the message refused = conn.sendmail(envsender, recips, msgtext) except smtplib.SMTPRecipientsRefused, e: - flog.error('All recipients refused: %s, msgid: %s', e, msgid) + flog.error('%s recipients refused: %s', msgid, e) refused = e.recipients except smtplib.SMTPResponseException, e: - flog.error('SMTP session failure: %s, %s, msgid: %s', - e.smtp_code, e.smtp_error, msgid) + flog.error('%s SMTP session failure: %s, %s', + msgid, e.smtp_code, e.smtp_error) # If this was a permanent failure, don't add the recipients to the # refused, because we don't want them to be added to failures. # Otherwise, if the MTA rejects the message because of the message @@ -390,7 +412,7 @@ def bulkdeliver(mlist, msg, msgdata, envsender, failures, conn): # MTA not responding, or other socket problems, or any other kind of # SMTPException. In that case, nothing got delivered, so treat this # as a temporary failure. - flog.error('Low level smtp error: %s, msgid: %s', e, msgid) + flog.error('%s low level smtp error: %s', msgid, e) error = str(e) for r in recips: refused[r] = (-1, error) diff --git a/mailman/queue/__init__.py b/mailman/queue/__init__.py index 17386cda6..5748b38b2 100644 --- a/mailman/queue/__init__.py +++ b/mailman/queue/__init__.py @@ -60,7 +60,8 @@ shamax = 0xffffffffffffffffffffffffffffffffffffffffL # prevents skipping one of two entries with the same time until the next pass. DELTA = .0001 -log = logging.getLogger('mailman.error') +elog = logging.getLogger('mailman.error') +dlog = logging.getLogger('mailman.debug') @@ -168,7 +169,8 @@ class Switchboard: else: os.unlink(bakfile) except EnvironmentError, e: - log.exception('Failed to unlink/preserve backup file: %s', bakfile) + elog.exception( + 'Failed to unlink/preserve backup file: %s', bakfile) @property def files(self): @@ -257,12 +259,15 @@ class Runner: self._cleanup() def _oneloop(self): + me = self.__class__.__name__ + dlog.debug('[%s] starting oneloop', me) # First, list all the files in our queue directory. # Switchboard.files() is guaranteed to hand us the files in FIFO # order. Return an integer count of the number of files that were # available for this qrunner to process. files = self._switchboard.files for filebase in files: + dlog.debug('[%s] processing filebase: %s', me, filebase) try: # Ask the switchboard for the message and metadata objects # associated with this filebase. @@ -274,13 +279,15 @@ class Runner: # We don't want the runner to die, so we just log and skip # this entry, but preserve it for analysis. self._log(e) - log.error('Skipping and preserving unparseable message: %s', - filebase) + elog.error('Skipping and preserving unparseable message: %s', + filebase) self._switchboard.finish(filebase, preserve=True) config.db.abort() continue try: + dlog.debug('[%s] processing onefile', me) self._onefile(msg, msgdata) + dlog.debug('[%s] finishing filebase: %s', me, filebase) self._switchboard.finish(filebase) except Exception, e: # All runners that implement _dispose() must guarantee that @@ -297,23 +304,30 @@ class Runner: # message. Try to be graceful. try: new_filebase = self._shunt.enqueue(msg, msgdata) - log.error('SHUNTING: %s', new_filebase) + elog.error('SHUNTING: %s', new_filebase) self._switchboard.finish(filebase) except Exception, e: # The message wasn't successfully shunted. Log the # exception and try to preserve the original queue entry # for possible analysis. self._log(e) - log.error('SHUNTING FAILED, preserving original entry: %s', - filebase) + elog.error( + 'SHUNTING FAILED, preserving original entry: %s', + filebase) self._switchboard.finish(filebase, preserve=True) - config.db.abort() + config.db.abort() # Other work we want to do each time through the loop. + dlog.debug('[%s] reaping', me) Utils.reap(self._kids, once=True) + dlog.debug('[%s] doing periodic', me) self._doperiodic() + dlog.debug('[%s] checking short circuit', me) if self._shortcircuit(): + dlog.debug('[%s] short circuiting', me) break + dlog.debug('[%s] commiting', me) config.db.commit() + dlog.debug('[%s] ending oneloop: %s', me, len(files)) return len(files) def _onefile(self, msg, msgdata): @@ -327,8 +341,8 @@ class Runner: listname = msgdata.get('listname') mlist = config.db.list_manager.get(listname) if not mlist: - log.error('Dequeuing message destined for missing list: %s', - listname) + elog.error('Dequeuing message destined for missing list: %s', + listname) self._shunt.enqueue(msg, msgdata) return # Now process this message, keeping track of any subprocesses that may @@ -357,10 +371,10 @@ class Runner: self._switchboard.enqueue(msg, msgdata) def _log(self, exc): - log.error('Uncaught runner exception: %s', exc) + elog.error('Uncaught runner exception: %s', exc) s = StringIO() traceback.print_exc(file=s) - log.error('%s', s.getvalue()) + elog.error('%s', s.getvalue()) # # Subclasses can override these methods. diff --git a/mailman/tests/test_safedict.py b/mailman/tests/test_safedict.py index c1eeb15f2..544824870 100644 --- a/mailman/tests/test_safedict.py +++ b/mailman/tests/test_safedict.py @@ -42,57 +42,7 @@ class TestSafeDict(unittest.TestCase): -class TestMsgSafeDict(unittest.TestCase): - def setUp(self): - self._msg = email.message_from_string("""To: foo -From: bar -Subject: baz -Cc: aperson@dom.ain -Cc: bperson@dom.ain - -""") - - def test_normal_key(self): - sd = SafeDict.MsgSafeDict(self._msg, {'key': 'value'}) - si = '%(key)s' % sd - self.assertEqual(si, 'value') - - def test_msg_key(self): - sd = SafeDict.MsgSafeDict(self._msg, {'to': 'value'}) - si = '%(msg_to)s' % sd - self.assertEqual(si, 'foo') - - def test_allmsg_key(self): - sd = SafeDict.MsgSafeDict(self._msg, {'cc': 'value'}) - si = '%(allmsg_cc)s' % sd - self.assertEqual(si, 'aperson@dom.ain, bperson@dom.ain') - - def test_msg_no_key(self): - sd = SafeDict.MsgSafeDict(self._msg) - si = '%(msg_date)s' % sd - self.assertEqual(si, 'n/a') - - def test_allmsg_no_key(self): - sd = SafeDict.MsgSafeDict(self._msg) - si = '%(allmsg_date)s' % sd - self.assertEqual(si, 'n/a') - - def test_copy(self): - sd = SafeDict.MsgSafeDict(self._msg, {'foo': 'bar'}) - copy = sd.copy() - items = copy.items() - items.sort() - self.assertEqual(items, [ - ('allmsg_cc', 'aperson@dom.ain, bperson@dom.ain'), - ('foo', 'bar'), - ('msg_from', 'bar'), - ('msg_subject', 'baz'), - ('msg_to', 'foo'), - ]) - - def test_suite(): suite = unittest.TestSuite() suite.addTest(unittest.makeSuite(TestSafeDict)) - suite.addTest(unittest.makeSuite(TestMsgSafeDict)) return suite |