diff options
| -rw-r--r-- | src/mailman/commands/docs/membership.rst | 4 | ||||
| -rw-r--r-- | src/mailman/commands/docs/qfile.rst | 1 | ||||
| -rw-r--r-- | src/mailman/config/schema.cfg | 16 | ||||
| -rw-r--r-- | src/mailman/core/switchboard.py | 3 | ||||
| -rw-r--r-- | src/mailman/docs/NEWS.rst | 23 | ||||
| -rw-r--r-- | src/mailman/handlers/after_delivery.py | 5 | ||||
| -rw-r--r-- | src/mailman/handlers/docs/after-delivery.rst | 5 | ||||
| -rw-r--r-- | src/mailman/handlers/docs/replybot.rst | 6 | ||||
| -rw-r--r-- | src/mailman/handlers/to_digest.py | 4 | ||||
| -rw-r--r-- | src/mailman/interfaces/archiver.py | 9 | ||||
| -rw-r--r-- | src/mailman/model/docs/registration.rst | 3 | ||||
| -rw-r--r-- | src/mailman/model/docs/requests.rst | 3 | ||||
| -rw-r--r-- | src/mailman/model/listmanager.py | 5 | ||||
| -rw-r--r-- | src/mailman/model/pending.py | 12 | ||||
| -rw-r--r-- | src/mailman/mta/postfix.py | 4 | ||||
| -rw-r--r-- | src/mailman/runners/archive.py | 113 | ||||
| -rw-r--r-- | src/mailman/runners/lmtp.py | 6 | ||||
| -rw-r--r-- | src/mailman/runners/tests/test_archiver.py | 146 | ||||
| -rw-r--r-- | src/mailman/runners/tests/test_lmtp.py | 16 | ||||
| -rw-r--r-- | src/mailman/styles/default.py | 10 | ||||
| -rw-r--r-- | src/mailman/utilities/datetime.py | 47 |
21 files changed, 345 insertions, 96 deletions
diff --git a/src/mailman/commands/docs/membership.rst b/src/mailman/commands/docs/membership.rst index 638705e91..3faccfe6a 100644 --- a/src/mailman/commands/docs/membership.rst +++ b/src/mailman/commands/docs/membership.rst @@ -292,8 +292,8 @@ Once Anne has verified her alternative address though, it can be used to unsubscribe her from the list. :: - >>> from datetime import datetime - >>> address.verified_on = datetime.now() + >>> from mailman.utilities.datetime import now + >>> address.verified_on = now() >>> results = Results() >>> print leave.process(mlist, msg, {}, (), results) diff --git a/src/mailman/commands/docs/qfile.rst b/src/mailman/commands/docs/qfile.rst index 74ede1b64..8ec0a3952 100644 --- a/src/mailman/commands/docs/qfile.rst +++ b/src/mailman/commands/docs/qfile.rst @@ -59,7 +59,6 @@ Once we've figured out the file name of the shunted message, we can print it. 'bad': u'yes', 'bar': u'baz', 'foo': 7, - u'received_time': ... u'version': 3} [----- end pickle -----] diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg index 3344e965a..88f378159 100644 --- a/src/mailman/config/schema.cfg +++ b/src/mailman/config/schema.cfg @@ -540,6 +540,22 @@ recipient: archive@archive.example.com # command to call. command: /bin/echo +# When sending the message to the archiver, you have the option of +# "clobbering" the Date: header, specifically to make it more sane. Some +# archivers can't handle dates that are wildly off from reality. This does +# not change the Date: header for any other delivery vector except this +# specific archive. +# +# When the original Date header is clobbered, it will always be stored in +# X-Original-Date. The new Date header will always be set to the date at +# which the messages was received by the Mailman server, in UTC. +# +# Your options here are: +# * never -- Leaves the original Date header alone. +# * always -- Always override the Date header. +# * maybe -- Override the Date only if it is outside the clobber_skew period. +clobber_date: maybe +clobber_skew: 1d [archiver.mhonarc] # This is the stock MHonArc archiver. diff --git a/src/mailman/core/switchboard.py b/src/mailman/core/switchboard.py index ba8b2ec3f..7cab4f4ad 100644 --- a/src/mailman/core/switchboard.py +++ b/src/mailman/core/switchboard.py @@ -137,8 +137,7 @@ class Switchboard: # file name consists of two parts separated by a '+': the received # time for this message (i.e. when it first showed up on this system) # and the sha hex digest. - rcvtime = data.setdefault('received_time', now) - filebase = repr(rcvtime) + '+' + hashlib.sha1(hashfood).hexdigest() + filebase = repr(now) + '+' + hashlib.sha1(hashfood).hexdigest() filename = os.path.join(self.queue_directory, filebase + '.pck') tmpfile = filename + '.tmp' # Always add the metadata schema version number diff --git a/src/mailman/docs/NEWS.rst b/src/mailman/docs/NEWS.rst index d3ed65876..b8417472b 100644 --- a/src/mailman/docs/NEWS.rst +++ b/src/mailman/docs/NEWS.rst @@ -8,6 +8,29 @@ Copyright (C) 1998-2012 by the Free Software Foundation, Inc. Here is a history of user visible changes to Mailman. +3.0 beta 1 -- "Freeze" +====================== +(20XX-XX-XX) + +Architecture +------------ + * Internally, all datetimes are kept in the UTC timezone, however because of + LP: #280708, they are stored in the database in naive format. + * `received_time` is now added to the message metadata by the LMTP runner + instead of by `Switchboard.enqueue()`. This latter no longer depends on + `received_time` in the metadata. + * The `ArchiveRunner` no longer acquires a lock before it calls the + individual archiver implementations, since not all of them need a lock. If + they do, the implementations must acquire said lock themselves. + +Configuration +------------- + * New configuration variables `clobber_date` and `clobber_skew` supported in + every `[archiver.<name>]` section. These are used to determine under what + circumstances a message destined for a specific archiver should have its + `Date:` header clobbered. + + 3.0 beta 1 -- "The Twilight Zone" ================================= (2012-03-23) diff --git a/src/mailman/handlers/after_delivery.py b/src/mailman/handlers/after_delivery.py index 46007092b..a964804b5 100644 --- a/src/mailman/handlers/after_delivery.py +++ b/src/mailman/handlers/after_delivery.py @@ -25,12 +25,11 @@ __all__ = [ ] -import datetime - from zope.interface import implements from mailman.core.i18n import _ from mailman.interfaces.handler import IHandler +from mailman.utilities.datetime import now @@ -44,5 +43,5 @@ class AfterDelivery: def process(self, mlist, msg, msgdata): """See `IHander`.""" - mlist.last_post_time = datetime.datetime.now() + mlist.last_post_time = now() mlist.post_id += 1 diff --git a/src/mailman/handlers/docs/after-delivery.rst b/src/mailman/handlers/docs/after-delivery.rst index c3e393cf2..b65b0e77b 100644 --- a/src/mailman/handlers/docs/after-delivery.rst +++ b/src/mailman/handlers/docs/after-delivery.rst @@ -6,9 +6,10 @@ After a message is delivered, or more correctly, after it has been processed by the rest of the handlers in the incoming queue pipeline, a couple of bookkeeping pieces of information are updated. - >>> import datetime + >>> from datetime import timedelta + >>> from mailman.utilities.datetime import now >>> mlist = create_list('_xtest@example.com') - >>> post_time = datetime.datetime.now() - datetime.timedelta(minutes=10) + >>> post_time = now() - timedelta(minutes=10) >>> mlist.last_post_time = post_time >>> mlist.post_id = 10 diff --git a/src/mailman/handlers/docs/replybot.rst b/src/mailman/handlers/docs/replybot.rst index 7cdd7c928..2793e4f75 100644 --- a/src/mailman/handlers/docs/replybot.rst +++ b/src/mailman/handlers/docs/replybot.rst @@ -21,11 +21,11 @@ automatic response grace period which specifies how much time must pass before a second response will be sent, with 0 meaning "there is no grace period". :: - >>> import datetime + >>> from datetime import timedelta >>> from mailman.interfaces.autorespond import ResponseAction >>> mlist.autorespond_owner = ResponseAction.respond_and_continue - >>> mlist.autoresponse_grace_period = datetime.timedelta() + >>> mlist.autoresponse_grace_period = timedelta() >>> mlist.autoresponse_owner_text = 'owner autoresponse text' >>> msg = message_from_string("""\ @@ -242,7 +242,7 @@ Automatic responses have a grace period, during which no additional responses will be sent. This is so as not to bombard the sender with responses. The grace period is measured in days. - >>> mlist.autoresponse_grace_period = datetime.timedelta(days=10) + >>> mlist.autoresponse_grace_period = timedelta(days=10) When a response is sent to a person via any of the owner, request, or postings addresses, the response date is recorded. The grace period is usually diff --git a/src/mailman/handlers/to_digest.py b/src/mailman/handlers/to_digest.py index 698f16e1e..71511f136 100644 --- a/src/mailman/handlers/to_digest.py +++ b/src/mailman/handlers/to_digest.py @@ -26,7 +26,6 @@ __all__ = [ import os -import datetime from zope.interface import implements @@ -35,6 +34,7 @@ from mailman.core.i18n import _ from mailman.email.message import Message from mailman.interfaces.digests import DigestFrequency from mailman.interfaces.handler import IHandler +from mailman.utilities.datetime import now as right_now from mailman.utilities.mailbox import Mailbox @@ -85,7 +85,7 @@ class ToDigest: def bump_digest_number_and_volume(mlist): """Bump the digest number and volume.""" - now = datetime.datetime.now() + now = right_now() if mlist.digest_last_sent_at is None: # There has been no previous digest. bump = False diff --git a/src/mailman/interfaces/archiver.py b/src/mailman/interfaces/archiver.py index a06bbdede..f3edc7719 100644 --- a/src/mailman/interfaces/archiver.py +++ b/src/mailman/interfaces/archiver.py @@ -21,14 +21,23 @@ from __future__ import absolute_import, unicode_literals __metaclass__ = type __all__ = [ + 'ClobberDate', 'IArchiver', ] +from flufl.enum import Enum from zope.interface import Interface, Attribute +class ClobberDate(Enum): + never = 1 + maybe = 2 + always = 3 + + + class IArchiver(Interface): """An interface to the archiver.""" diff --git a/src/mailman/model/docs/registration.rst b/src/mailman/model/docs/registration.rst index 112864f21..eecb3a8cd 100644 --- a/src/mailman/model/docs/registration.rst +++ b/src/mailman/model/docs/registration.rst @@ -275,12 +275,13 @@ different except that the new address will still need to be verified before it can be used. :: + >>> from mailman.utilities.datetime import now >>> dperson = user_manager.create_user( ... 'dperson@example.com', 'Dave Person') >>> dperson <User "Dave Person" (...) at ...> >>> address = user_manager.get_address('dperson@example.com') - >>> address.verified_on = datetime.now() + >>> address.verified_on = now() >>> from operator import attrgetter >>> dump_list(repr(address) for address in dperson.addresses) diff --git a/src/mailman/model/docs/requests.rst b/src/mailman/model/docs/requests.rst index 068cc84f6..a20823a91 100644 --- a/src/mailman/model/docs/requests.rst +++ b/src/mailman/model/docs/requests.rst @@ -230,8 +230,7 @@ We can also hold a message with some additional metadata. # collisions in the message storage. >>> del msg['message-id'] >>> msgdata = dict(sender='aperson@example.com', - ... approved=True, - ... received_time=123.45) + ... approved=True) >>> id_2 = moderator.hold_message(mlist, msg, msgdata, 'Feeling ornery') >>> requests.get_request(id_2) is not None True diff --git a/src/mailman/model/listmanager.py b/src/mailman/model/listmanager.py index cb56a36b6..0ea87a082 100644 --- a/src/mailman/model/listmanager.py +++ b/src/mailman/model/listmanager.py @@ -25,8 +25,6 @@ __all__ = [ ] -import datetime - from zope.event import notify from zope.interface import implements @@ -36,6 +34,7 @@ from mailman.interfaces.listmanager import ( IListManager, ListAlreadyExistsError, ListCreatedEvent, ListCreatingEvent, ListDeletedEvent, ListDeletingEvent) from mailman.model.mailinglist import MailingList +from mailman.utilities.datetime import now @@ -57,7 +56,7 @@ class ListManager: if mlist: raise ListAlreadyExistsError(fqdn_listname) mlist = MailingList(fqdn_listname) - mlist.created_at = datetime.datetime.now() + mlist.created_at = now() config.db.store.add(mlist) notify(ListCreatedEvent(mlist)) return mlist diff --git a/src/mailman/model/pending.py b/src/mailman/model/pending.py index 72746295b..557361c6f 100644 --- a/src/mailman/model/pending.py +++ b/src/mailman/model/pending.py @@ -29,7 +29,6 @@ __all__ = [ import time import random import hashlib -import datetime from lazr.config import as_timedelta from storm.locals import DateTime, Int, RawStr, ReferenceSet, Unicode @@ -40,6 +39,7 @@ from mailman.config import config from mailman.database.model import Model from mailman.interfaces.pending import ( IPendable, IPended, IPendedKeyValue, IPendings) +from mailman.utilities.datetime import now from mailman.utilities.modules import call_name @@ -98,8 +98,8 @@ class Pendings: # does the hash calculation. The integral parts of the time values # are discarded because they're the most predictable bits. for attempts in range(3): - now = time.time() - x = random.random() + now % 1.0 + time.clock() % 1.0 + right_now = time.time() + x = random.random() + right_now % 1.0 + time.clock() % 1.0 # Use sha1 because it produces shorter strings. token = hashlib.sha1(repr(x)).hexdigest() # In practice, we'll never get a duplicate, but we'll be anal @@ -111,7 +111,7 @@ class Pendings: # Create the record, and then the individual key/value pairs. pending = Pended( token=token, - expiration_date=datetime.datetime.now() + lifetime) + expiration_date=now() + lifetime) for key, value in pendable.items(): if isinstance(key, str): key = unicode(key, 'utf-8') @@ -160,9 +160,9 @@ class Pendings: def evict(self): store = config.db.store - now = datetime.datetime.now() + right_now = now() for pending in store.find(Pended): - if pending.expiration_date < now: + if pending.expiration_date < right_now: # Find all PendedKeyValue entries that are associated with the # pending object's ID. q = store.find(PendedKeyValue, diff --git a/src/mailman/mta/postfix.py b/src/mailman/mta/postfix.py index 14f19635a..32bdb8268 100644 --- a/src/mailman/mta/postfix.py +++ b/src/mailman/mta/postfix.py @@ -27,7 +27,6 @@ __all__ = [ import os import logging -import datetime from flufl.lock import Lock from operator import attrgetter @@ -38,6 +37,7 @@ from mailman.config import config from mailman.interfaces.listmanager import IListManager from mailman.interfaces.mta import ( IMailTransportAgentAliases, IMailTransportAgentLifecycle) +from mailman.utilities.datetime import now log = logging.getLogger('mailman.error') @@ -124,7 +124,7 @@ class LMTP: # file. YOU SHOULD NOT MANUALLY EDIT THIS FILE unless you know what you're # doing, and can keep the two files properly in sync. If you screw it up, # you're on your own. -""".format(datetime.datetime.now().replace(microsecond=0)) +""".format(now().replace(microsecond=0)) sort_key = attrgetter('list_name') for domain in sorted(by_domain): print >> fp, """\ diff --git a/src/mailman/runners/archive.py b/src/mailman/runners/archive.py index 1c0a24785..7295a5c57 100644 --- a/src/mailman/runners/archive.py +++ b/src/mailman/runners/archive.py @@ -25,68 +25,83 @@ __all__ = [ ] -import os +import copy import logging +from email.utils import parsedate_tz, mktime_tz from datetime import datetime -from email.utils import parsedate_tz, mktime_tz, formatdate -from flufl.lock import Lock from lazr.config import as_timedelta from mailman.config import config from mailman.core.runner import Runner +from mailman.interfaces.archiver import ClobberDate +from mailman.utilities.datetime import RFC822_DATE_FMT, now + log = logging.getLogger('mailman.error') +def _should_clobber(msg, msgdata, archiver): + """Should the Date header in the original message get clobbered?""" + # Calculate the Date header of the message as a datetime. What if there + # are multiple Date headers, even in violation of the RFC? For now, take + # the first one. If there are no Date headers, then definitely clobber. + original_date = msg.get('date') + if original_date is None: + return True + section = getattr(config.archiver, archiver, None) + if section is None: + log.error('No archiver config section found: {0}'.format(archiver)) + return False + try: + clobber = ClobberDate[section.clobber_date] + except ValueError: + log.error('Invalid clobber_date for "{0}": {1}'.format( + archiver, section.clobber_date)) + return False + if clobber is ClobberDate.always: + return True + elif clobber is ClobberDate.never: + return False + # Maybe we'll clobber the date. Let's see if it's farther off from now + # than the skew period. + skew = as_timedelta(section.clobber_skew) + try: + time_tuple = parsedate_tz(original_date) + except (ValueError, OverflowError): + # The likely cause of this is that the year in the Date: field is + # horribly incorrect, e.g. (from SF bug # 571634): + # + # Date: Tue, 18 Jun 0102 05:12:09 +0500 + # + # Obviously clobber such dates. + return True + if time_tuple is None: + # There was some other bogosity in the Date header. + return True + claimed_date = datetime.fromtimestamp(mktime_tz(time_tuple)) + return (abs(now() - claimed_date) > skew) + + + class ArchiveRunner(Runner): """The archive runner.""" def _dispose(self, mlist, msg, msgdata): - # Support clobber_date, i.e. setting the date in the archive to the - # received date, not the (potentially bogus) Date: header of the - # original message. - clobber = False - original_date = msg.get('date') - received_time = formatdate(msgdata['received_time']) - # FIXME 2012-03-23 BAW: LP: #963612 - ## if not original_date: - ## clobber = True - ## elif int(config.archiver.pipermail.clobber_date_policy) == 1: - ## clobber = True - ## elif int(config.archiver.pipermail.clobber_date_policy) == 2: - ## # What's the timestamp on the original message? - ## timetup = parsedate_tz(original_date) - ## now = datetime.now() - ## try: - ## if not timetup: - ## clobber = True - ## else: - ## utc_timestamp = datetime.fromtimestamp(mktime_tz(timetup)) - ## date_skew = as_timedelta( - ## config.archiver.pipermail.allowable_sane_date_skew) - ## clobber = (abs(now - utc_timestamp) > date_skew) - ## except (ValueError, OverflowError): - ## # The likely cause of this is that the year in the Date: field - ## # is horribly incorrect, e.g. (from SF bug # 571634): - ## # Date: Tue, 18 Jun 0102 05:12:09 +0500 - ## # Obviously clobber such dates. - ## clobber = True - ## if clobber: - ## del msg['date'] - ## del msg['x-original-date'] - ## msg['Date'] = received_time - ## if original_date: - ## msg['X-Original-Date'] = original_date - # Always put an indication of when we received the message. - msg['X-List-Received-Date'] = received_time - # While a list archiving lock is acquired, archive the message. - with Lock(os.path.join(mlist.data_path, 'archive.lck')): - for archiver in config.archivers: - # A problem in one archiver should not prevent other archivers - # from running. - try: - archiver.archive_message(mlist, msg) - except Exception: - log.exception('Broken archiver: %s' % archiver.name) + received_time = msgdata.get('received_time', now(strip_tzinfo=False)) + for archiver in config.archivers: + msg_copy = copy.deepcopy(msg) + if _should_clobber(msg, msgdata, archiver.name): + original_date = msg_copy['date'] + del msg_copy['date'] + del msg_copy['x-original-date'] + msg_copy['Date'] = received_time.strftime(RFC822_DATE_FMT) + if original_date: + msg_copy['X-Original-Date'] = original_date + # A problem in one archiver should not prevent other archivers + # from running. + try: + archiver.archive_message(mlist, msg_copy) + except Exception: + log.exception('Broken archiver: %s' % archiver.name) diff --git a/src/mailman/runners/lmtp.py b/src/mailman/runners/lmtp.py index bee111ad1..45fa5a783 100644 --- a/src/mailman/runners/lmtp.py +++ b/src/mailman/runners/lmtp.py @@ -44,8 +44,10 @@ from mailman.core.runner import Runner from mailman.database.transaction import txn from mailman.email.message import Message from mailman.interfaces.listmanager import IListManager +from mailman.utilities.datetime import now from mailman.utilities.email import add_message_hash + elog = logging.getLogger('mailman.error') qlog = logging.getLogger('mailman.runner') slog = logging.getLogger('mailman.smtp') @@ -181,6 +183,7 @@ class LMTPRunner(Runner, smtpd.SMTPServer): # see if it's destined for a valid mailing list. If so, then queue # the message to the appropriate place and record a 250 status for # that recipient. If not, record a failure status for that recipient. + received_time = now() for to in rcpttos: try: to = parseaddr(to)[1].lower() @@ -196,7 +199,8 @@ class LMTPRunner(Runner, smtpd.SMTPServer): # queue. queue = None msgdata = dict(listname=listname, - original_size=msg.original_size) + original_size=msg.original_size, + received_time=received_time) canonical_subaddress = SUBADDRESS_NAMES.get(subaddress) queue = SUBADDRESS_QUEUES.get(canonical_subaddress) if subaddress is None: diff --git a/src/mailman/runners/tests/test_archiver.py b/src/mailman/runners/tests/test_archiver.py index 865a2be67..ca09de9fa 100644 --- a/src/mailman/runners/tests/test_archiver.py +++ b/src/mailman/runners/tests/test_archiver.py @@ -39,6 +39,31 @@ from mailman.testing.helpers import ( make_testable_runner, specialized_message_from_string as mfs) from mailman.testing.layers import ConfigLayer +from mailman.utilities.datetime import RFC822_DATE_FMT, factory, now + + + +# This helper will set up a specific archiver as appropriate for a specific +# test. It assumes the setUp() will just disable all archivers. +def archiver(name, enable=False, clobber=None, skew=None): + def decorator(func): + def wrapper(*args, **kws): + config_name = 'archiver {0}'.format(name) + section = """ + [archiver.{0}] + enable: {1} + clobber_date: {2} + clobber_skew: {3} + """.format(name, + 'yes' if enable else 'no', + clobber, skew) + config.push(config_name, section) + try: + return func(*args, **kws) + finally: + config.pop(config_name) + return wrapper + return decorator @@ -54,7 +79,7 @@ class DummyArchiver: def permalink(mlist, msg): filename = msg['x-message-id-hash'] return 'http://archive.example.com/' + filename - + @staticmethod def archive_message(mlist, msg): filename = msg['x-message-id-hash'] @@ -73,11 +98,12 @@ class TestArchiveRunner(unittest.TestCase): def setUp(self): self._mlist = create_list('test@example.com') + self._now = now() # Enable just the dummy archiver. config.push('dummy', """ [archiver.dummy] class: mailman.runners.tests.test_archiver.DummyArchiver - enable: yes + enable: no [archiver.prototype] enable: no [archiver.mhonarc] @@ -100,10 +126,13 @@ First post! def tearDown(self): config.pop('dummy') + @archiver('dummy', enable=True) def test_archive_runner(self): # Ensure that the archive runner ends up archiving the message. self._archiveq.enqueue( - self._msg, {}, listname=self._mlist.fqdn_listname) + self._msg, {}, + listname=self._mlist.fqdn_listname, + received_time=now()) self._runner.run() # There should now be a copy of the message in the file system. filename = os.path.join( @@ -112,11 +141,114 @@ First post! archived = message_from_file(fp) self.assertEqual(archived['message-id'], '<first>') + @archiver('dummy', enable=True) def test_archive_runner_with_dated_message(self): - # LP: #963612 FIXME - self._msg['Date'] = 'Sat, 11 Mar 2011 03:19:38 -0500' + # Date headers don't throw off the archiver runner. + self._msg['Date'] = now(strip_tzinfo=False).strftime(RFC822_DATE_FMT) + self._archiveq.enqueue( + self._msg, {}, + listname=self._mlist.fqdn_listname, + received_time=now()) + self._runner.run() + # There should now be a copy of the message in the file system. + filename = os.path.join( + config.MESSAGES_DIR, '4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB') + with open(filename) as fp: + archived = message_from_file(fp) + self.assertEqual(archived['message-id'], '<first>') + self.assertEqual(archived['date'], 'Mon, 01 Aug 2005 07:49:23 +0000') + + @archiver('dummy', enable=True, clobber='never') + def test_clobber_date_never(self): + # Even if the Date header is insanely off from the received time of + # the message, if clobber_date is 'never', the header is not clobbered. + self._msg['Date'] = now(strip_tzinfo=False).strftime(RFC822_DATE_FMT) + self._archiveq.enqueue( + self._msg, {}, + listname=self._mlist.fqdn_listname, + received_time=now()) + self._runner.run() + # There should now be a copy of the message in the file system. + filename = os.path.join( + config.MESSAGES_DIR, '4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB') + with open(filename) as fp: + archived = message_from_file(fp) + self.assertEqual(archived['message-id'], '<first>') + self.assertEqual(archived['date'], 'Mon, 01 Aug 2005 07:49:23 +0000') + + @archiver('dummy', enable=True) + def test_clobber_dateless(self): + # A message with no Date header will always get clobbered. + self.assertEqual(self._msg['date'], None) + # Now, before enqueuing the message (well, really, calling 'now()' + # again), fast forward a few days. + self._archiveq.enqueue( + self._msg, {}, + listname=self._mlist.fqdn_listname, + received_time=now(strip_tzinfo=False)) + self._runner.run() + # There should now be a copy of the message in the file system. + filename = os.path.join( + config.MESSAGES_DIR, '4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB') + with open(filename) as fp: + archived = message_from_file(fp) + self.assertEqual(archived['message-id'], '<first>') + self.assertEqual(archived['date'], 'Mon, 01 Aug 2005 07:49:23 +0000') + + @archiver('dummy', enable=True, clobber='always') + def test_clobber_date_always(self): + # The date always gets clobbered with the current received time. + self._msg['Date'] = now(strip_tzinfo=False).strftime(RFC822_DATE_FMT) + # Now, before enqueuing the message (well, really, calling 'now()' + # again as will happen in the runner), fast forward a few days. + self._archiveq.enqueue( + self._msg, {}, + listname=self._mlist.fqdn_listname) + factory.fast_forward(days=4) + self._runner.run() + # There should now be a copy of the message in the file system. + filename = os.path.join( + config.MESSAGES_DIR, '4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB') + with open(filename) as fp: + archived = message_from_file(fp) + self.assertEqual(archived['message-id'], '<first>') + self.assertEqual(archived['date'], 'Fri, 05 Aug 2005 07:49:23 +0000') + self.assertEqual(archived['x-original-date'], + 'Mon, 01 Aug 2005 07:49:23 +0000') + + @archiver('dummy', enable=True, clobber='maybe', skew='1d') + def test_clobber_date_maybe_when_insane(self): + # The date is clobbered if it's farther off from now than its skew + # period. + self._msg['Date'] = now(strip_tzinfo=False).strftime(RFC822_DATE_FMT) + # Now, before enqueuing the message (well, really, calling 'now()' + # again as will happen in the runner), fast forward a few days. + self._archiveq.enqueue( + self._msg, {}, + listname=self._mlist.fqdn_listname) + factory.fast_forward(days=4) + self._runner.run() + # There should now be a copy of the message in the file system. + filename = os.path.join( + config.MESSAGES_DIR, '4CMWUN6BHVCMHMDAOSJZ2Q72G5M32MWB') + with open(filename) as fp: + archived = message_from_file(fp) + self.assertEqual(archived['message-id'], '<first>') + self.assertEqual(archived['date'], 'Fri, 05 Aug 2005 07:49:23 +0000') + self.assertEqual(archived['x-original-date'], + 'Mon, 01 Aug 2005 07:49:23 +0000') + + @archiver('dummy', enable=True, clobber='maybe', skew='10d') + def test_clobber_date_maybe_when_sane(self): + # The date is not clobbered if it's nearer to now than its skew + # period. + self._msg['Date'] = now(strip_tzinfo=False).strftime(RFC822_DATE_FMT) + # Now, before enqueuing the message (well, really, calling 'now()' + # again as will happen in the runner), fast forward a few days. self._archiveq.enqueue( - self._msg, {}, listname=self._mlist.fqdn_listname) + self._msg, {}, + listname=self._mlist.fqdn_listname) + factory.fast_forward(days=4) self._runner.run() # There should now be a copy of the message in the file system. filename = os.path.join( @@ -124,3 +256,5 @@ First post! with open(filename) as fp: archived = message_from_file(fp) self.assertEqual(archived['message-id'], '<first>') + self.assertEqual(archived['date'], 'Mon, 01 Aug 2005 07:49:23 +0000') + self.assertEqual(archived['x-original-date'], None) diff --git a/src/mailman/runners/tests/test_lmtp.py b/src/mailman/runners/tests/test_lmtp.py index 2c4defe59..87b69c7e4 100644 --- a/src/mailman/runners/tests/test_lmtp.py +++ b/src/mailman/runners/tests/test_lmtp.py @@ -28,6 +28,8 @@ __all__ = [ import smtplib import unittest +from datetime import datetime + from mailman.app.lifecycle import create_list from mailman.config import config from mailman.testing.helpers import get_lmtp_client, get_queue_messages @@ -96,3 +98,17 @@ Subject: This has a Message-ID but no X-Message-ID-Hash self.assertEqual(len(all_headers), 1) self.assertEqual(messages[0].msg['x-message-id-hash'], 'MS6QLWERIJLGCRF44J7USBFDELMNT2BW') + + def test_received_time(self): + # The LMTP runner adds a `received_time` key to the metadata. + self._lmtp.sendmail('anne@example.com', ['test@example.com'], """\ +From: anne@example.com +To: test@example.com +Subject: This has no Message-ID header +Message-ID: <ant> + +""") + messages = get_queue_messages('in') + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0].msgdata['received_time'], + datetime(2005, 8, 1, 7, 49, 23)) diff --git a/src/mailman/styles/default.py b/src/mailman/styles/default.py index 18a145d3c..b6900dca6 100644 --- a/src/mailman/styles/default.py +++ b/src/mailman/styles/default.py @@ -27,8 +27,7 @@ __all__ = [ # XXX Styles need to be reconciled with lazr.config. -import datetime - +from datetime import timedelta from zope.interface import implements from mailman.core.i18n import _ @@ -153,16 +152,15 @@ from: .*@uplinkpro.com mlist.autoresponse_postings_text = '' mlist.autorespond_requests = ResponseAction.none mlist.autoresponse_request_text = '' - mlist.autoresponse_grace_period = datetime.timedelta(days=90) + mlist.autoresponse_grace_period = timedelta(days=90) # Bounces mlist.forward_unrecognized_bounces_to = ( UnrecognizedBounceDisposition.administrators) mlist.process_bounces = True mlist.bounce_score_threshold = 5.0 - mlist.bounce_info_stale_after = datetime.timedelta(days=7) + mlist.bounce_info_stale_after = timedelta(days=7) mlist.bounce_you_are_disabled_warnings = 3 - mlist.bounce_you_are_disabled_warnings_interval = ( - datetime.timedelta(days=7)) + mlist.bounce_you_are_disabled_warnings_interval = timedelta(days=7) mlist.bounce_notify_owner_on_disable = True mlist.bounce_notify_owner_on_removal = True # This holds legacy member related information. It's keyed by the diff --git a/src/mailman/utilities/datetime.py b/src/mailman/utilities/datetime.py index 96d14fce5..3f451efaf 100644 --- a/src/mailman/utilities/datetime.py +++ b/src/mailman/utilities/datetime.py @@ -28,9 +28,12 @@ from __future__ import absolute_import, unicode_literals __metaclass__ = type __all__ = [ 'DateFactory', + 'RFC822_DATE_FMT', + 'UTC', 'factory', 'now', 'today', + 'utc', ] @@ -39,6 +42,30 @@ import datetime from mailman.testing import layers +# Python always sets the locale to 'C' locale unless the user explicitly calls +# locale.setlocale(locale.LC_ALL, ''). Since we never do this in Mailman (and +# no library better do it either!) this will safely give us expected RFC 5322 +# Date headers. +RFC822_DATE_FMT = '%a, %d %b %Y %H:%M:%S %z' + + + +# Definition of UTC timezone, taken from +# http://docs.python.org/library/datetime.html +ZERO = datetime.timedelta(0) + +class UTC(datetime.tzinfo): + def utcoffset(self, dt): + return ZERO + def tzname(self, dt): + return 'UTC' + def dst(self, dt): + return ZERO + +utc = UTC() +_missing = object() + + class DateFactory: """A factory for today() and now() that works with testing.""" @@ -47,12 +74,21 @@ class DateFactory: predictable_now = None predictable_today = None - def now(self, tz=None): + def now(self, tz=_missing, strip_tzinfo=True): # We can't automatically fast-forward because some tests require us to # stay on the same day for a while, e.g. autorespond.txt. - return (self.predictable_now - if layers.is_testing() - else datetime.datetime.now(tz)) + if tz is _missing: + tz = utc + # Storm cannot yet handle datetimes with tz suffixes. Assume we're + # using UTC datetimes everywhere, so set the tzinfo to None. This + # does *not* change the actual time values. LP: #280708 + tz_now = (self.predictable_now + if layers.is_testing() + else datetime.datetime.now(tz)) + return (tz_now.replace(tzinfo=None) + if strip_tzinfo + else tz_now) + def today(self): return (self.predictable_today @@ -61,7 +97,8 @@ class DateFactory: @classmethod def reset(cls): - cls.predictable_now = datetime.datetime(2005, 8, 1, 7, 49, 23) + cls.predictable_now = datetime.datetime(2005, 8, 1, 7, 49, 23, + tzinfo=utc) cls.predictable_today = cls.predictable_now.date() @classmethod |