diff options
| -rw-r--r-- | src/mailman/archiving/docs/common.rst | 9 | ||||
| -rw-r--r-- | src/mailman/archiving/prototype.py | 59 | ||||
| -rw-r--r-- | src/mailman/archiving/tests/__init__.py | 0 | ||||
| -rw-r--r-- | src/mailman/archiving/tests/test_prototype.py | 171 | ||||
| -rw-r--r-- | src/mailman/commands/docs/info.rst | 1 | ||||
| -rw-r--r-- | src/mailman/config/config.py | 1 | ||||
| -rw-r--r-- | src/mailman/config/schema.cfg | 3 | ||||
| -rw-r--r-- | src/mailman/docs/NEWS.rst | 2 |
8 files changed, 240 insertions, 6 deletions
diff --git a/src/mailman/archiving/docs/common.rst b/src/mailman/archiving/docs/common.rst index 45ec8f194..5f7cfe42b 100644 --- a/src/mailman/archiving/docs/common.rst +++ b/src/mailman/archiving/docs/common.rst @@ -62,12 +62,13 @@ The archiver is also able to archive the message. >>> os.path.exists(pckpath) True -Note however that the prototype archiver can't archive messages. +The `prototype` archiver archives messages to a maildir. >>> archivers['prototype'].archive_message(mlist, msg) - Traceback (most recent call last): - ... - NotImplementedError + >>> archive_path = os.path.join( + ... config.ARCHIVE_DIR, 'prototype', mlist.fqdn_listname, 'new') + >>> len(os.listdir(archive_path)) + 1 The Mail-Archive.com diff --git a/src/mailman/archiving/prototype.py b/src/mailman/archiving/prototype.py index 55d78074e..453c6c770 100644 --- a/src/mailman/archiving/prototype.py +++ b/src/mailman/archiving/prototype.py @@ -25,11 +25,22 @@ __all__ = [ ] +import os +import errno +import logging + +from datetime import timedelta +from mailbox import Maildir from urlparse import urljoin + +from flufl.lock import Lock, TimeOutError from zope.interface import implements +from mailman.config import config from mailman.interfaces.archiver import IArchiver +log = logging.getLogger('mailman.error') + class Prototype: @@ -61,5 +72,49 @@ class Prototype: @staticmethod def archive_message(mlist, message): - """See `IArchiver`.""" - raise NotImplementedError + """See `IArchiver`. + + This archiver saves messages into a maildir. + """ + archive_dir = os.path.join(config.ARCHIVE_DIR, 'prototype') + try: + os.makedirs(archive_dir, 0775) + except OSError as error: + # If this already exists, then we're fine + if error.errno != errno.EEXIST: + raise + + # Maildir will throw an error if the directories are partially created + # (for instance the toplevel exists but cur, new, or tmp do not) + # therefore we don't create the toplevel as we did above. + list_dir = os.path.join(archive_dir, mlist.fqdn_listname) + mailbox = Maildir(list_dir, create=True, factory=None) + lock_file = os.path.join( + config.LOCK_DIR, '{0}-maildir.lock'.format(mlist.fqdn_listname)) + + # Lock the maildir as Maildir.add() is not threadsafe. Don't use the + # context manager because it's not an error if we can't acquire the + # archiver lock. We'll just log the problem and continue. + # + # XXX 2012-03-14 BAW: When we extend the chain/pipeline architecture + # to other runners, e.g. the archive runner, it would be better to let + # any TimeOutError propagate up. That would cause the message to be + # re-queued and tried again later, rather than being discarded as + # happens now below. + lock = Lock(lock_file) + try: + lock.lock(timeout=timedelta(seconds=1)) + # Add the message to the maildir. The return value could be used + # to construct the file path if necessary. E.g. + # + # os.path.join(archive_dir, mlist.fqdn_listname, 'new', + # message_key) + mailbox.add(message) + except TimeOutError: + # Log the error and go on. + log.error('Unable to acquire prototype archiver lock for {0}, ' + 'discarding: {1}'.format( + mlist.fqdn_listname, + message.get('message-id', 'n/a'))) + finally: + lock.unlock(unconditionally=True) diff --git a/src/mailman/archiving/tests/__init__.py b/src/mailman/archiving/tests/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/mailman/archiving/tests/__init__.py diff --git a/src/mailman/archiving/tests/test_prototype.py b/src/mailman/archiving/tests/test_prototype.py new file mode 100644 index 000000000..9c229db1c --- /dev/null +++ b/src/mailman/archiving/tests/test_prototype.py @@ -0,0 +1,171 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test the prototype archiver.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestPrototypeArchiver', + ] + + +import os +import shutil +import tempfile +import unittest +import threading + +from email import message_from_file +from flufl.lock import Lock + +from mailman.app.lifecycle import create_list +from mailman.archiving.prototype import Prototype +from mailman.config import config +from mailman.testing.helpers import LogFileMark +from mailman.testing.helpers import ( + specialized_message_from_string as mfs) +from mailman.testing.layers import ConfigLayer +from mailman.utilities.email import add_message_hash + + +class TestPrototypeArchiver(unittest.TestCase): + """Test the prototype archiver.""" + + layer = ConfigLayer + + def setUp(self): + # Create a fake mailing list and message object + self._msg = mfs("""\ +To: test@example.com +From: anne@example.com +Subject: Testing the test list +Message-ID: <ant> +X-Message-ID-Hash: MS6QLWERIJLGCRF44J7USBFDELMNT2BW + +Tests are better than no tests +but the water deserves to be swum. +""") + self._mlist = create_list('test@example.com') + config.db.commit() + # Set up a temporary directory for the prototype archiver so that it's + # easier to clean up. + self._tempdir = tempfile.mkdtemp() + config.push('prototype', """ + [paths.testing] + archive_dir: {0} + """.format(self._tempdir)) + # Capture the structure of a maildir. + self._expected_dir_structure = set( + (os.path.join(config.ARCHIVE_DIR, path) for path in ( + 'prototype', + os.path.join('prototype', self._mlist.fqdn_listname), + os.path.join('prototype', self._mlist.fqdn_listname, 'cur'), + os.path.join('prototype', self._mlist.fqdn_listname, 'new'), + os.path.join('prototype', self._mlist.fqdn_listname, 'tmp'), + ))) + self._expected_dir_structure.add(config.ARCHIVE_DIR) + + def tearDown(self): + shutil.rmtree(self._tempdir) + config.pop('prototype') + + def _find(self, path): + all_filenames = set() + for dirpath, dirnames, filenames in os.walk(path): + if not isinstance(dirpath, unicode): + dirpath = unicode(dirpath) + all_filenames.add(dirpath) + for filename in filenames: + new_filename = os.path.join(dirpath, filename) + if not isinstance(new_filename, unicode): + new_filename = unicode(new_filename) + all_filenames.add(new_filename) + return all_filenames + + def test_archive_maildir_created(self): + # Archiving a message to the prototype archiver should create the + # expected directory structure. + Prototype.archive_message(self._mlist, self._msg) + all_filenames = self._find(config.ARCHIVE_DIR) + # Check that the directory structure has been created and we have one + # more file (the archived message) than expected directories. + archived_messages = all_filenames - self._expected_dir_structure + self.assertEqual(self._expected_dir_structure, + all_filenames - archived_messages) + + def test_archive_maildir_existence_does_not_raise(self): + # Archiving a second message does not cause an EEXIST to be raised + # when a second message is archived. + new_dir = None + Prototype.archive_message(self._mlist, self._msg) + for directory in ('cur', 'new', 'tmp'): + path = os.path.join(config.ARCHIVE_DIR, 'prototype', + self._mlist.fqdn_listname, directory) + if directory == 'new': + new_dir = path + self.assertTrue(os.path.isdir(path)) + # There should be one message in the 'new' directory. + self.assertEqual(len(os.listdir(new_dir)), 1) + # Archive a second message. If an exception occurs, let it fail the + # test. Afterward, two messages should be in the 'new' directory. + del self._msg['message-id'] + del self._msg['x-message-id-hash'] + self._msg['Message-ID'] = '<bee>' + add_message_hash(self._msg) + Prototype.archive_message(self._mlist, self._msg) + self.assertEqual(len(os.listdir(new_dir)), 2) + + def test_archive_lock_used(self): + # Test that locking the maildir when adding works as a failure here + # could mean we lose mail. + lock_file = os.path.join( + config.LOCK_DIR, '{0}-maildir.lock'.format( + self._mlist.fqdn_listname)) + with Lock(lock_file): + # Acquire the archiver lock, Hen make sure the archiver logs the + # fact that it could not acquire the lock. + archive_thread = threading.Thread( + target=Prototype.archive_message, + args=(self._mlist, self._msg)) + mark = LogFileMark('mailman.error') + archive_thread.run() + # Test that the archiver output the correct error. + line = mark.readline() + self.assertEqual( + # Strip out the timestamp. + line[28:-1], + 'Unable to acquire prototype archiver lock for {0}, ' + 'discarding: {1}'.format( + self._mlist.fqdn_listname, + self._msg.get('message-id'))) + # Check that the message didn't get archived. + created_files = self._find(config.ARCHIVE_DIR) + self.assertEqual(self._expected_dir_structure, created_files) + + def test_prototype_archiver_good_path(self): + # Verify the good path; the message gets archived. + Prototype.archive_message(self._mlist, self._msg) + new_path = os.path.join( + config.ARCHIVE_DIR, 'prototype', self._mlist.fqdn_listname, 'new') + archived_messages = list(os.listdir(new_path)) + self.assertEqual(len(archived_messages), 1) + # Check that the email has been added. + with open(os.path.join(new_path, archived_messages[0])) as fp: + archived_message = message_from_file(fp) + self.assertEqual(self._msg.as_string(), archived_message.as_string()) diff --git a/src/mailman/commands/docs/info.rst b/src/mailman/commands/docs/info.rst index 34883711e..ad034a1a6 100644 --- a/src/mailman/commands/docs/info.rst +++ b/src/mailman/commands/docs/info.rst @@ -59,6 +59,7 @@ The File System Hierarchy layout is the same every by definition. Python ... ... File system paths: + ARCHIVE_DIR = /var/lib/mailman/archives BIN_DIR = /sbin DATA_DIR = /var/lib/mailman/data ETC_DIR = /etc diff --git a/src/mailman/config/config.py b/src/mailman/config/config.py index 034b76b4f..e3b4f88a7 100644 --- a/src/mailman/config/config.py +++ b/src/mailman/config/config.py @@ -173,6 +173,7 @@ class Configuration: lock_dir = category.lock_dir, log_dir = category.log_dir, messages_dir = category.messages_dir, + archive_dir = category.archive_dir, pipermail_private_dir = category.pipermail_private_dir, pipermail_public_dir = category.pipermail_public_dir, queue_dir = category.queue_dir, diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg index e662633e6..7e15ab82f 100644 --- a/src/mailman/config/schema.cfg +++ b/src/mailman/config/schema.cfg @@ -113,6 +113,9 @@ etc_dir: $var_dir/etc ext_dir: $var_dir/ext # Directory where the default IMessageStore puts its messages. messages_dir: $var_dir/messages +# Directory for archive backends to store their messages in. Archivers should +# create a subdirectory in here to store their files. +archive_dir: $var_dir/archives # Directory for public Pipermail archiver artifacts. pipermail_public_dir: $var_dir/archives/public # Directory for private Pipermail archiver artifacts. diff --git a/src/mailman/docs/NEWS.rst b/src/mailman/docs/NEWS.rst index 6869e2889..c2e4d1e51 100644 --- a/src/mailman/docs/NEWS.rst +++ b/src/mailman/docs/NEWS.rst @@ -46,6 +46,8 @@ Architecture attribute on the message object, instead of trusting a possibly incorrect value if it's already set. The individual `IArchiver` implementations no longer set the `X-Message-ID-Hash` header. + * The Prototype archiver now stores its files in maildir format inside of + `$var_dir/archives/prototype`, given by Toshio Kuratomi. Database -------- |