From 902856e597524a2ec6ee6e7d90d6804d9a994cf8 Mon Sep 17 00:00:00 2001 From: toshio Date: Tue, 13 Mar 2012 00:28:57 +0000 Subject: Implement preliminary archiving to maildir in the prototype archiver --- src/mailman/archiving/prototype.py | 44 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/mailman/archiving/prototype.py b/src/mailman/archiving/prototype.py index 0a857fd77..3d50ddfd8 100644 --- a/src/mailman/archiving/prototype.py +++ b/src/mailman/archiving/prototype.py @@ -28,11 +28,20 @@ __all__ = [ import hashlib from base64 import b32encode +from datetime import timedelta +import errno +import logging +from mailbox import Maildir +import os from urlparse import urljoin from zope.interface import implements +from flufl.lock import Lock, TimeOutError + +from mailman.config import config from mailman.interfaces.archiver import IArchiver +elog = logging.getLogger('mailman.error') class Prototype: @@ -72,5 +81,36 @@ class Prototype: @staticmethod def archive_message(mlist, message): - """See `IArchiver`.""" - raise NotImplementedError + """See `IArchiver`. + + This sample archiver saves nmessages into a maildir + """ + archive_dir = os.path.join(config.ARCHIVES_DIR, 'prototype') + try: + os.makedirs(archive_dir, 0775) + except OSError, e: + # If this already exists, then we're fine + if e.errno != errno.EEXIST: + raise + + # Maildir will throw an error if the directories are partially created + # (for instance the toplevel exists but cur, new, or tmp do not) + # therefore we don't create the toplevel as we did above + list_dir = os.path.join(archive_dir, mlist.fqdn_listname) + mail_box = Maildir(list_dir, create=True, factory=None) + + # Lock the maildir as Maildir.add() is not threadsafe + lock = Lock(os.path.join(config.LOCK_DIR, '%s-maildir.lock' % mlist.fqdn_listname)) + try: + lock.lock(timeout=timedelta(seconds=1)) + # Add the message to the Maildir + # Message_key could be used to construct the file path if + # necessary: + # os.path.join(archive_dir,mlist.fqdn_listname,'new',message_key) + message_key = mail_box.add(message) + except TimeOutError: + # log the error and go on + elog.error('Unable to lock archive for %s, discarded message: %s' % (mlist.fqdn_listname, message.get('message-id', ''))) + finally: + # unlock the maildir + lock.unlock(unconditionally=1) -- cgit v1.3.1 From e308754546f28acae22267bdcd5d27d9c32a75ad Mon Sep 17 00:00:00 2001 From: toshio Date: Tue, 13 Mar 2012 05:01:03 +0000 Subject: Fix up the archive_dir in the config file --- src/mailman/archiving/prototype.py | 2 +- src/mailman/config/config.py | 1 + src/mailman/config/schema.cfg | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/mailman/archiving/prototype.py b/src/mailman/archiving/prototype.py index 3d50ddfd8..b26dc9768 100644 --- a/src/mailman/archiving/prototype.py +++ b/src/mailman/archiving/prototype.py @@ -85,7 +85,7 @@ class Prototype: This sample archiver saves nmessages into a maildir """ - archive_dir = os.path.join(config.ARCHIVES_DIR, 'prototype') + archive_dir = os.path.join(config.ARCHIVE_DIR, 'prototype') try: os.makedirs(archive_dir, 0775) except OSError, e: diff --git a/src/mailman/config/config.py b/src/mailman/config/config.py index 034b76b4f..e3b4f88a7 100644 --- a/src/mailman/config/config.py +++ b/src/mailman/config/config.py @@ -173,6 +173,7 @@ class Configuration: lock_dir = category.lock_dir, log_dir = category.log_dir, messages_dir = category.messages_dir, + archive_dir = category.archive_dir, pipermail_private_dir = category.pipermail_private_dir, pipermail_public_dir = category.pipermail_public_dir, queue_dir = category.queue_dir, diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg index cde01cbd7..ebbc50fc8 100644 --- a/src/mailman/config/schema.cfg +++ b/src/mailman/config/schema.cfg @@ -98,6 +98,9 @@ etc_dir: $var_dir/etc ext_dir: $var_dir/ext # Directory where the default IMessageStore puts its messages. messages_dir: $var_dir/messages +# Directory for archive backends to store their archives in. +# Archivers should create a subdirectory in here to store their files +archive_dir: $var_dir/archives # Directory for public Pipermail archiver artifacts. pipermail_public_dir: $var_dir/archives/public # Directory for private Pipermail archiver artifacts. -- cgit v1.3.1 From f4f5226e97e53826ebbe8794e2e7b770e53db5f2 Mon Sep 17 00:00:00 2001 From: toshio Date: Tue, 13 Mar 2012 05:50:53 +0000 Subject: Style cleanups for the prototype patch --- src/mailman/archiving/prototype.py | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) (limited to 'src') diff --git a/src/mailman/archiving/prototype.py b/src/mailman/archiving/prototype.py index b26dc9768..b3490009a 100644 --- a/src/mailman/archiving/prototype.py +++ b/src/mailman/archiving/prototype.py @@ -25,18 +25,18 @@ __all__ = [ ] +import os +import errno import hashlib +import logging from base64 import b32encode from datetime import timedelta -import errno -import logging from mailbox import Maildir -import os from urlparse import urljoin -from zope.interface import implements from flufl.lock import Lock, TimeOutError +from zope.interface import implements from mailman.config import config from mailman.interfaces.archiver import IArchiver @@ -100,17 +100,19 @@ class Prototype: mail_box = Maildir(list_dir, create=True, factory=None) # Lock the maildir as Maildir.add() is not threadsafe - lock = Lock(os.path.join(config.LOCK_DIR, '%s-maildir.lock' % mlist.fqdn_listname)) - try: - lock.lock(timeout=timedelta(seconds=1)) - # Add the message to the Maildir - # Message_key could be used to construct the file path if - # necessary: - # os.path.join(archive_dir,mlist.fqdn_listname,'new',message_key) - message_key = mail_box.add(message) - except TimeOutError: - # log the error and go on - elog.error('Unable to lock archive for %s, discarded message: %s' % (mlist.fqdn_listname, message.get('message-id', ''))) - finally: - # unlock the maildir - lock.unlock(unconditionally=1) + lock = Lock(os.path.join(config.LOCK_DIR, '%s-maildir.lock' + % mlist.fqdn_listname)) + with lock: + try: + lock.lock(timeout=timedelta(seconds=1)) + # Add the message to the Maildir + # Message_key could be used to construct the file path if + # necessary:: + # os.path.join(archive_dir, mlist.fqdn_listname, 'new', + # message_key) + message_key = mail_box.add(message) + except TimeOutError: + # log the error and go on + elog.error('Unable to lock archive for %s, discarded' + ' message: %s' % (mlist.fqdn_listname, + message.get('message-id', ''))) -- cgit v1.3.1 From 3f911e76bec2241365974726c36055fed39407df Mon Sep 17 00:00:00 2001 From: toshio Date: Tue, 13 Mar 2012 06:20:20 +0000 Subject: Update documentation for the prototype archiver --- src/mailman/archiving/docs/common.rst | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/mailman/archiving/docs/common.rst b/src/mailman/archiving/docs/common.rst index 330c8e307..7fe788ee4 100644 --- a/src/mailman/archiving/docs/common.rst +++ b/src/mailman/archiving/docs/common.rst @@ -61,13 +61,17 @@ The archiver is also able to archive the message. >>> os.path.exists(pckpath) True -Note however that the prototype archiver can't archive messages. +The prototype archiver is available to simplistically archive messages. +:: >>> archivers['prototype'].archive_message(mlist, msg) - Traceback (most recent call last): - ... - NotImplementedError + >>> import os + >>> from mailman import config + >>> archivepath = os.path.join(config.ARCHIVE_DIR, 'prototype', + ... mlist.fqdn_listname, 'new') + >>> len (os.listdir(archivepath)) >= 1 + True The Mail-Archive.com ==================== -- cgit v1.3.1 From 41fb93ff917b161b292fa945f82c024bd393b9e9 Mon Sep 17 00:00:00 2001 From: toshio Date: Tue, 13 Mar 2012 14:56:04 +0000 Subject: Use 2.6+ syntax for assigning an exception --- src/mailman/archiving/prototype.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/mailman/archiving/prototype.py b/src/mailman/archiving/prototype.py index b3490009a..453ac3dba 100644 --- a/src/mailman/archiving/prototype.py +++ b/src/mailman/archiving/prototype.py @@ -88,7 +88,7 @@ class Prototype: archive_dir = os.path.join(config.ARCHIVE_DIR, 'prototype') try: os.makedirs(archive_dir, 0775) - except OSError, e: + except OSError as e: # If this already exists, then we're fine if e.errno != errno.EEXIST: raise -- cgit v1.3.1 From 446cf202e419d532df5aaec4bebc2eca5178b5b5 Mon Sep 17 00:00:00 2001 From: toshio Date: Wed, 14 Mar 2012 05:26:15 +0000 Subject: Unittests for the prototype archiver --- src/mailman/archiving/tests/__init__.py | 0 src/mailman/archiving/tests/test_prototype.py | 151 ++++++++++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 src/mailman/archiving/tests/__init__.py create mode 100644 src/mailman/archiving/tests/test_prototype.py (limited to 'src') diff --git a/src/mailman/archiving/tests/__init__.py b/src/mailman/archiving/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/mailman/archiving/tests/test_prototype.py b/src/mailman/archiving/tests/test_prototype.py new file mode 100644 index 000000000..06c456076 --- /dev/null +++ b/src/mailman/archiving/tests/test_prototype.py @@ -0,0 +1,151 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +"""Test the prototype archiver.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + ] + +import os +import shutil +import tempfile +import unittest +import threading + +from flufl.lock import Lock + +from mailman.app.lifecycle import create_list +from mailman.archiving import prototype +from mailman.config import config +from mailman.testing.helpers import LogFileMark +from mailman.testing.helpers import specialized_message_from_string as smfs +from mailman.testing.layers import ConfigLayer + + +class test_PrototypeArchiveMethod(unittest.TestCase): + layer = ConfigLayer + + def setUp(self): + # Create a fake mailing list and message object + self.message = smfs('''\ +To: test@example.com +From: admin@example.com +Subject: Testing the test list +Message-ID: + +Tests are better than not tests +but the water deserves to be swum + ''') + + self.mlist = create_list('test@example.com') + config.db.commit() + + # Set some config directories where we won't bash real data + config.ARCHIVE_DIR = '%s%s' % (tempfile.mkdtemp(), os.path.sep) + config.LOCK_DIR = tempfile.mkdtemp() + + # Structure of a maildir + self.expected_dir_structure = frozenset( + (os.path.join(config.ARCHIVE_DIR, f) for f in ( + '', + 'prototype', + os.path.join('prototype', self.mlist.fqdn_listname), + os.path.join('prototype', self.mlist.fqdn_listname, 'cur'), + os.path.join('prototype', self.mlist.fqdn_listname, 'new'), + os.path.join('prototype', self.mlist.fqdn_listname, 'tmp'), + ) + ) + ) + + def tearDown(self): + shutil.rmtree(config.ARCHIVE_DIR) + shutil.rmtree(config.LOCK_DIR) + + def _find(self, path): + all_filenames = set() + for dirs in os.walk(path): + directory = dirs[0] + if not isinstance(directory, unicode): + directory = unicode(directory) + all_filenames.add(directory) + if dirs[2]: + for filename in dirs[2]: + new_filename = os.path.join(dirs[0], filename) + if not isinstance(new_filename, unicode): + new_filename = unicode(new_filename) + all_filenames.add(new_filename) + return all_filenames + + def test_archive_maildir_created(self): + prototype.Prototype.archive_message(self.mlist, self.message) + all_filenames = self._find(config.ARCHIVE_DIR) + # Check that the directory structure has been created and we have one + # more file (the archived message) than expected directories + self.assertTrue(self.expected_dir_structure.issubset(all_filenames)) + self.assertEqual(len(all_filenames), len(self.expected_dir_structure) + 1) + + def test_archive_maildir_existance_does_not_raise(self): + os.makedirs(os.path.join(config.ARCHIVE_DIR, 'prototype', + self.mlist.fqdn_listname, 'cur')) + os.mkdir(os.path.join(config.ARCHIVE_DIR, 'prototype', + self.mlist.fqdn_listname, 'new')) + os.mkdir(os.path.join(config.ARCHIVE_DIR, 'prototype', + self.mlist.fqdn_listname, 'tmp')) + + # Checking that no exception is raised in this circumstance because it + # will be the common case (adding a new message to an archive whose + # directories have alreay been created) + try: + prototype.Prototype.archive_message(self.mlist, self.message) + except: + self.assertTrue(False, 'Exception raised when the archive' + ' directory structure already in place') + + def test_archive_lock_used(self): + # Test that locking the maildir when adding works as a failure here + # could mean we lose mail + lock = Lock(os.path.join(config.LOCK_DIR, '%s-maildir.lock' + % self.mlist.fqdn_listname)) + with lock: + # Take this lock. Then make sure the archiver fails while that's + # working. + archive_thread = threading.Thread( + target=prototype.Prototype.archive_message, + args=(self.mlist, self.message)) + mark = LogFileMark('mailman.error') + archive_thread.run() + # Test that the archiver output the correct error + line = mark.readline() + self.assertTrue(line.endswith('Unable to lock archive for %s,' + ' discarded message: %s\n' % (self.mlist.fqdn_listname, + self.message.get('message-id')))) + + # Check that the file didn't get created + created_files = self._find(config.ARCHIVE_DIR) + self.assertEqual(self.expected_dir_structure, created_files) + + def test_mail_added(self): + prototype.Prototype.archive_message(self.mlist, self.message) + for filename in os.listdir(os.path.join(config.ARCHIVE_DIR, + 'prototype', self.mlist.fqdn_listname, 'new')): + # Check that the email has been added + email = open(os.path.join(config.ARCHIVE_DIR, 'prototype', + self.mlist.fqdn_listname, 'new', filename)) + self.assertTrue((repr(self.message)).endswith(email.read())) -- cgit v1.3.1 From ccde42a936f6c87032c7afd80f33ca5f3fa00b54 Mon Sep 17 00:00:00 2001 From: toshio Date: Wed, 14 Mar 2012 05:26:40 +0000 Subject: Revert to using a try: except: finally: for locking as the context manager fails when the lock is stolen out from under it (as from a thread) --- src/mailman/archiving/prototype.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) (limited to 'src') diff --git a/src/mailman/archiving/prototype.py b/src/mailman/archiving/prototype.py index 453ac3dba..266b1da07 100644 --- a/src/mailman/archiving/prototype.py +++ b/src/mailman/archiving/prototype.py @@ -102,17 +102,18 @@ class Prototype: # Lock the maildir as Maildir.add() is not threadsafe lock = Lock(os.path.join(config.LOCK_DIR, '%s-maildir.lock' % mlist.fqdn_listname)) - with lock: - try: - lock.lock(timeout=timedelta(seconds=1)) - # Add the message to the Maildir - # Message_key could be used to construct the file path if - # necessary:: - # os.path.join(archive_dir, mlist.fqdn_listname, 'new', - # message_key) - message_key = mail_box.add(message) - except TimeOutError: - # log the error and go on - elog.error('Unable to lock archive for %s, discarded' - ' message: %s' % (mlist.fqdn_listname, - message.get('message-id', ''))) + try: + lock.lock(timeout=timedelta(seconds=1)) + # Add the message to the Maildir + # Message_key could be used to construct the file path if + # necessary:: + # os.path.join(archive_dir, mlist.fqdn_listname, 'new', + # message_key) + message_key = mail_box.add(message) + except TimeOutError: + # log the error and go on + elog.error('Unable to lock archive for %s, discarded' + ' message: %s' % (mlist.fqdn_listname, + message.get('message-id', ''))) + finally: + lock.unlock(unconditionally=True) -- cgit v1.3.1