diff options
| -rw-r--r-- | src/mailman/core/initialize.py | 3 | ||||
| -rw-r--r-- | src/mailman/database/base.py | 15 | ||||
| -rw-r--r-- | src/mailman/database/docs/migration.rst | 41 | ||||
| -rw-r--r-- | src/mailman/database/schema/mm_20120407000000.py | 68 | ||||
| -rw-r--r-- | src/mailman/database/schema/sqlite.sql | 3 | ||||
| -rw-r--r-- | src/mailman/database/schema/sqlite_20120407000000_01.sql | 243 | ||||
| -rw-r--r-- | src/mailman/database/tests/__init__.py | 0 | ||||
| -rw-r--r-- | src/mailman/database/tests/test_migrations.py | 133 | ||||
| -rw-r--r-- | src/mailman/interfaces/archiver.py | 8 | ||||
| -rw-r--r-- | src/mailman/testing/helpers.py | 30 | ||||
| -rw-r--r-- | src/mailman/tests/test_documentation.py | 20 |
11 files changed, 537 insertions, 27 deletions
diff --git a/src/mailman/core/initialize.py b/src/mailman/core/initialize.py index 389a45f3b..5659661a7 100644 --- a/src/mailman/core/initialize.py +++ b/src/mailman/core/initialize.py @@ -124,6 +124,7 @@ def initialize_1(config_path=None): def initialize_2(debug=False, propagate_logs=None): """Second initialization step. + * Database * Logging * Pre-hook * Rules @@ -148,6 +149,8 @@ def initialize_2(debug=False, propagate_logs=None): database = call_name(database_class) verifyObject(IDatabase, database) database.initialize(debug) + database.load_migrations() + database.commit() config.db = database # Initialize the rules and chains. Do the imports here so as to avoid # circular imports. diff --git a/src/mailman/database/base.py b/src/mailman/database/base.py index a69e99395..34f644afa 100644 --- a/src/mailman/database/base.py +++ b/src/mailman/database/base.py @@ -143,11 +143,15 @@ class StormBaseDatabase: database.DEBUG = (as_boolean(config.database.debug) if debug is None else debug) self.store = store - self.load_migrations() store.commit() - def load_migrations(self): - """Load all not-yet loaded migrations.""" + def load_migrations(self, until=None): + """Load schema migrations. + + :param until: Load only the migrations up to the specified timestamp. + With default value of None, load all migrations. + :type until: string + """ migrations_path = config.database.migrations_path if '.' in migrations_path: parent, dot, child = migrations_path.rpartition('.') @@ -170,9 +174,12 @@ class StormBaseDatabase: if len(parts) < 2: continue version = parts[1] - if version in versions: + if len(version.strip()) == 0 or version in versions: # This one is already loaded. continue + if until is not None and version > until: + # We're done. + break module_path = migrations_path + '.' + module_fn __import__(module_path) upgrade = getattr(sys.modules[module_path], 'upgrade', None) diff --git a/src/mailman/database/docs/migration.rst b/src/mailman/database/docs/migration.rst index 9897b1ef2..999700ca7 100644 --- a/src/mailman/database/docs/migration.rst +++ b/src/mailman/database/docs/migration.rst @@ -55,6 +55,14 @@ specified in the configuration file. ... migrations_path: migrations ... """) +.. Clean this up at the end of the doctest. + >>> def cleanup(): + ... import shutil + ... from mailman.config import config + ... config.pop('migrations') + ... shutil.rmtree(tempdir) + >>> cleanups.append(cleanup) + Here is an example migrations module. The key part of this interface is the ``upgrade()`` method, which takes four arguments: @@ -138,9 +146,32 @@ We do not get an 802 marker because the migration has already been loaded. 00000000000801 20120211000000 -.. Clean up the temporary directory:: - >>> config.pop('migrations') - >>> sys.path.remove(tempdir) - >>> import shutil - >>> shutil.rmtree(tempdir) +Partial upgrades +================ + +It's possible (mostly for testing purposes) to only do a partial upgrade, by +providing a timestamp to `load_migrations()`. To demonstrate this, we add two +additional migrations, intended to be applied in sequential order. + + >>> from shutil import copyfile + >>> from mailman.testing.helpers import chdir + >>> with chdir(path): + ... copyfile('mm_20120211000000.py', 'mm_20120211000002.py') + ... copyfile('mm_20120211000000.py', 'mm_20120211000003.py') + ... copyfile('mm_20120211000000.py', 'mm_20120211000004.py') + +Now, only migrate to the ...03 timestamp. + + >>> config.db.load_migrations('20120211000003') + +You'll notice that the ...04 version is not present. + + >>> results = config.db.store.find(Version, component='schema') + >>> for result in sorted(result.version for result in results): + ... print result + 00000000000000 + 20120211000000 + 20120211000001 + 20120211000002 + 20120211000003 diff --git a/src/mailman/database/schema/mm_20120407000000.py b/src/mailman/database/schema/mm_20120407000000.py new file mode 100644 index 000000000..84c563d45 --- /dev/null +++ b/src/mailman/database/schema/mm_20120407000000.py @@ -0,0 +1,68 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""3.0b1 -> 3.0b2 schema migrations.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + ] + + +from mailman.interfaces.archiver import ArchivePolicy +from mailman.interfaces.database import DatabaseError + + + +def upgrade(database, store, version, module_path): + if database.TAG == 'sqlite': + upgrade_sqlite(database, store, version, module_path) + else: + # XXX 2012-04-07 BAW: Implement PostgreSQL migration. + raise DatabaseError('Database {0} migration not support: {1}'.format( + database.TAG, version)) + + + +def upgrade_sqlite(database, store, version, module_path): + # Load the first part of the migration. This creates a temporary table to + # hold the new mailinglist table columns. The problem is that some of the + # changes must be performed in Python, so after the first part is loaded, + # we do the Python changes, drop the old mailing list table, and then + # rename the temporary table to its place. + database.load_schema( + store, version, 'sqlite_{0}_01.sql'.format(version), module_path) + results = store.execute( + 'select id, news_prefix_subject_too, archive, archive_private ' + 'from mailinglist;') + for value in results: + id, news_prefix, archive, archive_private = value + # Figure out what the new archive_policy column value should be. + if archive == 0: + archive_policy = int(ArchivePolicy.never) + elif archive_private == 1: + archive_policy = int(ArchivePolicy.private) + else: + archive_policy = int(ArchivePolicy.public) + store.execute( + 'update ml_backup set ' + ' nntp_prefix_subject_too = {0}, ' + ' archive_policy = {1} ' + 'where id = {2};'.format(news_prefix, archive_policy, id)) + store.execute('drop table mailinglist;') + store.execute('alter table ml_backup rename to mailinglist;') diff --git a/src/mailman/database/schema/sqlite.sql b/src/mailman/database/schema/sqlite.sql index e6211bf53..e2b2d3814 100644 --- a/src/mailman/database/schema/sqlite.sql +++ b/src/mailman/database/schema/sqlite.sql @@ -1,3 +1,6 @@ +-- THIS FILE HAS BEEN FROZEN AS OF 3.0b1 +-- SEE THE SCHEMA MIGRATIONS FOR DIFFERENCES. + PRAGMA foreign_keys = ON; CREATE TABLE _request ( diff --git a/src/mailman/database/schema/sqlite_20120407000000_01.sql b/src/mailman/database/schema/sqlite_20120407000000_01.sql new file mode 100644 index 000000000..4fae9ed9d --- /dev/null +++ b/src/mailman/database/schema/sqlite_20120407000000_01.sql @@ -0,0 +1,243 @@ +-- THIS FILE CONTAINS THE SQLITE3 SCHEMA MIGRATION FROM +-- 3.0b1 TO 3.0b2 +-- +-- AFTER 3.0b2 IS RELEASED YOU MAY NOT EDIT THIS FILE. + +-- For SQLite3 migration strategy, see +-- http://sqlite.org/faq.html#q11 + +-- This is the base mailinglist table but with these changes: +-- REM archive +-- REM archive_private +-- REM archive_volume_frequency +-- REM news_prefix_subject_too +-- REM nntp_host +-- +-- THESE COLUMNS ARE ADDED BY THE PYTHON MIGRATION LAYER: +-- ADD archive_policy +-- ADD nntp_prefix_subject_too +-- LP: #971013 + +CREATE TABLE ml_backup( + id INTEGER NOT NULL, + -- List identity + list_name TEXT, + mail_host TEXT, + include_list_post_header BOOLEAN, + include_rfc2369_headers BOOLEAN, + -- Attributes not directly modifiable via the web u/i + created_at TIMESTAMP, + admin_member_chunksize INTEGER, + next_request_id INTEGER, + next_digest_number INTEGER, + digest_last_sent_at TIMESTAMP, + volume INTEGER, + last_post_at TIMESTAMP, + accept_these_nonmembers BLOB, + acceptable_aliases_id INTEGER, + admin_immed_notify BOOLEAN, + admin_notify_mchanges BOOLEAN, + administrivia BOOLEAN, + advertised BOOLEAN, + anonymous_list BOOLEAN, + -- Automatic responses. + autorespond_owner INTEGER, + autoresponse_owner_text TEXT, + autorespond_postings INTEGER, + autoresponse_postings_text TEXT, + autorespond_requests INTEGER, + autoresponse_request_text TEXT, + autoresponse_grace_period TEXT, + -- Bounces. + forward_unrecognized_bounces_to INTEGER, + process_bounces BOOLEAN, + bounce_info_stale_after TEXT, + bounce_matching_headers TEXT, + bounce_notify_owner_on_disable BOOLEAN, + bounce_notify_owner_on_removal BOOLEAN, + bounce_score_threshold INTEGER, + bounce_you_are_disabled_warnings INTEGER, + bounce_you_are_disabled_warnings_interval TEXT, + -- Content filtering. + filter_action INTEGER, + filter_content BOOLEAN, + collapse_alternatives BOOLEAN, + convert_html_to_plaintext BOOLEAN, + default_member_action INTEGER, + default_nonmember_action INTEGER, + description TEXT, + digest_footer_uri TEXT, + digest_header_uri TEXT, + digest_is_default BOOLEAN, + digest_send_periodic BOOLEAN, + digest_size_threshold FLOAT, + digest_volume_frequency INTEGER, + digestable BOOLEAN, + discard_these_nonmembers BLOB, + emergency BOOLEAN, + encode_ascii_prefixes BOOLEAN, + first_strip_reply_to BOOLEAN, + footer_uri TEXT, + forward_auto_discards BOOLEAN, + gateway_to_mail BOOLEAN, + gateway_to_news BOOLEAN, + generic_nonmember_action INTEGER, + goodbye_message_uri TEXT, + header_matches BLOB, + header_uri TEXT, + hold_these_nonmembers BLOB, + info TEXT, + linked_newsgroup TEXT, + max_days_to_hold INTEGER, + max_message_size INTEGER, + max_num_recipients INTEGER, + member_moderation_notice TEXT, + mime_is_default_digest BOOLEAN, + moderator_password TEXT, + new_member_options INTEGER, + news_moderation INTEGER, + nondigestable BOOLEAN, + nonmember_rejection_notice TEXT, + obscure_addresses BOOLEAN, + owner_chain TEXT, + owner_pipeline TEXT, + personalize INTEGER, + post_id INTEGER, + posting_chain TEXT, + posting_pipeline TEXT, + preferred_language TEXT, + private_roster BOOLEAN, + display_name TEXT, + reject_these_nonmembers BLOB, + reply_goes_to_list INTEGER, + reply_to_address TEXT, + require_explicit_destination BOOLEAN, + respond_to_post_requests BOOLEAN, + scrub_nondigest BOOLEAN, + send_goodbye_message BOOLEAN, + send_reminders BOOLEAN, + send_welcome_message BOOLEAN, + subject_prefix TEXT, + subscribe_auto_approval BLOB, + subscribe_policy INTEGER, + topics BLOB, + topics_bodylines_limit INTEGER, + topics_enabled BOOLEAN, + unsubscribe_policy INTEGER, + welcome_message_uri TEXT, + PRIMARY KEY (id) + ); + + +INSERT INTO ml_backup SELECT + id, + -- List identity + list_name, + mail_host, + include_list_post_header, + include_rfc2369_headers, + -- Attributes not directly modifiable via the web u/i + created_at, + admin_member_chunksize, + next_request_id, + next_digest_number, + digest_last_sent_at, + volume, + last_post_at, + accept_these_nonmembers, + acceptable_aliases_id, + admin_immed_notify, + admin_notify_mchanges, + administrivia, + advertised, + anonymous_list, + -- Automatic responses. + autorespond_owner, + autoresponse_owner_text, + autorespond_postings, + autoresponse_postings_text, + autorespond_requests, + autoresponse_request_text, + autoresponse_grace_period, + -- Bounces. + forward_unrecognized_bounces_to, + process_bounces, + bounce_info_stale_after, + bounce_matching_headers, + bounce_notify_owner_on_disable, + bounce_notify_owner_on_removal, + bounce_score_threshold, + bounce_you_are_disabled_warnings, + bounce_you_are_disabled_warnings_interval, + -- Content filtering. + filter_action, + filter_content, + collapse_alternatives, + convert_html_to_plaintext, + default_member_action, + default_nonmember_action, + description, + digest_footer_uri, + digest_header_uri, + digest_is_default, + digest_send_periodic, + digest_size_threshold, + digest_volume_frequency, + digestable, + discard_these_nonmembers, + emergency, + encode_ascii_prefixes, + first_strip_reply_to, + footer_uri, + forward_auto_discards, + gateway_to_mail, + gateway_to_news, + generic_nonmember_action, + goodbye_message_uri, + header_matches, + header_uri, + hold_these_nonmembers, + info, + linked_newsgroup, + max_days_to_hold, + max_message_size, + max_num_recipients, + member_moderation_notice, + mime_is_default_digest, + moderator_password, + new_member_options, + news_moderation, + nondigestable, + nonmember_rejection_notice, + obscure_addresses, + owner_chain, + owner_pipeline, + personalize, + post_id, + posting_chain, + posting_pipeline, + preferred_language, + private_roster, + display_name, + reject_these_nonmembers, + reply_goes_to_list, + reply_to_address, + require_explicit_destination, + respond_to_post_requests, + scrub_nondigest, + send_goodbye_message, + send_reminders, + send_welcome_message, + subject_prefix, + subscribe_auto_approval, + subscribe_policy, + topics, + topics_bodylines_limit, + topics_enabled, + unsubscribe_policy, + welcome_message_uri + FROM mailinglist; + +-- Add the new columns. They'll get inserted at the Python layer. +ALTER TABLE ml_backup ADD COLUMN archive_policy INTEGER; +ALTER TABLE ml_backup ADD COLUMN nntp_prefix_subject_too INTEGER; diff --git a/src/mailman/database/tests/__init__.py b/src/mailman/database/tests/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/mailman/database/tests/__init__.py diff --git a/src/mailman/database/tests/test_migrations.py b/src/mailman/database/tests/test_migrations.py new file mode 100644 index 000000000..9800a7bd7 --- /dev/null +++ b/src/mailman/database/tests/test_migrations.py @@ -0,0 +1,133 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Test schema migrations.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'TestMigration20120407', + ] + + +import os +import shutil +import sqlite3 +import tempfile +import unittest + +from zope.component import getUtility + +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.interfaces.domain import IDomainManager +from mailman.testing.helpers import configuration, temporary_db +from mailman.testing.layers import ConfigLayer +from mailman.utilities.modules import call_name + + + +class TestMigration20120407(unittest.TestCase): + """Test the dated migration (LP: #971013) + + Circa: 3.0b1 -> 3.0b2 + + table mailinglist: + * news_prefix_subject_too -> nntp_prefix_subject_too + * ADD archive_policy + * REMOVE archive + * REMOVE archive_private + * REMOVE archive_volume_frequency + * REMOVE nntp_host + """ + + layer = ConfigLayer + + def setUp(self): + self._tempdir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self._tempdir) + + def test_sqlite_base(self): + # Test the SQLite base schema. + url = 'sqlite:///' + os.path.join(self._tempdir, 'mailman.db') + database_class = config.database['class'] + database = call_name(database_class) + with configuration('database', url=url): + database.initialize() + # Load all the database SQL to just before ours. + database.load_migrations('20120406999999') + # Populate the test database with a domain and a mailing list. + with temporary_db(database): + getUtility(IDomainManager).add( + 'example.com', 'An example domain.', + 'http://lists.example.com', 'postmaster@example.com') + mlist = create_list('test@example.com') + del mlist + database.commit() + # Verify that the database has not yet been migrated. + for missing in ('archive_policy', + 'nntp_prefix_subject_too'): + self.assertRaises(sqlite3.OperationalError, + database.store.execute, + 'select {0} from mailinglist;'.format(missing)) + for present in ('archive', + 'archive_private', + 'archive_volume_frequency', + 'news_prefix_subject_too', + 'nntp_host'): + # This should not produce an exception. Is there some better test + # that we can perform? + database.store.execute( + 'select {0} from mailinglist;'.format(present)) + + def test_sqlite_migration(self): + # Test the SQLite base schema. + url = 'sqlite:///' + os.path.join(self._tempdir, 'mailman.db') + database_class = config.database['class'] + database = call_name(database_class) + with configuration('database', url=url): + database.initialize() + # Load all the database SQL to just before ours. + database.load_migrations('20120406999999') + # Populate the test database with a domain and a mailing list. + with temporary_db(database): + getUtility(IDomainManager).add( + 'example.com', 'An example domain.', + 'http://lists.example.com', 'postmaster@example.com') + mlist = create_list('test@example.com') + del mlist + database.commit() + # Load all migrations, up to and including this one. + database.load_migrations('20120407000000') + # Verify that the database has been migrated. + for present in ('archive_policy', + 'nntp_prefix_subject_too'): + # This should not produce an exception. Is there some better test + # that we can perform? + database.store.execute( + 'select {0} from mailinglist;'.format(present)) + for missing in ('archive', + 'archive_private', + 'archive_volume_frequency', + 'news_prefix_subject_too', + 'nntp_host'): + self.assertRaises(sqlite3.OperationalError, + database.store.execute, + 'select {0} from mailinglist;'.format(missing)) diff --git a/src/mailman/interfaces/archiver.py b/src/mailman/interfaces/archiver.py index f3edc7719..1f7e57ef0 100644 --- a/src/mailman/interfaces/archiver.py +++ b/src/mailman/interfaces/archiver.py @@ -21,6 +21,7 @@ from __future__ import absolute_import, unicode_literals __metaclass__ = type __all__ = [ + 'ArchivePolicy', 'ClobberDate', 'IArchiver', ] @@ -31,6 +32,13 @@ from zope.interface import Interface, Attribute +class ArchivePolicy(Enum): + never = 0 + private = 1 + public = 2 + + + class ClobberDate(Enum): never = 1 maybe = 2 diff --git a/src/mailman/testing/helpers.py b/src/mailman/testing/helpers.py index 3648a6710..d9885cbac 100644 --- a/src/mailman/testing/helpers.py +++ b/src/mailman/testing/helpers.py @@ -25,6 +25,7 @@ __all__ = [ 'TestableMaster', 'body_line_iterator', 'call_api', + 'chdir', 'configuration', 'digest_mbox', 'event_subscribers', @@ -35,6 +36,7 @@ __all__ = [ 'reset_the_world', 'specialized_message_from_string', 'subscribe', + 'temporary_db', 'wait_for_webservice', ] @@ -392,6 +394,34 @@ class configuration: +@contextmanager +def temporary_db(db): + real_db = config.db + config.db = db + try: + yield + finally: + config.db = real_db + + + +class chdir: + """A context manager for temporary directory changing.""" + def __init__(self, directory): + self._curdir = None + self._directory = directory + + def __enter__(self): + self._curdir = os.getcwd() + os.chdir(self._directory) + + def __exit__(self, exc_type, exc_val, exc_tb): + os.chdir(self._curdir) + # Don't suppress exceptions. + return False + + + def subscribe(mlist, first_name, role=MemberRole.member): """Helper for subscribing a sample person to a mailing list.""" user_manager = getUtility(IUserManager) diff --git a/src/mailman/tests/test_documentation.py b/src/mailman/tests/test_documentation.py index a2c1ab592..329e0176a 100644 --- a/src/mailman/tests/test_documentation.py +++ b/src/mailman/tests/test_documentation.py @@ -38,7 +38,8 @@ import mailman from mailman.app.lifecycle import create_list from mailman.config import config -from mailman.testing.helpers import call_api, specialized_message_from_string +from mailman.testing.helpers import ( + call_api, chdir, specialized_message_from_string) from mailman.testing.layers import SMTPLayer @@ -46,23 +47,6 @@ DOT = '.' -class chdir: - """A context manager for temporary directory changing.""" - def __init__(self, directory): - self._curdir = None - self._directory = directory - - def __enter__(self): - self._curdir = os.getcwd() - os.chdir(self._directory) - - def __exit__(self, exc_type, exc_val, exc_tb): - os.chdir(self._curdir) - # Don't suppress exceptions. - return False - - - def stop(): """Call into pdb.set_trace()""" # Do the import here so that you get the wacky special hacked pdb instead |