- Refactor the way databases are schema-migrated so that load_migrations()

   can be tested separately.
 - Add an `until` argument to load_migrations() so that we can load only up to
   a given timestamp.
 - In load_migrations(), ignore files in which the `version` part of the file
   name is empty.
 - In migrations.rst, use the new, better way of ensuring post-test cleanup.
 - Add tests for partial upgrades.
 - LP: #971013 - schema migrations for beta 2.
 - LP: #967238 - IMailingList.archive + IMailingList.archive_private ->
   IMailingList.archive_policy, and add ArchivePolicy enum.
 - Move the `chdir` context manager to helpers.py and add `temporary_db`
   context manager.

7 files changed, 494 insertions, 9 deletions

diff --git a/src/mailman/database/base.py b/src/mailman/database/base.py
index a69e99395..34f644afa 100644
--- a/src/mailman/database/base.py
+++ b/src/mailman/database/base.py
@@ -143,11 +143,15 @@ class StormBaseDatabase:
         database.DEBUG = (as_boolean(config.database.debug)
                           if debug is None else debug)
         self.store = store
-        self.load_migrations()
         store.commit()
 
-    def load_migrations(self):
-        """Load all not-yet loaded migrations."""
+    def load_migrations(self, until=None):
+        """Load schema migrations.
+
+        :param until: Load only the migrations up to the specified timestamp.
+            With default value of None, load all migrations.
+        :type until: string
+        """
         migrations_path = config.database.migrations_path
         if '.' in migrations_path:
             parent, dot, child = migrations_path.rpartition('.')
@@ -170,9 +174,12 @@ class StormBaseDatabase:
             if len(parts) < 2:
                 continue
             version = parts[1]
-            if version in versions:
+            if len(version.strip()) == 0 or version in versions:
                 # This one is already loaded.
                 continue
+            if until is not None and version > until:
+                # We're done.
+                break
             module_path = migrations_path + '.' + module_fn
             __import__(module_path)
             upgrade = getattr(sys.modules[module_path], 'upgrade', None)
diff --git a/src/mailman/database/docs/migration.rst b/src/mailman/database/docs/migration.rst
index 9897b1ef2..999700ca7 100644
--- a/src/mailman/database/docs/migration.rst
+++ b/src/mailman/database/docs/migration.rst
@@ -55,6 +55,14 @@ specified in the configuration file.
     ... migrations_path: migrations
     ... """)
 
+.. Clean this up at the end of the doctest.
+    >>> def cleanup():
+    ...     import shutil
+    ...     from mailman.config import config
+    ...     config.pop('migrations')
+    ...     shutil.rmtree(tempdir)
+    >>> cleanups.append(cleanup)
+
 Here is an example migrations module.  The key part of this interface is the
 ``upgrade()`` method, which takes four arguments:
 
@@ -138,9 +146,32 @@ We do not get an 802 marker because the migration has already been loaded.
     00000000000801
     20120211000000
 
-.. Clean up the temporary directory::
 
-    >>> config.pop('migrations')
-    >>> sys.path.remove(tempdir)
-    >>> import shutil
-    >>> shutil.rmtree(tempdir)
+Partial upgrades
+================
+
+It's possible (mostly for testing purposes) to only do a partial upgrade, by
+providing a timestamp to `load_migrations()`.  To demonstrate this, we add two
+additional migrations, intended to be applied in sequential order.
+
+    >>> from shutil import copyfile
+    >>> from mailman.testing.helpers import chdir
+    >>> with chdir(path):
+    ...     copyfile('mm_20120211000000.py', 'mm_20120211000002.py')
+    ...     copyfile('mm_20120211000000.py', 'mm_20120211000003.py')
+    ...     copyfile('mm_20120211000000.py', 'mm_20120211000004.py')
+
+Now, only migrate to the ...03 timestamp.
+
+    >>> config.db.load_migrations('20120211000003')
+
+You'll notice that the ...04 version is not present.
+
+    >>> results = config.db.store.find(Version, component='schema')
+    >>> for result in sorted(result.version for result in results):
+    ...     print result
+    00000000000000
+    20120211000000
+    20120211000001
+    20120211000002
+    20120211000003
diff --git a/src/mailman/database/schema/mm_20120407000000.py b/src/mailman/database/schema/mm_20120407000000.py
new file mode 100644
index 000000000..84c563d45
--- /dev/null
+++ b/src/mailman/database/schema/mm_20120407000000.py
@@ -0,0 +1,68 @@
+# Copyright (C) 2012 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""3.0b1 -> 3.0b2 schema migrations."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    ]
+
+
+from mailman.interfaces.archiver import ArchivePolicy
+from mailman.interfaces.database import DatabaseError
+
+
+
+def upgrade(database, store, version, module_path):
+    if database.TAG == 'sqlite':
+        upgrade_sqlite(database, store, version, module_path)
+    else:
+        # XXX 2012-04-07 BAW: Implement PostgreSQL migration.
+        raise DatabaseError('Database {0} migration not support: {1}'.format(
+            database.TAG, version))
+
+
+
+def upgrade_sqlite(database, store, version, module_path):
+    # Load the first part of the migration.  This creates a temporary table to
+    # hold the new mailinglist table columns.  The problem is that some of the
+    # changes must be performed in Python, so after the first part is loaded,
+    # we do the Python changes, drop the old mailing list table, and then
+    # rename the temporary table to its place.
+    database.load_schema(
+        store, version, 'sqlite_{0}_01.sql'.format(version), module_path)
+    results = store.execute(
+        'select id, news_prefix_subject_too, archive, archive_private '
+        'from mailinglist;')
+    for value in results:
+        id, news_prefix, archive, archive_private = value
+        # Figure out what the new archive_policy column value should be.
+        if archive == 0:
+            archive_policy = int(ArchivePolicy.never)
+        elif archive_private == 1:
+            archive_policy = int(ArchivePolicy.private)
+        else:
+            archive_policy = int(ArchivePolicy.public)
+        store.execute(
+            'update ml_backup set '
+            '    nntp_prefix_subject_too = {0}, '
+            '    archive_policy = {1} '
+            'where id = {2};'.format(news_prefix, archive_policy, id))
+    store.execute('drop table mailinglist;')
+    store.execute('alter table ml_backup rename to mailinglist;')
diff --git a/src/mailman/database/schema/sqlite.sql b/src/mailman/database/schema/sqlite.sql
index e6211bf53..e2b2d3814 100644
--- a/src/mailman/database/schema/sqlite.sql
+++ b/src/mailman/database/schema/sqlite.sql
@@ -1,3 +1,6 @@
+-- THIS FILE HAS BEEN FROZEN AS OF 3.0b1
+-- SEE THE SCHEMA MIGRATIONS FOR DIFFERENCES.
+
 PRAGMA foreign_keys = ON;
 
 CREATE TABLE _request (
diff --git a/src/mailman/database/schema/sqlite_20120407000000_01.sql b/src/mailman/database/schema/sqlite_20120407000000_01.sql
new file mode 100644
index 000000000..4fae9ed9d
--- /dev/null
+++ b/src/mailman/database/schema/sqlite_20120407000000_01.sql
@@ -0,0 +1,243 @@
+-- THIS FILE CONTAINS THE SQLITE3 SCHEMA MIGRATION FROM
+-- 3.0b1 TO 3.0b2
+--
+-- AFTER 3.0b2 IS RELEASED YOU MAY NOT EDIT THIS FILE.
+
+-- For SQLite3 migration strategy, see
+-- http://sqlite.org/faq.html#q11
+
+-- This is the base mailinglist table but with these changes:
+-- REM archive
+-- REM archive_private
+-- REM archive_volume_frequency
+-- REM news_prefix_subject_too
+-- REM nntp_host
+--
+-- THESE COLUMNS ARE ADDED BY THE PYTHON MIGRATION LAYER:
+-- ADD archive_policy
+-- ADD nntp_prefix_subject_too
+-- LP: #971013
+
+CREATE TABLE ml_backup(
+    id INTEGER NOT NULL,
+    -- List identity
+    list_name TEXT,
+    mail_host TEXT,
+    include_list_post_header BOOLEAN,
+    include_rfc2369_headers BOOLEAN,
+    -- Attributes not directly modifiable via the web u/i
+    created_at TIMESTAMP,
+    admin_member_chunksize INTEGER,
+    next_request_id INTEGER,
+    next_digest_number INTEGER,
+    digest_last_sent_at TIMESTAMP,
+    volume INTEGER,
+    last_post_at TIMESTAMP,
+    accept_these_nonmembers BLOB,
+    acceptable_aliases_id INTEGER,
+    admin_immed_notify BOOLEAN,
+    admin_notify_mchanges BOOLEAN,
+    administrivia BOOLEAN,
+    advertised BOOLEAN,
+    anonymous_list BOOLEAN,
+    -- Automatic responses.
+    autorespond_owner INTEGER,
+    autoresponse_owner_text TEXT,
+    autorespond_postings INTEGER,
+    autoresponse_postings_text TEXT,
+    autorespond_requests INTEGER,
+    autoresponse_request_text TEXT,
+    autoresponse_grace_period TEXT,
+    -- Bounces.
+    forward_unrecognized_bounces_to INTEGER,
+    process_bounces BOOLEAN,
+    bounce_info_stale_after TEXT,
+    bounce_matching_headers TEXT,
+    bounce_notify_owner_on_disable BOOLEAN,
+    bounce_notify_owner_on_removal BOOLEAN,
+    bounce_score_threshold INTEGER,
+    bounce_you_are_disabled_warnings INTEGER,
+    bounce_you_are_disabled_warnings_interval TEXT,
+    -- Content filtering.
+    filter_action INTEGER,
+    filter_content BOOLEAN,
+    collapse_alternatives BOOLEAN,
+    convert_html_to_plaintext BOOLEAN,
+    default_member_action INTEGER,
+    default_nonmember_action INTEGER,
+    description TEXT,
+    digest_footer_uri TEXT,
+    digest_header_uri TEXT,
+    digest_is_default BOOLEAN,
+    digest_send_periodic BOOLEAN,
+    digest_size_threshold FLOAT,
+    digest_volume_frequency INTEGER,
+    digestable BOOLEAN,
+    discard_these_nonmembers BLOB,
+    emergency BOOLEAN,
+    encode_ascii_prefixes BOOLEAN,
+    first_strip_reply_to BOOLEAN,
+    footer_uri TEXT,
+    forward_auto_discards BOOLEAN,
+    gateway_to_mail BOOLEAN,
+    gateway_to_news BOOLEAN,
+    generic_nonmember_action INTEGER,
+    goodbye_message_uri TEXT,
+    header_matches BLOB,
+    header_uri TEXT,
+    hold_these_nonmembers BLOB,
+    info TEXT,
+    linked_newsgroup TEXT,
+    max_days_to_hold INTEGER,
+    max_message_size INTEGER,
+    max_num_recipients INTEGER,
+    member_moderation_notice TEXT,
+    mime_is_default_digest BOOLEAN,
+    moderator_password TEXT,
+    new_member_options INTEGER,
+    news_moderation INTEGER,
+    nondigestable BOOLEAN,
+    nonmember_rejection_notice TEXT,
+    obscure_addresses BOOLEAN,
+    owner_chain TEXT,
+    owner_pipeline TEXT,
+    personalize INTEGER,
+    post_id INTEGER,
+    posting_chain TEXT,
+    posting_pipeline TEXT,
+    preferred_language TEXT,
+    private_roster BOOLEAN,
+    display_name TEXT,
+    reject_these_nonmembers BLOB,
+    reply_goes_to_list INTEGER,
+    reply_to_address TEXT,
+    require_explicit_destination BOOLEAN,
+    respond_to_post_requests BOOLEAN,
+    scrub_nondigest BOOLEAN,
+    send_goodbye_message BOOLEAN,
+    send_reminders BOOLEAN,
+    send_welcome_message BOOLEAN,
+    subject_prefix TEXT,
+    subscribe_auto_approval BLOB,
+    subscribe_policy INTEGER,
+    topics BLOB,
+    topics_bodylines_limit INTEGER,
+    topics_enabled BOOLEAN,
+    unsubscribe_policy INTEGER,
+    welcome_message_uri TEXT,
+    PRIMARY KEY (id)
+    );
+
+
+INSERT INTO ml_backup SELECT
+    id,
+    -- List identity
+    list_name,
+    mail_host,
+    include_list_post_header,
+    include_rfc2369_headers,
+    -- Attributes not directly modifiable via the web u/i
+    created_at,
+    admin_member_chunksize,
+    next_request_id,
+    next_digest_number,
+    digest_last_sent_at,
+    volume,
+    last_post_at,
+    accept_these_nonmembers,
+    acceptable_aliases_id,
+    admin_immed_notify,
+    admin_notify_mchanges,
+    administrivia,
+    advertised,
+    anonymous_list,
+    -- Automatic responses.
+    autorespond_owner,
+    autoresponse_owner_text,
+    autorespond_postings,
+    autoresponse_postings_text,
+    autorespond_requests,
+    autoresponse_request_text,
+    autoresponse_grace_period,
+    -- Bounces.
+    forward_unrecognized_bounces_to,
+    process_bounces,
+    bounce_info_stale_after,
+    bounce_matching_headers,
+    bounce_notify_owner_on_disable,
+    bounce_notify_owner_on_removal,
+    bounce_score_threshold,
+    bounce_you_are_disabled_warnings,
+    bounce_you_are_disabled_warnings_interval,
+    -- Content filtering.
+    filter_action,
+    filter_content,
+    collapse_alternatives,
+    convert_html_to_plaintext,
+    default_member_action,
+    default_nonmember_action,
+    description,
+    digest_footer_uri,
+    digest_header_uri,
+    digest_is_default,
+    digest_send_periodic,
+    digest_size_threshold,
+    digest_volume_frequency,
+    digestable,
+    discard_these_nonmembers,
+    emergency,
+    encode_ascii_prefixes,
+    first_strip_reply_to,
+    footer_uri,
+    forward_auto_discards,
+    gateway_to_mail,
+    gateway_to_news,
+    generic_nonmember_action,
+    goodbye_message_uri,
+    header_matches,
+    header_uri,
+    hold_these_nonmembers,
+    info,
+    linked_newsgroup,
+    max_days_to_hold,
+    max_message_size,
+    max_num_recipients,
+    member_moderation_notice,
+    mime_is_default_digest,
+    moderator_password,
+    new_member_options,
+    news_moderation,
+    nondigestable,
+    nonmember_rejection_notice,
+    obscure_addresses,
+    owner_chain,
+    owner_pipeline,
+    personalize,
+    post_id,
+    posting_chain,
+    posting_pipeline,
+    preferred_language,
+    private_roster,
+    display_name,
+    reject_these_nonmembers,
+    reply_goes_to_list,
+    reply_to_address,
+    require_explicit_destination,
+    respond_to_post_requests,
+    scrub_nondigest,
+    send_goodbye_message,
+    send_reminders,
+    send_welcome_message,
+    subject_prefix,
+    subscribe_auto_approval,
+    subscribe_policy,
+    topics,
+    topics_bodylines_limit,
+    topics_enabled,
+    unsubscribe_policy,
+    welcome_message_uri
+    FROM mailinglist;
+
+-- Add the new columns.  They'll get inserted at the Python layer.
+ALTER TABLE ml_backup ADD COLUMN archive_policy INTEGER;
+ALTER TABLE ml_backup ADD COLUMN nntp_prefix_subject_too INTEGER;
diff --git a/src/mailman/database/tests/__init__.py b/src/mailman/database/tests/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/src/mailman/database/tests/__init__.py
diff --git a/src/mailman/database/tests/test_migrations.py b/src/mailman/database/tests/test_migrations.py
new file mode 100644
index 000000000..9800a7bd7
--- /dev/null
+++ b/src/mailman/database/tests/test_migrations.py
@@ -0,0 +1,133 @@
+# Copyright (C) 2012 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Test schema migrations."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    'TestMigration20120407',
+    ]
+
+
+import os
+import shutil
+import sqlite3
+import tempfile
+import unittest
+
+from zope.component import getUtility
+
+from mailman.app.lifecycle import create_list
+from mailman.config import config
+from mailman.interfaces.domain import IDomainManager
+from mailman.testing.helpers import configuration, temporary_db
+from mailman.testing.layers import ConfigLayer
+from mailman.utilities.modules import call_name
+
+
+
+class TestMigration20120407(unittest.TestCase):
+    """Test the dated migration (LP: #971013)
+
+    Circa: 3.0b1 -> 3.0b2
+
+    table mailinglist:
+    * news_prefix_subject_too -> nntp_prefix_subject_too
+    * ADD archive_policy
+    * REMOVE archive
+    * REMOVE archive_private
+    * REMOVE archive_volume_frequency
+    * REMOVE nntp_host
+    """
+
+    layer = ConfigLayer
+
+    def setUp(self):
+        self._tempdir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self._tempdir)
+
+    def test_sqlite_base(self):
+        # Test the SQLite base schema.
+        url = 'sqlite:///' + os.path.join(self._tempdir, 'mailman.db')
+        database_class = config.database['class']
+        database = call_name(database_class)
+        with configuration('database', url=url):
+            database.initialize()
+        # Load all the database SQL to just before ours.
+        database.load_migrations('20120406999999')
+        # Populate the test database with a domain and a mailing list.
+        with temporary_db(database):
+            getUtility(IDomainManager).add(
+                'example.com', 'An example domain.',
+                'http://lists.example.com', 'postmaster@example.com')
+            mlist = create_list('test@example.com')
+            del mlist
+        database.commit()
+        # Verify that the database has not yet been migrated.
+        for missing in ('archive_policy',
+                        'nntp_prefix_subject_too'):
+            self.assertRaises(sqlite3.OperationalError,
+                              database.store.execute,
+                              'select {0} from mailinglist;'.format(missing))
+        for present in ('archive',
+                        'archive_private',
+                        'archive_volume_frequency',
+                        'news_prefix_subject_too',
+                        'nntp_host'):
+            # This should not produce an exception.  Is there some better test
+            # that we can perform?
+            database.store.execute(
+                'select {0} from mailinglist;'.format(present))
+
+    def test_sqlite_migration(self):
+        # Test the SQLite base schema.
+        url = 'sqlite:///' + os.path.join(self._tempdir, 'mailman.db')
+        database_class = config.database['class']
+        database = call_name(database_class)
+        with configuration('database', url=url):
+            database.initialize()
+        # Load all the database SQL to just before ours.
+        database.load_migrations('20120406999999')
+        # Populate the test database with a domain and a mailing list.
+        with temporary_db(database):
+            getUtility(IDomainManager).add(
+                'example.com', 'An example domain.',
+                'http://lists.example.com', 'postmaster@example.com')
+            mlist = create_list('test@example.com')
+            del mlist
+        database.commit()
+        # Load all migrations, up to and including this one.
+        database.load_migrations('20120407000000')
+        # Verify that the database has been migrated.
+        for present in ('archive_policy',
+                        'nntp_prefix_subject_too'):
+            # This should not produce an exception.  Is there some better test
+            # that we can perform?
+            database.store.execute(
+                'select {0} from mailinglist;'.format(present))
+        for missing in ('archive',
+                        'archive_private',
+                        'archive_volume_frequency',
+                        'news_prefix_subject_too',
+                        'nntp_host'):
+            self.assertRaises(sqlite3.OperationalError,
+                              database.store.execute,
+                              'select {0} from mailinglist;'.format(missing))