- Refactor the way databases are schema-migrated so that load_migrations()

   can be tested separately.
 - Add an `until` argument to load_migrations() so that we can load only up to
   a given timestamp.
 - In load_migrations(), ignore files in which the `version` part of the file
   name is empty.
 - In migrations.rst, use the new, better way of ensuring post-test cleanup.
 - Add tests for partial upgrades.
 - LP: #971013 - schema migrations for beta 2.
 - LP: #967238 - IMailingList.archive + IMailingList.archive_private ->
   IMailingList.archive_policy, and add ArchivePolicy enum.
 - Move the `chdir` context manager to helpers.py and add `temporary_db`
   context manager.

3 files changed, 314 insertions, 0 deletions

diff --git a/src/mailman/database/schema/mm_20120407000000.py b/src/mailman/database/schema/mm_20120407000000.py
new file mode 100644
index 000000000..84c563d45
--- /dev/null
+++ b/src/mailman/database/schema/mm_20120407000000.py
@@ -0,0 +1,68 @@
+# Copyright (C) 2012 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""3.0b1 -> 3.0b2 schema migrations."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    ]
+
+
+from mailman.interfaces.archiver import ArchivePolicy
+from mailman.interfaces.database import DatabaseError
+
+
+
+def upgrade(database, store, version, module_path):
+    if database.TAG == 'sqlite':
+        upgrade_sqlite(database, store, version, module_path)
+    else:
+        # XXX 2012-04-07 BAW: Implement PostgreSQL migration.
+        raise DatabaseError('Database {0} migration not support: {1}'.format(
+            database.TAG, version))
+
+
+
+def upgrade_sqlite(database, store, version, module_path):
+    # Load the first part of the migration.  This creates a temporary table to
+    # hold the new mailinglist table columns.  The problem is that some of the
+    # changes must be performed in Python, so after the first part is loaded,
+    # we do the Python changes, drop the old mailing list table, and then
+    # rename the temporary table to its place.
+    database.load_schema(
+        store, version, 'sqlite_{0}_01.sql'.format(version), module_path)
+    results = store.execute(
+        'select id, news_prefix_subject_too, archive, archive_private '
+        'from mailinglist;')
+    for value in results:
+        id, news_prefix, archive, archive_private = value
+        # Figure out what the new archive_policy column value should be.
+        if archive == 0:
+            archive_policy = int(ArchivePolicy.never)
+        elif archive_private == 1:
+            archive_policy = int(ArchivePolicy.private)
+        else:
+            archive_policy = int(ArchivePolicy.public)
+        store.execute(
+            'update ml_backup set '
+            '    nntp_prefix_subject_too = {0}, '
+            '    archive_policy = {1} '
+            'where id = {2};'.format(news_prefix, archive_policy, id))
+    store.execute('drop table mailinglist;')
+    store.execute('alter table ml_backup rename to mailinglist;')
diff --git a/src/mailman/database/schema/sqlite.sql b/src/mailman/database/schema/sqlite.sql
index e6211bf53..e2b2d3814 100644
--- a/src/mailman/database/schema/sqlite.sql
+++ b/src/mailman/database/schema/sqlite.sql
@@ -1,3 +1,6 @@
+-- THIS FILE HAS BEEN FROZEN AS OF 3.0b1
+-- SEE THE SCHEMA MIGRATIONS FOR DIFFERENCES.
+
 PRAGMA foreign_keys = ON;
 
 CREATE TABLE _request (
diff --git a/src/mailman/database/schema/sqlite_20120407000000_01.sql b/src/mailman/database/schema/sqlite_20120407000000_01.sql
new file mode 100644
index 000000000..4fae9ed9d
--- /dev/null
+++ b/src/mailman/database/schema/sqlite_20120407000000_01.sql
@@ -0,0 +1,243 @@
+-- THIS FILE CONTAINS THE SQLITE3 SCHEMA MIGRATION FROM
+-- 3.0b1 TO 3.0b2
+--
+-- AFTER 3.0b2 IS RELEASED YOU MAY NOT EDIT THIS FILE.
+
+-- For SQLite3 migration strategy, see
+-- http://sqlite.org/faq.html#q11
+
+-- This is the base mailinglist table but with these changes:
+-- REM archive
+-- REM archive_private
+-- REM archive_volume_frequency
+-- REM news_prefix_subject_too
+-- REM nntp_host
+--
+-- THESE COLUMNS ARE ADDED BY THE PYTHON MIGRATION LAYER:
+-- ADD archive_policy
+-- ADD nntp_prefix_subject_too
+-- LP: #971013
+
+CREATE TABLE ml_backup(
+    id INTEGER NOT NULL,
+    -- List identity
+    list_name TEXT,
+    mail_host TEXT,
+    include_list_post_header BOOLEAN,
+    include_rfc2369_headers BOOLEAN,
+    -- Attributes not directly modifiable via the web u/i
+    created_at TIMESTAMP,
+    admin_member_chunksize INTEGER,
+    next_request_id INTEGER,
+    next_digest_number INTEGER,
+    digest_last_sent_at TIMESTAMP,
+    volume INTEGER,
+    last_post_at TIMESTAMP,
+    accept_these_nonmembers BLOB,
+    acceptable_aliases_id INTEGER,
+    admin_immed_notify BOOLEAN,
+    admin_notify_mchanges BOOLEAN,
+    administrivia BOOLEAN,
+    advertised BOOLEAN,
+    anonymous_list BOOLEAN,
+    -- Automatic responses.
+    autorespond_owner INTEGER,
+    autoresponse_owner_text TEXT,
+    autorespond_postings INTEGER,
+    autoresponse_postings_text TEXT,
+    autorespond_requests INTEGER,
+    autoresponse_request_text TEXT,
+    autoresponse_grace_period TEXT,
+    -- Bounces.
+    forward_unrecognized_bounces_to INTEGER,
+    process_bounces BOOLEAN,
+    bounce_info_stale_after TEXT,
+    bounce_matching_headers TEXT,
+    bounce_notify_owner_on_disable BOOLEAN,
+    bounce_notify_owner_on_removal BOOLEAN,
+    bounce_score_threshold INTEGER,
+    bounce_you_are_disabled_warnings INTEGER,
+    bounce_you_are_disabled_warnings_interval TEXT,
+    -- Content filtering.
+    filter_action INTEGER,
+    filter_content BOOLEAN,
+    collapse_alternatives BOOLEAN,
+    convert_html_to_plaintext BOOLEAN,
+    default_member_action INTEGER,
+    default_nonmember_action INTEGER,
+    description TEXT,
+    digest_footer_uri TEXT,
+    digest_header_uri TEXT,
+    digest_is_default BOOLEAN,
+    digest_send_periodic BOOLEAN,
+    digest_size_threshold FLOAT,
+    digest_volume_frequency INTEGER,
+    digestable BOOLEAN,
+    discard_these_nonmembers BLOB,
+    emergency BOOLEAN,
+    encode_ascii_prefixes BOOLEAN,
+    first_strip_reply_to BOOLEAN,
+    footer_uri TEXT,
+    forward_auto_discards BOOLEAN,
+    gateway_to_mail BOOLEAN,
+    gateway_to_news BOOLEAN,
+    generic_nonmember_action INTEGER,
+    goodbye_message_uri TEXT,
+    header_matches BLOB,
+    header_uri TEXT,
+    hold_these_nonmembers BLOB,
+    info TEXT,
+    linked_newsgroup TEXT,
+    max_days_to_hold INTEGER,
+    max_message_size INTEGER,
+    max_num_recipients INTEGER,
+    member_moderation_notice TEXT,
+    mime_is_default_digest BOOLEAN,
+    moderator_password TEXT,
+    new_member_options INTEGER,
+    news_moderation INTEGER,
+    nondigestable BOOLEAN,
+    nonmember_rejection_notice TEXT,
+    obscure_addresses BOOLEAN,
+    owner_chain TEXT,
+    owner_pipeline TEXT,
+    personalize INTEGER,
+    post_id INTEGER,
+    posting_chain TEXT,
+    posting_pipeline TEXT,
+    preferred_language TEXT,
+    private_roster BOOLEAN,
+    display_name TEXT,
+    reject_these_nonmembers BLOB,
+    reply_goes_to_list INTEGER,
+    reply_to_address TEXT,
+    require_explicit_destination BOOLEAN,
+    respond_to_post_requests BOOLEAN,
+    scrub_nondigest BOOLEAN,
+    send_goodbye_message BOOLEAN,
+    send_reminders BOOLEAN,
+    send_welcome_message BOOLEAN,
+    subject_prefix TEXT,
+    subscribe_auto_approval BLOB,
+    subscribe_policy INTEGER,
+    topics BLOB,
+    topics_bodylines_limit INTEGER,
+    topics_enabled BOOLEAN,
+    unsubscribe_policy INTEGER,
+    welcome_message_uri TEXT,
+    PRIMARY KEY (id)
+    );
+
+
+INSERT INTO ml_backup SELECT
+    id,
+    -- List identity
+    list_name,
+    mail_host,
+    include_list_post_header,
+    include_rfc2369_headers,
+    -- Attributes not directly modifiable via the web u/i
+    created_at,
+    admin_member_chunksize,
+    next_request_id,
+    next_digest_number,
+    digest_last_sent_at,
+    volume,
+    last_post_at,
+    accept_these_nonmembers,
+    acceptable_aliases_id,
+    admin_immed_notify,
+    admin_notify_mchanges,
+    administrivia,
+    advertised,
+    anonymous_list,
+    -- Automatic responses.
+    autorespond_owner,
+    autoresponse_owner_text,
+    autorespond_postings,
+    autoresponse_postings_text,
+    autorespond_requests,
+    autoresponse_request_text,
+    autoresponse_grace_period,
+    -- Bounces.
+    forward_unrecognized_bounces_to,
+    process_bounces,
+    bounce_info_stale_after,
+    bounce_matching_headers,
+    bounce_notify_owner_on_disable,
+    bounce_notify_owner_on_removal,
+    bounce_score_threshold,
+    bounce_you_are_disabled_warnings,
+    bounce_you_are_disabled_warnings_interval,
+    -- Content filtering.
+    filter_action,
+    filter_content,
+    collapse_alternatives,
+    convert_html_to_plaintext,
+    default_member_action,
+    default_nonmember_action,
+    description,
+    digest_footer_uri,
+    digest_header_uri,
+    digest_is_default,
+    digest_send_periodic,
+    digest_size_threshold,
+    digest_volume_frequency,
+    digestable,
+    discard_these_nonmembers,
+    emergency,
+    encode_ascii_prefixes,
+    first_strip_reply_to,
+    footer_uri,
+    forward_auto_discards,
+    gateway_to_mail,
+    gateway_to_news,
+    generic_nonmember_action,
+    goodbye_message_uri,
+    header_matches,
+    header_uri,
+    hold_these_nonmembers,
+    info,
+    linked_newsgroup,
+    max_days_to_hold,
+    max_message_size,
+    max_num_recipients,
+    member_moderation_notice,
+    mime_is_default_digest,
+    moderator_password,
+    new_member_options,
+    news_moderation,
+    nondigestable,
+    nonmember_rejection_notice,
+    obscure_addresses,
+    owner_chain,
+    owner_pipeline,
+    personalize,
+    post_id,
+    posting_chain,
+    posting_pipeline,
+    preferred_language,
+    private_roster,
+    display_name,
+    reject_these_nonmembers,
+    reply_goes_to_list,
+    reply_to_address,
+    require_explicit_destination,
+    respond_to_post_requests,
+    scrub_nondigest,
+    send_goodbye_message,
+    send_reminders,
+    send_welcome_message,
+    subject_prefix,
+    subscribe_auto_approval,
+    subscribe_policy,
+    topics,
+    topics_bodylines_limit,
+    topics_enabled,
+    unsubscribe_policy,
+    welcome_message_uri
+    FROM mailinglist;
+
+-- Add the new columns.  They'll get inserted at the Python layer.
+ALTER TABLE ml_backup ADD COLUMN archive_policy INTEGER;
+ALTER TABLE ml_backup ADD COLUMN nntp_prefix_subject_too INTEGER;