11 files changed, 262 insertions, 83 deletions

diff --git a/src/mailman/database/base.py b/src/mailman/database/base.py
index 98e30de7c..0b23c8adc 100644
--- a/src/mailman/database/base.py
+++ b/src/mailman/database/base.py
@@ -27,7 +27,6 @@ import os
 import sys
 import logging
 
-from flufl.lock import Lock
 from lazr.config import as_boolean
 from pkg_resources import resource_listdir, resource_string
 from storm.cache import GenerationalCache
@@ -60,13 +59,6 @@ class StormBaseDatabase:
         self.url = None
         self.store = None
 
-    def initialize(self, debug=None):
-        """See `IDatabase`."""
-        # Serialize this so we don't get multiple processes trying to create
-        # the database at the same time.
-        with Lock(os.path.join(config.LOCK_DIR, 'dbcreate.lck')):
-            self._create(debug)
-
     def begin(self):
         """See `IDatabase`."""
         # Storm takes care of this for us.
@@ -117,7 +109,8 @@ class StormBaseDatabase:
         """
         pass
 
-    def _create(self, debug):
+    def initialize(self, debug=None):
+        """See `IDatabase`."""
         # Calculate the engine url.
         url = expand(config.database.url, config.paths)
         log.debug('Database url: %s', url)
diff --git a/src/mailman/database/docs/migration.rst b/src/mailman/database/docs/migration.rst
index 2988579d3..bf1cfa925 100644
--- a/src/mailman/database/docs/migration.rst
+++ b/src/mailman/database/docs/migration.rst
@@ -30,13 +30,14 @@ already applied.
     >>> from mailman.model.version import Version
     >>> results = config.db.store.find(Version, component='schema')
     >>> results.count()
-    3
+    4
     >>> versions = sorted(result.version for result in results)
     >>> for version in versions:
     ...     print version
     00000000000000
     20120407000000
     20121015000000
+    20130406000000
 
 
 Migrations
@@ -79,7 +80,7 @@ This migration module just adds a marker to the `version` table.
 
     >>> with open(os.path.join(path, '__init__.py'), 'w') as fp:
     ...     pass
-    >>> with open(os.path.join(path, 'mm_20129999000000.py'), 'w') as fp:
+    >>> with open(os.path.join(path, 'mm_20159999000000.py'), 'w') as fp:
     ...     print >> fp, """
     ... from __future__ import unicode_literals
     ... from mailman.model.version import Version
@@ -98,14 +99,15 @@ This will load the new migration, since it hasn't been loaded before.
     00000000000000
     20120407000000
     20121015000000
-    20129999000000
+    20130406000000
+    20159999000000
     >>> test = config.db.store.find(Version, component='test').one()
     >>> print test.version
-    20129999000000
+    20159999000000
 
 Migrations will only be loaded once.
 
-    >>> with open(os.path.join(path, 'mm_20129999000001.py'), 'w') as fp:
+    >>> with open(os.path.join(path, 'mm_20159999000001.py'), 'w') as fp:
     ...     print >> fp, """
     ... from __future__ import unicode_literals
     ... from mailman.model.version import Version
@@ -129,13 +131,14 @@ The first time we load this new migration, we'll get the 801 marker.
     00000000000000
     20120407000000
     20121015000000
-    20129999000000
-    20129999000001
+    20130406000000
+    20159999000000
+    20159999000001
     >>> test = config.db.store.find(Version, component='test')
     >>> for marker in sorted(marker.version for marker in test):
     ...     print marker
     00000000000801
-    20129999000000
+    20159999000000
 
 We do not get an 802 marker because the migration has already been loaded.
 
@@ -146,13 +149,14 @@ We do not get an 802 marker because the migration has already been loaded.
     00000000000000
     20120407000000
     20121015000000
-    20129999000000
-    20129999000001
+    20130406000000
+    20159999000000
+    20159999000001
     >>> test = config.db.store.find(Version, component='test')
     >>> for marker in sorted(marker.version for marker in test):
     ...     print marker
     00000000000801
-    20129999000000
+    20159999000000
 
 
 Partial upgrades
@@ -165,13 +169,13 @@ additional migrations, intended to be applied in sequential order.
     >>> from shutil import copyfile
     >>> from mailman.testing.helpers import chdir
     >>> with chdir(path):
-    ...     copyfile('mm_20129999000000.py', 'mm_20129999000002.py')
-    ...     copyfile('mm_20129999000000.py', 'mm_20129999000003.py')
-    ...     copyfile('mm_20129999000000.py', 'mm_20129999000004.py')
+    ...     copyfile('mm_20159999000000.py', 'mm_20159999000002.py')
+    ...     copyfile('mm_20159999000000.py', 'mm_20159999000003.py')
+    ...     copyfile('mm_20159999000000.py', 'mm_20159999000004.py')
 
 Now, only migrate to the ...03 timestamp.
 
-    >>> config.db.load_migrations('20129999000003')
+    >>> config.db.load_migrations('20159999000003')
 
 You'll notice that the ...04 version is not present.
 
@@ -181,10 +185,11 @@ You'll notice that the ...04 version is not present.
     00000000000000
     20120407000000
     20121015000000
-    20129999000000
-    20129999000001
-    20129999000002
-    20129999000003
+    20130406000000
+    20159999000000
+    20159999000001
+    20159999000002
+    20159999000003
 
 
 .. cleanup:
diff --git a/src/mailman/database/factory.py b/src/mailman/database/factory.py
index bf4d0df7a..bae3fdc11 100644
--- a/src/mailman/database/factory.py
+++ b/src/mailman/database/factory.py
@@ -27,8 +27,10 @@ __all__ = [
     ]
 
 
+import os
 import types
 
+from flufl.lock import Lock
 from zope.component import getAdapter
 from zope.interface import implementer
 from zope.interface.verify import verifyObject
@@ -47,13 +49,15 @@ class DatabaseFactory:
     @staticmethod
     def create():
         """See `IDatabaseFactory`."""
-        database_class = config.database['class']
-        database = call_name(database_class)
-        verifyObject(IDatabase, database)
-        database.initialize()
-        database.load_migrations()
-        database.commit()
-        return database
+        with Lock(os.path.join(config.LOCK_DIR, 'dbcreate.lck')):
+            database_class = config.database['class']
+            database = call_name(database_class)
+            verifyObject(IDatabase, database)
+            database.initialize()
+            database.load_migrations()
+            database.commit()
+            import sys; print('db -> done', os.getpid(), file=sys.stderr)
+            return database
 
 
 
diff --git a/src/mailman/database/schema/helpers.py b/src/mailman/database/schema/helpers.py
new file mode 100644
index 000000000..c8638a12a
--- /dev/null
+++ b/src/mailman/database/schema/helpers.py
@@ -0,0 +1,43 @@
+# Copyright (C) 2013 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Schema migration helpers."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    'make_listid',
+    ]
+
+
+
+def make_listid(fqdn_listname):
+    """Turn a FQDN list name into a List-ID."""
+    list_name, at, mail_host = fqdn_listname.partition('@')
+    if at == '':
+        # If there is no @ sign in the value, assume it already contains the
+        # list-id.
+        return fqdn_listname
+    return '{0}.{1}'.format(list_name, mail_host)
+
+
+
+def pivot(store, table_name):
+    """Pivot a backup table into the real table name."""
+    store.execute('DROP TABLE {}'.format(table_name))
+    store.execute('ALTER TABLE {0}_backup RENAME TO {0}'.format(table_name))
diff --git a/src/mailman/database/schema/mm_20120407000000.py b/src/mailman/database/schema/mm_20120407000000.py
index 8855df5af..463635fd9 100644
--- a/src/mailman/database/schema/mm_20120407000000.py
+++ b/src/mailman/database/schema/mm_20120407000000.py
@@ -48,11 +48,11 @@ __all__ = [
     ]
 
 
+from mailman.database.schema.helpers import pivot
 from mailman.interfaces.archiver import ArchivePolicy
 
 
 VERSION = '20120407000000'
-_helper = None
 
 
 
@@ -98,7 +98,7 @@ def upgrade_sqlite(database, store, version, module_path):
         list_id = '{0}.{1}'.format(list_name, mail_host)
         fqdn_listname = '{0}@{1}'.format(list_name, mail_host)
         store.execute("""
-            UPDATE ml_backup SET
+            UPDATE mailinglist_backup SET
                 allow_list_posts = {0},
                 newsgroup_moderation = {1},
                 nntp_prefix_subject_too = {2},
@@ -120,8 +120,7 @@ def upgrade_sqlite(database, store, version, module_path):
             WHERE mailing_list = '{1}';
             """.format(list_id, fqdn_listname))
     # Pivot the backup table to the real thing.
-    store.execute('DROP TABLE mailinglist;')
-    store.execute('ALTER TABLE ml_backup RENAME TO mailinglist;')
+    pivot(store, 'mailinglist')
     # Now add some indexes that were previously missing.
     store.execute(
         'CREATE INDEX ix_mailinglist_list_id ON mailinglist (list_id);')
@@ -137,12 +136,11 @@ def upgrade_sqlite(database, store, version, module_path):
         else:
             list_id = '{0}.{1}'.format(list_name, mail_host)
         store.execute("""
-            UPDATE mem_backup SET list_id = '{0}'
+            UPDATE member_backup SET list_id = '{0}'
             WHERE id = {1};
             """.format(list_id, id))
     # Pivot the backup table to the real thing.
-    store.execute('DROP TABLE member;')
-    store.execute('ALTER TABLE mem_backup RENAME TO member;')
+    pivot(store, 'member')
 
 
 
diff --git a/src/mailman/database/schema/mm_20121015000000.py b/src/mailman/database/schema/mm_20121015000000.py
index 09078901d..da3671998 100644
--- a/src/mailman/database/schema/mm_20121015000000.py
+++ b/src/mailman/database/schema/mm_20121015000000.py
@@ -33,6 +33,9 @@ __all__ = [
     ]
 
 
+from mailman.database.schema.helpers import make_listid, pivot
+
+
 VERSION = '20121015000000'
 
 
@@ -45,19 +48,9 @@ def upgrade(database, store, version, module_path):
 
 
 
-def _make_listid(fqdn_listname):
-    list_name, at, mail_host = fqdn_listname.partition('@')
-    if at == '':
-        # If there is no @ sign in the value, assume it already contains the
-        # list-id.
-        return fqdn_listname
-    return '{0}.{1}'.format(list_name, mail_host)
-
-
-
 def upgrade_sqlite(database, store, version, module_path):
     database.load_schema(
-        store, version, 'sqlite_{0}_01.sql'.format(version), module_path)
+        store, version, 'sqlite_{}_01.sql'.format(version), module_path)
     results = store.execute("""
         SELECT id, mailing_list
         FROM ban;
@@ -67,15 +60,12 @@ def upgrade_sqlite(database, store, version, module_path):
         if mailing_list is None:
             continue
         store.execute("""
-            UPDATE ban_backup SET list_id = '{0}'
-            WHERE id = {1};
-            """.format(_make_listid(mailing_list), id))
+            UPDATE ban_backup SET list_id = '{}'
+            WHERE id = {};
+            """.format(make_listid(mailing_list), id))
     # Pivot the bans backup table to the real thing.
-    store.execute('DROP TABLE ban;')
-    store.execute('ALTER TABLE ban_backup RENAME TO ban;')
-    # Pivot the mailinglist backup table to the real thing.
-    store.execute('DROP TABLE mailinglist;')
-    store.execute('ALTER TABLE ml_backup RENAME TO mailinglist;')
+    pivot(store, 'ban')
+    pivot(store, 'mailinglist')
 
 
 
@@ -90,7 +80,7 @@ def upgrade_postgres(database, store, version, module_path):
         store.execute("""
             UPDATE ban SET list_id = '{0}'
             WHERE id = {1};
-            """.format(_make_listid(mailing_list), id))
+            """.format(make_listid(mailing_list), id))
     store.execute('ALTER TABLE ban DROP COLUMN mailing_list;')
     store.execute('ALTER TABLE mailinglist DROP COLUMN new_member_options;')
     store.execute('ALTER TABLE mailinglist DROP COLUMN send_reminders;')
diff --git a/src/mailman/database/schema/mm_20130406000000.py b/src/mailman/database/schema/mm_20130406000000.py
new file mode 100644
index 000000000..11ba70869
--- /dev/null
+++ b/src/mailman/database/schema/mm_20130406000000.py
@@ -0,0 +1,65 @@
+# Copyright (C) 2013 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""3.0b3 -> 3.0b4 schema migrations.
+
+Renamed:
+ * bounceevent.list_name -> bounceevent.list_id
+"""
+
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    'upgrade'
+    ]
+
+
+from mailman.database.schema.helpers import make_listid, pivot
+
+
+VERSION = '20130406000000'
+
+
+
+def upgrade(database, store, version, module_path):
+    if database.TAG == 'sqlite':
+        upgrade_sqlite(database, store, version, module_path)
+    else:
+        upgrade_postgres(database, store, version, module_path)
+
+
+
+def upgrade_sqlite(database, store, version, module_path):
+    database.load_schema(
+        store, version, 'sqlite_{}_01.sql'.format(version), module_path)
+    results = store.execute("""
+        SELECT id, list_name
+        FROM bounceevent;
+        """)
+    for id, list_name in results:
+        store.execute("""
+            UPDATE bounceevent_backup SET list_id = '{}'
+            WHERE id = {};
+            """.format(make_listid(list_name), id))
+    pivot(store, 'bounceevent')
+
+
+
+def upgrade_postgres(database, store, version, module_path):
+    pass
diff --git a/src/mailman/database/schema/sqlite_20120407000000_01.sql b/src/mailman/database/schema/sqlite_20120407000000_01.sql
index 5eacc4047..a8db75be9 100644
--- a/src/mailman/database/schema/sqlite_20120407000000_01.sql
+++ b/src/mailman/database/schema/sqlite_20120407000000_01.sql
@@ -1,12 +1,12 @@
--- THIS FILE CONTAINS THE SQLITE3 SCHEMA MIGRATION FROM
+-- This file contains the sqlite3 schema migration from
 -- 3.0b1 TO 3.0b2
 --
--- AFTER 3.0b2 IS RELEASED YOU MAY NOT EDIT THIS FILE.
+-- 3.0b2 has been released thus you MAY NOT edit this file.
 
 -- For SQLite3 migration strategy, see
 -- http://sqlite.org/faq.html#q11
 
--- REMOVALS from the mailinglist table.
+-- REMOVALS from the mailinglist table:
 -- REM archive
 -- REM archive_private
 -- REM archive_volume_frequency
@@ -15,7 +15,7 @@
 -- REM news_prefix_subject_too
 -- REM nntp_host
 --
--- ADDS to the mailing list table.
+-- ADDS to the mailing list table:
 -- ADD allow_list_posts
 -- ADD archive_policy
 -- ADD list_id
@@ -25,16 +25,16 @@
 -- LP: #971013
 -- LP: #967238
 
--- REMOVALS from the member table.
+-- REMOVALS from the member table:
 -- REM mailing_list
 
--- ADDS to the member table.
+-- ADDS to the member table:
 -- ADD list_id
 
 -- LP: #1024509
 
 
-CREATE TABLE ml_backup (
+CREATE TABLE mailinglist_backup (
     id INTEGER NOT NULL,
     -- List identity
     list_name TEXT,
@@ -142,7 +142,7 @@ CREATE TABLE ml_backup (
     PRIMARY KEY (id)
     );
 
-INSERT INTO ml_backup SELECT
+INSERT INTO mailinglist_backup SELECT
     id,
     -- List identity
     list_name,
@@ -249,7 +249,7 @@ INSERT INTO ml_backup SELECT
     welcome_message_uri
     FROM mailinglist;
 
-CREATE TABLE mem_backup(
+CREATE TABLE member_backup(
     id INTEGER NOT NULL,
     _member_id TEXT,
     role INTEGER,
@@ -260,7 +260,7 @@ CREATE TABLE mem_backup(
     PRIMARY KEY (id)
     );
 
-INSERT INTO mem_backup SELECT
+INSERT INTO member_backup SELECT
     id,
     _member_id,
     role,
@@ -272,9 +272,9 @@ INSERT INTO mem_backup SELECT
 
 
 -- Add the new columns.  They'll get inserted at the Python layer.
-ALTER TABLE ml_backup ADD COLUMN archive_policy INTEGER;
-ALTER TABLE ml_backup ADD COLUMN list_id TEXT;
-ALTER TABLE ml_backup ADD COLUMN nntp_prefix_subject_too INTEGER;
-ALTER TABLE ml_backup ADD COLUMN newsgroup_moderation INTEGER;
+ALTER TABLE mailinglist_backup ADD COLUMN archive_policy INTEGER;
+ALTER TABLE mailinglist_backup ADD COLUMN list_id TEXT;
+ALTER TABLE mailinglist_backup ADD COLUMN nntp_prefix_subject_too INTEGER;
+ALTER TABLE mailinglist_backup ADD COLUMN newsgroup_moderation INTEGER;
 
-ALTER TABLE mem_backup ADD COLUMN list_id TEXT;
+ALTER TABLE member_backup ADD COLUMN list_id TEXT;
diff --git a/src/mailman/database/schema/sqlite_20121015000000_01.sql b/src/mailman/database/schema/sqlite_20121015000000_01.sql
index 3e2410c3b..a80dc03df 100644
--- a/src/mailman/database/schema/sqlite_20121015000000_01.sql
+++ b/src/mailman/database/schema/sqlite_20121015000000_01.sql
@@ -1,12 +1,12 @@
--- THIS FILE CONTAINS THE SQLITE3 SCHEMA MIGRATION FROM
+-- This file contains the sqlite3 schema migration from
 -- 3.0b2 TO 3.0b3
 --
--- AFTER 3.0b3 IS RELEASED YOU MAY NOT EDIT THIS FILE.
+-- 3.0b3 has been released thus you MAY NOT edit this file.
 
--- REMOVALS from the ban table.
+-- REMOVALS from the ban table:
 -- REM mailing_list
 
--- ADDS to the ban table.
+-- ADDS to the ban table:
 -- ADD list_id
 
 CREATE TABLE ban_backup (
@@ -30,7 +30,7 @@ ALTER TABLE ban_backup ADD COLUMN list_id TEXT;
 -- REM private_roster
 -- REM admin_member_chunksize
 
-CREATE TABLE ml_backup (
+CREATE TABLE mailinglist_backup (
     id INTEGER NOT NULL,
     list_name TEXT,
     mail_host TEXT,
@@ -130,7 +130,7 @@ CREATE TABLE ml_backup (
     PRIMARY KEY (id)
     );
 
-INSERT INTO ml_backup SELECT
+INSERT INTO mailinglist_backup SELECT
     id,
     list_name,
     mail_host,
diff --git a/src/mailman/database/schema/sqlite_20130406000000_01.sql b/src/mailman/database/schema/sqlite_20130406000000_01.sql
new file mode 100644
index 000000000..d95a5fcc0
--- /dev/null
+++ b/src/mailman/database/schema/sqlite_20130406000000_01.sql
@@ -0,0 +1,31 @@
+-- This file contains the SQLite schema migration from
+-- 3.0b3 to 3.0b4
+--
+-- After 3.0b4 is released you may not edit this file.
+
+-- For SQLite3 migration strategy, see
+-- http://sqlite.org/faq.html#q11
+
+-- REMOVALS from the bounceevent table:
+-- REM list_name
+
+-- ADDS to the ban bounceevent table:
+-- ADD list_id
+
+CREATE TABLE bounceevent_backup (
+    id INTEGER NOT NULL,
+    email TEXT,
+    'timestamp' TIMESTAMP,
+    message_id TEXT,
+    context INTEGER,
+    processed BOOLEAN,
+    PRIMARY KEY (id)
+    );
+
+INSERT INTO bounceevent_backup SELECT
+    id, email, "timestamp", message_id,
+    context, processed
+    FROM bounceevent;
+
+ALTER TABLE bounceevent_backup ADD COLUMN list_id TEXT;
+
diff --git a/src/mailman/database/tests/test_migrations.py b/src/mailman/database/tests/test_migrations.py
index 410192605..c264948ae 100644
--- a/src/mailman/database/tests/test_migrations.py
+++ b/src/mailman/database/tests/test_migrations.py
@@ -26,11 +26,14 @@ __all__ = [
     'TestMigration20120407UnchangedData',
     'TestMigration20121015MigratedData',
     'TestMigration20121015Schema',
+    'TestMigration20130406MigratedData',
+    'TestMigration20130406Schema',
     ]
 
 
 import unittest
 
+from datetime import datetime
 from operator import attrgetter
 from pkg_resources import resource_string
 from storm.exceptions import DatabaseError
@@ -44,6 +47,7 @@ from mailman.interfaces.mailinglist import IAcceptableAliasSet
 from mailman.interfaces.nntp import NewsgroupModeration
 from mailman.interfaces.subscriptions import ISubscriptionService
 from mailman.model.bans import Ban
+from mailman.model.bounce import BounceContext, BounceEvent
 from mailman.testing.helpers import temporary_db
 from mailman.testing.layers import ConfigLayer
 
@@ -426,3 +430,49 @@ class TestMigration20121015MigratedData(MigrationTestBase):
         self.assertEqual(bans[0].list_id, 'test.example.com')
         self.assertEqual(bans[1].email, 'bart@example.com')
         self.assertEqual(bans[1].list_id, None)
+
+
+
+class TestMigration20130406Schema(MigrationTestBase):
+    """Test column migrations."""
+
+    def test_pre_upgrade_column_migrations(self):
+        self._missing_present('bounceevent',
+                              ['20130405999999'],
+                              ('list_id',),
+                              ('list_name',))
+
+    def test_post_upgrade_column_migrations(self):
+        self._missing_present('bounceevent',
+                              ['20130405999999',
+                               '20130406000000'],
+                              ('list_name',),
+                              ('list_id',))
+
+
+
+class TestMigration20130406MigratedData(MigrationTestBase):
+    """Test migrated data."""
+
+    def test_migration_bounceevent(self):
+        # Load all migrations to just before the one we're testing.
+        self._database.load_migrations('20130405999999')
+        # Insert a bounce event.
+        self._database.store.execute("""
+            INSERT INTO bounceevent VALUES (
+                1, 'test@example.com', 'anne@example.com',
+                '2013-04-06 21:12:00', '<abc@example.com>',
+                1, 0);
+            """)
+        # Update to the current migration we're testing
+        self._database.load_migrations('20130406000000')
+        # The bounce event should exist, but with a list-id instead of a fqdn
+        # list name.
+        events = list(self._database.store.find(BounceEvent))
+        self.assertEqual(len(events), 1)
+        self.assertEqual(events[0].list_id, 'test.example.com')
+        self.assertEqual(events[0].email, 'anne@example.com')
+        self.assertEqual(events[0].timestamp, datetime(2013, 4, 6, 21, 12))
+        self.assertEqual(events[0].message_id, '<abc@example.com>')
+        self.assertEqual(events[0].context, BounceContext[1])
+        self.assertFalse(events[0].processed)