20 files changed, 318 insertions, 69 deletions

diff --git a/README.rst b/README.rst
index 736b886ba..6091ecdd0 100644
--- a/README.rst
+++ b/README.rst
@@ -2,7 +2,7 @@
 Mailman - The GNU Mailing List Management System
 ================================================
 
-Copyright (C) 1998-2011 by the Free Software Foundation, Inc.
+Copyright (C) 1998-2012 by the Free Software Foundation, Inc.
 
 This is GNU Mailman, a mailing list management system distributed under the
 terms of the GNU General Public License (GPL) version 3 or later.  The name of
diff --git a/conf.py b/conf.py
index 936bb4c0a..fd3511d10 100644
--- a/conf.py
+++ b/conf.py
@@ -41,7 +41,7 @@ master_doc = 'README'
 
 # General information about the project.
 project = u'GNU Mailman'
-copyright = u'1998-2011 by the Free Software Foundation, Inc.'
+copyright = u'1998-2012 by the Free Software Foundation, Inc.'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
diff --git a/src/mailman/bin/mailman.py b/src/mailman/bin/mailman.py
index 14d25f2a7..2a230b49f 100644
--- a/src/mailman/bin/mailman.py
+++ b/src/mailman/bin/mailman.py
@@ -44,7 +44,7 @@ def main():
     parser = argparse.ArgumentParser(
         description=_("""\
         The GNU Mailman mailing list management system
-        Copyright 1998-2011 by the Free Software Foundation, Inc.
+        Copyright 1998-2012 by the Free Software Foundation, Inc.
         http://www.list.org
         """),
         formatter_class=argparse.RawDescriptionHelpFormatter)
diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg
index c94a3828b..5a64db268 100644
--- a/src/mailman/config/schema.cfg
+++ b/src/mailman/config/schema.cfg
@@ -189,6 +189,9 @@ class: mailman.database.sqlite.SQLiteDatabase
 url: sqlite:///$DATA_DIR/mailman.db
 debug: no
 
+# The module path to the migrations modules.
+migrations_path: mailman.database.schema
+
 [logging.template]
 # This defines various log settings.  The options available are:
 #
diff --git a/src/mailman/database/base.py b/src/mailman/database/base.py
index e28ca1893..a69e99395 100644
--- a/src/mailman/database/base.py
+++ b/src/mailman/database/base.py
@@ -24,18 +24,18 @@ __all__ = [
 
 
 import os
+import sys
 import logging
 
 from flufl.lock import Lock
 from lazr.config import as_boolean
+from pkg_resources import resource_listdir, resource_string
 from storm.cache import GenerationalCache
 from storm.locals import create_database, Store
 from zope.interface import implements
 
-import mailman.version
-
 from mailman.config import config
-from mailman.interfaces.database import IDatabase, SchemaVersionMismatchError
+from mailman.interfaces.database import IDatabase
 from mailman.model.version import Version
 from mailman.utilities.string import expand
 
@@ -51,6 +51,10 @@ class StormBaseDatabase:
     Use this as a base class for your DB-specific derived classes.
     """
 
+    # Tag used to distinguish the database being used.  Override this in base
+    # classes.
+    TAG = ''
+
     implements(IDatabase)
 
     def __init__(self):
@@ -87,15 +91,6 @@ class StormBaseDatabase:
         """
         raise NotImplementedError
 
-    def _get_schema(self):
-        """Return the database schema as a string.
-
-        This will be loaded into the database when it is first created.
-
-        Base classes *must* override this.
-        """
-        raise NotImplementedError
-
     def _pre_reset(self, store):
         """Clean up method for testing.
 
@@ -147,34 +142,80 @@ class StormBaseDatabase:
         store = Store(database, GenerationalCache())
         database.DEBUG = (as_boolean(config.database.debug)
                           if debug is None else debug)
-        # Check the master / schema database to see if the version table
-        # exists.  If so, then we assume the database schema is correctly
-        # initialized.  Storm does not currently provide schema creation.
-        if not self._database_exists(store):
-            # Initialize the database.  Start by getting the schema and
-            # discarding all blank and comment lines.
-            lines = self._get_schema().splitlines()
-            lines = (line for line in lines
+        self.store = store
+        self.load_migrations()
+        store.commit()
+
+    def load_migrations(self):
+        """Load all not-yet loaded migrations."""
+        migrations_path = config.database.migrations_path
+        if '.' in migrations_path:
+            parent, dot, child = migrations_path.rpartition('.')
+        else:
+            parent = migrations_path
+            child =''
+        # If the database does not yet exist, load the base schema.
+        filenames = sorted(resource_listdir(parent, child))
+        # Find out which schema migrations have already been loaded.
+        if self._database_exists(self.store):
+            versions = set(version.version for version in
+                           self.store.find(Version, component='schema'))
+        else:
+            versions = set()
+        for filename in filenames:
+            module_fn, extension = os.path.splitext(filename)
+            if extension != '.py':
+                continue
+            parts = module_fn.split('_')
+            if len(parts) < 2:
+                continue
+            version = parts[1]
+            if version in versions:
+                # This one is already loaded.
+                continue
+            module_path = migrations_path + '.' + module_fn
+            __import__(module_path)
+            upgrade = getattr(sys.modules[module_path], 'upgrade', None)
+            if upgrade is None:
+                continue
+            upgrade(self, self.store, version, module_path)
+
+    def load_schema(self, store, version, filename, module_path):
+        """Load the schema from a file.
+
+        This is a helper method for migration classes to call.
+
+        :param store: The Storm store to load the schema into.
+        :type store: storm.locals.Store`
+        :param version: The schema version identifier of the form 
+            YYYYMMDDHHMMSS.
+        :type version: string
+        :param filename: The file name containing the schema to load.  Pass
+            `None` if there is no schema file to load.
+        :type filename: string
+        :param module_path: The fully qualified Python module path to the
+            migration module being loaded.  This is used to record information
+            for use by the test suite.
+        :type module_path: string
+        """
+        if filename is not None:
+            contents = resource_string('mailman.database.schema', filename)
+            # Discard all blank and comment lines.
+            lines = (line for line in contents.splitlines()
                      if line.strip() != '' and line.strip()[:2] != '--')
             sql = NL.join(lines)
             for statement in sql.split(';'):
                 if statement.strip() != '':
                     store.execute(statement + ';')
-        # Validate schema version.
-        v = store.find(Version, component='schema').one()
-        if not v:
-            # Database has not yet been initialized
-            v = Version(component='schema',
-                        version=mailman.version.DATABASE_SCHEMA_VERSION)
-            store.add(v)
-        elif v.version <> mailman.version.DATABASE_SCHEMA_VERSION:
-            # XXX Update schema
-            raise SchemaVersionMismatchError(v.version)
-        self.store = store
-        store.commit()
+        # Add a marker that indicates the migration version being applied.
+        store.add(Version(component='schema', version=version))
+        # Add a marker so that the module name can be found later.  This is
+        # used by the test suite to reset the database between tests.
+        store.add(Version(component=version, version=module_path))
 
     def _reset(self):
         """See `IDatabase`."""
         from mailman.database.model import ModelMeta
         self.store.rollback()
         ModelMeta._reset(self.store)
+        self.store.commit()
diff --git a/src/mailman/database/sql/__init__.py b/src/mailman/database/docs/__init__.py
index e69de29bb..e69de29bb 100644
--- a/src/mailman/database/sql/__init__.py
+++ b/src/mailman/database/docs/__init__.py
diff --git a/src/mailman/database/docs/migration.rst b/src/mailman/database/docs/migration.rst
new file mode 100644
index 000000000..930f7d245
--- /dev/null
+++ b/src/mailman/database/docs/migration.rst
@@ -0,0 +1,145 @@
+=================
+Schema migrations
+=================
+
+The SQL database schema will over time require upgrading to support new
+features.  This is supported via schema migration.
+
+Migrations are embodied in individual Python classes, which themselves may
+load SQL into the database.  The naming scheme for migration files is:
+
+    s_YYYYMMDDHHMMSS_comment.py
+
+where `YYYYMMDDHHMMSS` is a required numeric year, month, day, hour, minute,
+and second specifier providing unique ordering for processing.  Only this
+component of the file name is used to determine the ordering.  (The `s_`
+prefix is required due to Python module naming requirements).
+
+The optional `comment` part of the file name can be used as a short
+description for the migration, although comments and docstrings in the
+migration files should be used for more detailed descriptions.
+
+Migrations are applied automatically when Mailman starts up, but can also be
+applied at any time by calling in the API directly.  Once applied, a
+migration's version string is registered so it will not be applied again.
+
+We see that the base migration is already applied.
+
+    >>> from mailman.model.version import Version
+    >>> results = config.db.store.find(Version, component='schema')
+    >>> results.count()
+    1
+    >>> base = results.one()
+    >>> print base.component
+    schema
+    >>> print base.version
+    00000000000000
+
+
+Migrations
+==========
+
+Migrations can be loaded at any time, and can be found in the migrations path
+specified in the configuration file.
+
+.. Create a temporary directory for the migrations::
+
+    >>> import os, sys, tempfile
+    >>> tempdir = tempfile.mkdtemp()
+    >>> path = os.path.join(tempdir, 'migrations')
+    >>> os.makedirs(path)
+    >>> sys.path.append(tempdir)
+    >>> config.push('migrations', """
+    ... [database]
+    ... migrations_path: migrations
+    ... """)
+
+Here is an example migrations module.  The key part of this interface is the
+``upgrade()`` method, which takes four arguments:
+
+ * `database` - The database class, as derived from `StormBaseDatabase`
+ * `store` - The Storm `Store` object.
+ * `version` - The version string as derived from the migrations module's file
+   name.  This will include only the `YYYYMMDDHHMMSS` string.
+ * `module_path` - The dotted module path to the migrations module, suitable
+   for lookup in `sys.modules`.
+
+This migration module just adds a marker to the `version` table.
+
+    >>> with open(os.path.join(path, '__init__.py'), 'w') as fp:
+    ...     pass
+    >>> with open(os.path.join(path, 's_20120211000000.py'), 'w') as fp:
+    ...     print >> fp, """
+    ... from __future__ import unicode_literals
+    ... from mailman.model.version import Version
+    ... def upgrade(database, store, version, module_path):
+    ...     v = Version(component='test', version=version)
+    ...     store.add(v)
+    ...     database.load_schema(store, version, None, module_path)
+    ... """
+
+This will load the new migration, since it hasn't been loaded before.
+
+    >>> config.db.load_migrations()
+    >>> results = config.db.store.find(Version, component='schema')
+    >>> for result in sorted(result.version for result in results):
+    ...     print result
+    00000000000000
+    20120211000000
+    >>> test = config.db.store.find(Version, component='test').one()
+    >>> print test.version
+    20120211000000
+
+Migrations will only be loaded once.
+
+    >>> with open(os.path.join(path, 's_20120211000001.py'), 'w') as fp:
+    ...     print >> fp, """
+    ... from __future__ import unicode_literals
+    ... from mailman.model.version import Version
+    ... _marker = 801
+    ... def upgrade(database, store, version, module_path):
+    ...     global _marker
+    ...     # Pad enough zeros on the left to reach 14 characters wide.
+    ...     marker = '{0:=#014d}'.format(_marker)
+    ...     _marker += 1
+    ...     v = Version(component='test', version=marker)
+    ...     store.add(v)
+    ...     database.load_schema(store, version, None, module_path)
+    ... """
+
+The first time we load this new migration, we'll get the 801 marker.
+
+    >>> config.db.load_migrations()
+    >>> results = config.db.store.find(Version, component='schema')
+    >>> for result in sorted(result.version for result in results):
+    ...     print result
+    00000000000000
+    20120211000000
+    20120211000001
+    >>> test = config.db.store.find(Version, component='test')
+    >>> for marker in sorted(marker.version for marker in test):
+    ...     print marker
+    00000000000801
+    20120211000000
+
+We do not get an 802 marker because the migration has already been loaded.
+
+    >>> config.db.load_migrations()
+    >>> results = config.db.store.find(Version, component='schema')
+    >>> for result in sorted(result.version for result in results):
+    ...     print result
+    00000000000000
+    20120211000000
+    20120211000001
+    >>> test = config.db.store.find(Version, component='test')
+    >>> for marker in sorted(marker.version for marker in test):
+    ...     print marker
+    00000000000801
+    20120211000000
+
+.. Clean up the temporary directory::
+
+    >>> config.pop('migrations')
+    >>> sys.path.remove(tempdir)
+    >>> import shutil
+    >>> shutil.rmtree(tempdir)
diff --git a/src/mailman/database/model.py b/src/mailman/database/model.py
index 9f6bf9845..c45517c9b 100644
--- a/src/mailman/database/model.py
+++ b/src/mailman/database/model.py
@@ -25,6 +25,8 @@ __all__ = [
     ]
 
 
+import sys
+
 from operator import attrgetter
 
 from storm.properties import PropertyPublisherMeta
@@ -50,12 +52,37 @@ class ModelMeta(PropertyPublisherMeta):
     @staticmethod
     def _reset(store):
         from mailman.config import config
+        from mailman.model.version import Version
         config.db._pre_reset(store)
+        # Give each schema migration a chance to do its pre-reset.  See below
+        # for calling its post reset too.
+        versions = sorted(version.version for version in
+                          store.find(Version, component='schema'))
+        migrations = {}
+        for version in versions:
+            # We have to give the migrations module that loaded this version a
+            # chance to do both pre- and post-reset operations.  The following
+            # find the actual the module path for the migration.  See
+            # StormBaseDatabase.load_schema().
+            migration = store.find(Version, component=version).one()
+            if migration is None:
+                continue
+            migrations[version] = module_path = migration.version
+            module = sys.modules[module_path]
+            pre_reset = getattr(module, 'pre_reset', None)
+            if pre_reset is not None:
+                pre_reset(store)
         # Make sure this is deterministic, by sorting on the storm table name.
         classes = sorted(ModelMeta._class_registry,
                          key=attrgetter('__storm_table__'))
         for model_class in classes:
             store.find(model_class).remove()
+        # Now give each migration a chance to do post-reset operations.
+        for version in versions:
+            module = sys.modules[migrations[version]]
+            post_reset = getattr(module, 'post_reset', None)
+            if post_reset is not None:
+                post_reset(store)
         config.db._post_reset(store)
 
 
diff --git a/src/mailman/database/postgresql.py b/src/mailman/database/postgresql.py
index 9ad8f74b5..988f7a1af 100644
--- a/src/mailman/database/postgresql.py
+++ b/src/mailman/database/postgresql.py
@@ -26,7 +26,6 @@ __all__ = [
 
 
 from operator import attrgetter
-from pkg_resources import resource_string
 
 from mailman.database.base import StormBaseDatabase
 
@@ -35,6 +34,8 @@ from mailman.database.base import StormBaseDatabase
 class PostgreSQLDatabase(StormBaseDatabase):
     """Database class for PostgreSQL."""
 
+    TAG = 'postgres'
+
     def _database_exists(self, store):
         """See `BaseDatabase`."""
         table_query = ('SELECT table_name FROM information_schema.tables '
@@ -43,16 +44,13 @@ class PostgreSQLDatabase(StormBaseDatabase):
                           store.execute(table_query))
         return 'version' in table_names
 
-    def _get_schema(self):
-        """See `BaseDatabase`."""
-        return resource_string('mailman.database.sql', 'postgres.sql')
-
     def _post_reset(self, store):
         """PostgreSQL-specific test suite cleanup.
 
         Reset the <tablename>_id_seq.last_value so that primary key ids
         restart from zero for new tests.
         """
+        super(PostgreSQLDatabase, self)._post_reset(store)
         from mailman.database.model import ModelMeta
         classes = sorted(ModelMeta._class_registry,
                          key=attrgetter('__storm_table__'))
diff --git a/src/mailman/database/schema/__init__.py b/src/mailman/database/schema/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/src/mailman/database/schema/__init__.py
diff --git a/src/mailman/database/sql/postgres.sql b/src/mailman/database/schema/postgres.sql
index 60c05e340..bb209c4aa 100644
--- a/src/mailman/database/sql/postgres.sql
+++ b/src/mailman/database/schema/postgres.sql
@@ -319,7 +319,7 @@ CREATE TABLE pendedkeyvalue (
 CREATE TABLE version (
     id SERIAL NOT NULL,
     component TEXT,
-    version INTEGER,
+    version TEXT,
     PRIMARY KEY (id)
     );
 
diff --git a/src/mailman/database/schema/s_00000000000000_base.py b/src/mailman/database/schema/s_00000000000000_base.py
new file mode 100644
index 000000000..d703088d6
--- /dev/null
+++ b/src/mailman/database/schema/s_00000000000000_base.py
@@ -0,0 +1,55 @@
+# Copyright (C) 2012 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Load the base schema."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    'upgrade',
+    'post_reset',
+    'pre_reset',
+    ]
+
+
+_migration_path = None
+VERSION = '00000000000000'
+
+
+
+def upgrade(database, store, version, module_path):
+    filename = '{0}.sql'.format(database.TAG)
+    database.load_schema(store, version, filename, module_path)
+
+
+def pre_reset(store):
+    global _migration_path
+    # Save the entry in the Version table for the test suite reset.  This will
+    # be restored below.
+    from mailman.model.version import Version
+    result = store.find(Version, component=VERSION).one()
+    # Yes, we abuse this field.
+    _migration_path = result.version
+
+
+def post_reset(store):
+    from mailman.model.version import Version
+    # We need to preserve the Version table entry for this migration, since
+    # its existence defines the fact that the tables have been loaded.
+    store.add(Version(component='schema', version=VERSION))
+    store.add(Version(component=VERSION, version=_migration_path))
diff --git a/src/mailman/database/sql/sqlite.sql b/src/mailman/database/schema/sqlite.sql
index 5987f1879..74d523cf2 100644
--- a/src/mailman/database/sql/sqlite.sql
+++ b/src/mailman/database/schema/sqlite.sql
@@ -295,7 +295,7 @@ CREATE INDEX ix_user_user_id ON user (_user_id);
 CREATE TABLE version (
     id INTEGER NOT NULL,
     component TEXT,
-    version INTEGER,
+    version TEXT,
     PRIMARY KEY (id)
     );
 
diff --git a/src/mailman/database/sqlite.py b/src/mailman/database/sqlite.py
index f48ea19c7..2677d0d71 100644
--- a/src/mailman/database/sqlite.py
+++ b/src/mailman/database/sqlite.py
@@ -27,7 +27,6 @@ __all__ = [
 
 import os
 
-from pkg_resources import resource_string
 from urlparse import urlparse
 
 from mailman.database.base import StormBaseDatabase
@@ -37,6 +36,8 @@ from mailman.database.base import StormBaseDatabase
 class SQLiteDatabase(StormBaseDatabase):
     """Database class for SQLite."""
 
+    TAG = 'sqlite'
+
     def _database_exists(self, store):
         """See `BaseDatabase`."""
         table_query = 'select tbl_name from sqlite_master;'
@@ -53,7 +54,3 @@ class SQLiteDatabase(StormBaseDatabase):
         # Ignore errors
         if fd > 0:
             os.close(fd)
-
-    def _get_schema(self):
-        """See `BaseDatabase`."""
-        return resource_string('mailman.database.sql', 'sqlite.sql')
diff --git a/src/mailman/docs/ACKNOWLEDGMENTS.rst b/src/mailman/docs/ACKNOWLEDGMENTS.rst
index 36a386b57..b6c6cc16c 100644
--- a/src/mailman/docs/ACKNOWLEDGMENTS.rst
+++ b/src/mailman/docs/ACKNOWLEDGMENTS.rst
@@ -4,7 +4,7 @@
 GNU Mailman Acknowledgments
 ===========================
 
-Copyright (C) 1998-2011 by the Free Software Foundation, Inc.
+Copyright (C) 1998-2012 by the Free Software Foundation, Inc.
 
 
 Core Developers
diff --git a/src/mailman/docs/INTRODUCTION.rst b/src/mailman/docs/INTRODUCTION.rst
index e67439613..fd64efb42 100644
--- a/src/mailman/docs/INTRODUCTION.rst
+++ b/src/mailman/docs/INTRODUCTION.rst
@@ -18,7 +18,7 @@ Learn more about GNU Mailman in the `Getting Started`_ documentation.
 Copyright
 =========
 
-Copyright 1998-2011 by the Free Software Foundation, Inc.
+Copyright 1998-2012 by the Free Software Foundation, Inc.
 
 This file is part of GNU Mailman.
 
diff --git a/src/mailman/docs/NEWS.rst b/src/mailman/docs/NEWS.rst
index f101cae06..cdd81d49e 100644
--- a/src/mailman/docs/NEWS.rst
+++ b/src/mailman/docs/NEWS.rst
@@ -2,7 +2,7 @@
 Mailman - The GNU Mailing List Management System
 ================================================
 
-Copyright (C) 1998-2011 by the Free Software Foundation, Inc.
+Copyright (C) 1998-2012 by the Free Software Foundation, Inc.
 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
 
 Here is a history of user visible changes to Mailman.
@@ -14,6 +14,7 @@ Here is a history of user visible changes to Mailman.
 
 Architecture
 ------------
+ * Schema migrations have been implemented.
  * Implement the style manager as a utility instead of an attribute hanging
    off the `mailman.config.config` object.
  * PostgreSQL support contributed by Stephen A. Goss. (LP: #860159)
diff --git a/src/mailman/interfaces/database.py b/src/mailman/interfaces/database.py
index e628354b1..0530f83b9 100644
--- a/src/mailman/interfaces/database.py
+++ b/src/mailman/interfaces/database.py
@@ -23,14 +23,12 @@ __metaclass__ = type
 __all__ = [
     'DatabaseError',
     'IDatabase',
-    'SchemaVersionMismatchError',
     ]
 
 
 from zope.interface import Interface
 
 from mailman.interfaces.errors import MailmanError
-from mailman.version import DATABASE_SCHEMA_VERSION
 
 
 
@@ -38,19 +36,6 @@ class DatabaseError(MailmanError):
     """A problem with the database occurred."""
 
 
-class SchemaVersionMismatchError(DatabaseError):
-    """The database schema version number did not match what was expected."""
-
-    def __init__(self, got):
-        super(SchemaVersionMismatchError, self).__init__()
-        self._got = got
-
-    def __str__(self):
-        return ('Incompatible database schema version '
-                '(got: {0}, expected: {1})'.format(
-                    self._got, DATABASE_SCHEMA_VERSION))
-
-
 
 class IDatabase(Interface):
     """Database layer interface."""
diff --git a/src/mailman/model/version.py b/src/mailman/model/version.py
index 8d198a670..d6a4f3938 100644
--- a/src/mailman/model/version.py
+++ b/src/mailman/model/version.py
@@ -32,7 +32,7 @@ from mailman.database.model import Model
 class Version(Model):
     id = Int(primary=True)
     component = Unicode()
-    version = Int()
+    version = Unicode()
 
     def __init__(self, component, version):
         super(Version, self).__init__()
diff --git a/src/mailman/version.py b/src/mailman/version.py
index 079f3dd8d..00c5b6c7a 100644
--- a/src/mailman/version.py
+++ b/src/mailman/version.py
@@ -40,9 +40,6 @@ HEX_VERSION = ((MAJOR_REV << 24) | (MINOR_REV << 16) | (MICRO_REV << 8) |
                (REL_LEVEL << 4)  | (REL_SERIAL << 0))
 
 
-# SQL database schema version
-DATABASE_SCHEMA_VERSION = 1
-
 # qfile/*.db schema version number
 QFILE_SCHEMA_VERSION = 3