10 files changed, 256 insertions, 11 deletions

diff --git a/src/mailman/commands/cli_migrate.py b/src/mailman/commands/cli_migrate.py
new file mode 100644
index 000000000..8783b5c46
--- /dev/null
+++ b/src/mailman/commands/cli_migrate.py
@@ -0,0 +1,65 @@
+# Copyright (C) 2010-2014 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""bin/mailman migrate."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    'Migrate',
+    ]
+
+
+from alembic import command
+from alembic.config import Config
+from zope.interface import implementer
+
+from mailman.config import config
+from mailman.core.i18n import _
+from mailman.interfaces.command import ICLISubCommand
+from mailman.utilities.modules import expand_path
+
+
+
+@implementer(ICLISubCommand)
+class Migrate:
+    """Migrate the Mailman database to the latest schema."""
+
+    name = 'migrate'
+
+    def add(self, parser, command_parser):
+        """See `ICLISubCommand`."""
+        command_parser.add_argument(
+            '-a', '--autogenerate',
+            action='store_true', help=_("""\
+            Autogenerate the migration script using Alembic."""))
+        command.parser_add_argument(
+            '-q', '--quiet',
+            action='store_true', default=False,
+            help=('Produce less output.'))
+
+    def process(self, args):
+        alembic_cfg = Config()
+        alembic_cfg.set_main_option(
+            'script_location', expand_path(config.database['alembic_scripts']))
+        if args.autogenerate:
+            command.revision(alembic_cfg, autogenerate=True)
+        else:
+            command.upgrade(alembic_cfg, 'head')
+            if not args.quiet:
+                print('Updated the database schema.')
diff --git a/src/mailman/config/alembic.ini b/src/mailman/config/alembic.ini
new file mode 100644
index 000000000..a7247743c
--- /dev/null
+++ b/src/mailman/config/alembic.ini
@@ -0,0 +1,57 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = src/mailman/database/alembic
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S
diff --git a/src/mailman/config/config.py b/src/mailman/config/config.py
index e8c8ebc8b..52cac414f 100644
--- a/src/mailman/config/config.py
+++ b/src/mailman/config/config.py
@@ -33,7 +33,7 @@ import sys
 from ConfigParser import SafeConfigParser
 from flufl.lock import Lock
 from lazr.config import ConfigSchema, as_boolean
-from pkg_resources import resource_filename, resource_stream, resource_string
+from pkg_resources import resource_stream, resource_string
 from string import Template
 from zope.component import getUtility
 from zope.event import notify
@@ -46,7 +46,7 @@ from mailman.interfaces.configuration import (
     ConfigurationUpdatedEvent, IConfiguration, MissingConfigurationFileError)
 from mailman.interfaces.languages import ILanguageManager
 from mailman.utilities.filesystem import makedirs
-from mailman.utilities.modules import call_name
+from mailman.utilities.modules import call_name, expand_path
 
 
 SPACE = ' '
@@ -304,12 +304,7 @@ def external_configuration(path):
     :return: A `ConfigParser` instance.
     """
     # Is the context coming from a file system or Python path?
-    if path.startswith('python:'):
-        resource_path = path[7:]
-        package, dot, resource = resource_path.rpartition('.')
-        cfg_path = resource_filename(package, resource + '.cfg')
-    else:
-        cfg_path = path
+    cfg_path = expand_path(path)
     parser = SafeConfigParser()
     files = parser.read(cfg_path)
     if files != [cfg_path]:
diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg
index d7508a533..3ed7d72da 100644
--- a/src/mailman/config/schema.cfg
+++ b/src/mailman/config/schema.cfg
@@ -204,8 +204,10 @@ class: mailman.database.sqlite.SQLiteDatabase
 url: sqlite:///$DATA_DIR/mailman.db
 debug: no
 
-# The module path to the migrations modules.
-migrations_path: mailman.database.schema
+# Where can we find the Alembic migration scripts?  The `python:` schema means
+# that this is a module path relative to the Mailman 3 source installation.
+alembic_scripts: python:mailman.database.alembic
+
 
 [logging.template]
 # This defines various log settings.  The options available are:
diff --git a/src/mailman/database/alembic/__init__.py b/src/mailman/database/alembic/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/src/mailman/database/alembic/__init__.py
diff --git a/src/mailman/database/alembic/env.py b/src/mailman/database/alembic/env.py
new file mode 100644
index 000000000..ee6d8293f
--- /dev/null
+++ b/src/mailman/database/alembic/env.py
@@ -0,0 +1,80 @@
+# Copyright (C) 2014 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Alembic migration environment."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    'run_migrations_offline',
+    'run_migrations_online',
+    ]
+
+
+from alembic import context
+from alembic.config import Config
+from contextlib import closing
+from sqlalchemy import create_engine
+
+from mailman.config import config
+from mailman.database.model import Model
+from mailman.utilities.modules import expand_path
+from mailman.utilities.string import expand
+
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL and not an Engine,
+    though an Engine is acceptable here as well.  By skipping the Engine
+    creation we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the script
+    output.
+    """
+    url = expand(config.database.url, config.paths)
+    context.configure(url=url, target_metadata=Model.metadata)
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine and associate a
+    connection with the context.
+    """
+    alembic_cfg = Config()
+    alembic_cfg.set_main_option(
+        'script_location', expand_path(config.database['alembic_scripts']))
+    url = expand(config.database.url, config.paths)
+    engine = create_engine(url)
+
+    connection = engine.connect()
+    with closing(connection):
+        context.configure(
+            connection=connection, target_metadata=Model.metadata)
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()
diff --git a/src/mailman/database/alembic/script.py.mako b/src/mailman/database/alembic/script.py.mako
new file mode 100644
index 000000000..95702017e
--- /dev/null
+++ b/src/mailman/database/alembic/script.py.mako
@@ -0,0 +1,22 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}
diff --git a/src/mailman/database/factory.py b/src/mailman/database/factory.py
index c06f75031..189fd6ac4 100644
--- a/src/mailman/database/factory.py
+++ b/src/mailman/database/factory.py
@@ -29,7 +29,11 @@ __all__ = [
 import os
 import types
 
+from alembic import command
+from alembic.config import Config
+
 from flufl.lock import Lock
+from pkg_resources import resource_filename
 from zope.interface import implementer
 from zope.interface.verify import verifyObject
 
@@ -53,6 +57,9 @@ class DatabaseFactory:
             verifyObject(IDatabase, database)
             database.initialize()
             Model.metadata.create_all(database.engine)
+            alembic_cfg = Config(
+                resource_filename('mailman.config', 'alembic.ini'))
+            command.stamp(alembic_cfg, 'head')
             database.commit()
             return database
 
diff --git a/src/mailman/testing/layers.py b/src/mailman/testing/layers.py
index eb51e309f..4a9841fc2 100644
--- a/src/mailman/testing/layers.py
+++ b/src/mailman/testing/layers.py
@@ -190,6 +190,10 @@ class ConfigLayer(MockAndMonkeyLayer):
     @classmethod
     def tearDown(cls):
         assert cls.var_dir is not None, 'Layer not set up'
+        # Reset the test database after the tests are done so that there is no
+        # data in case the tests are rerun with a database layer like mysql or
+        # postgresql which are not deleted in teardown.
+        reset_the_world()
         config.pop('test config')
         shutil.rmtree(cls.var_dir)
         cls.var_dir = None
diff --git a/src/mailman/utilities/modules.py b/src/mailman/utilities/modules.py
index 5dfec95db..9ff0e50cd 100644
--- a/src/mailman/utilities/modules.py
+++ b/src/mailman/utilities/modules.py
@@ -22,6 +22,7 @@ from __future__ import absolute_import, print_function, unicode_literals
 __metaclass__ = type
 __all__ = [
     'call_name',
+    'expand_path',
     'find_components',
     'find_name',
     'scan_module',
@@ -31,7 +32,7 @@ __all__ = [
 import os
 import sys
 
-from pkg_resources import resource_listdir
+from pkg_resources import resource_filename, resource_listdir
 
 
 
@@ -110,3 +111,15 @@ def find_components(package, interface):
             continue
         for component in scan_module(module, interface):
             yield component
+
+
+
+def expand_path(url):
+    """Expand a python: path, returning the absolute file system path."""
+    # Is the context coming from a file system or Python path?
+    if url.startswith('python:'):
+        resource_path = url[7:]
+        package, dot, resource = resource_path.rpartition('.')
+        return resource_filename(package, resource + '.cfg')
+    else:
+        return url