From e9ab3f31504a176f483c8340b3ad2ba7679fe285 Mon Sep 17 00:00:00 2001
From: Abhilash Raj
Date: Wed, 24 Sep 2014 23:13:12 +0530
Subject: added support for migrations via alembic

---
 src/mailman/database/alembic/README         |  1 +
 src/mailman/database/alembic/env.py         | 72 +++++++++++++++++++++++++++++
 src/mailman/database/alembic/script.py.mako | 22 +++++++++
 src/mailman/database/factory.py             |  7 +++
 src/mailman/testing/layers.py               |  4 ++
 5 files changed, 106 insertions(+)
 create mode 100644 src/mailman/database/alembic/README
 create mode 100644 src/mailman/database/alembic/env.py
 create mode 100644 src/mailman/database/alembic/script.py.mako

(limited to 'src')

diff --git a/src/mailman/database/alembic/README b/src/mailman/database/alembic/README
new file mode 100644
index 000000000..98e4f9c44
--- /dev/null
+++ b/src/mailman/database/alembic/README
@@ -0,0 +1 @@
+Generic single-database configuration.
\ No newline at end of file
diff --git a/src/mailman/database/alembic/env.py b/src/mailman/database/alembic/env.py
new file mode 100644
index 000000000..8c5e0b342
--- /dev/null
+++ b/src/mailman/database/alembic/env.py
@@ -0,0 +1,72 @@
+from __future__ import with_statement
+from alembic import context
+from sqlalchemy import create_engine, pool
+from logging.config import fileConfig
+
+from mailman.config import config
+from mailman.utilities.string import expand
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+alembic_config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(alembic_config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = None
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = expand(config.database.url, config.paths)
+    context.configure(url=url, target_metadata=target_metadata)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    url = expand(config.database.url, config.paths)
+    engine = create_engine(url)
+    connection = engine.connect()
+    context.configure(
+        connection=connection,
+        target_metadata=target_metadata
+    )
+
+    try:
+        with context.begin_transaction():
+            context.run_migrations()
+    finally:
+        connection.close()
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()
diff --git a/src/mailman/database/alembic/script.py.mako b/src/mailman/database/alembic/script.py.mako
new file mode 100644
index 000000000..95702017e
--- /dev/null
+++ b/src/mailman/database/alembic/script.py.mako
@@ -0,0 +1,22 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}
diff --git a/src/mailman/database/factory.py b/src/mailman/database/factory.py
index c06f75031..01a1357dd 100644
--- a/src/mailman/database/factory.py
+++ b/src/mailman/database/factory.py
@@ -29,6 +29,9 @@ __all__ = [
 import os
 import types
 
+from alembic.config import Config
+from alembic import command
+
 from flufl.lock import Lock
 from zope.interface import implementer
 from zope.interface.verify import verifyObject
@@ -39,6 +42,8 @@ from mailman.interfaces.database import IDatabase, IDatabaseFactory
 from mailman.utilities.modules import call_name
 
 
+alembic_cfg = Config("./alembic.ini")
+
 
 @implementer(IDatabaseFactory)
 class DatabaseFactory:
@@ -53,6 +58,7 @@ class DatabaseFactory:
             verifyObject(IDatabase, database)
             database.initialize()
             Model.metadata.create_all(database.engine)
+            command.stamp(alembic_cfg, "head")
             database.commit()
             return database
 
@@ -81,6 +87,7 @@ class DatabaseTestingFactory:
         verifyObject(IDatabase, database)
         database.initialize()
         Model.metadata.create_all(database.engine)
+        command.stamp(alembic_cfg, "head")
         database.commit()
         # Make _reset() a bound method of the database instance.
         database._reset = types.MethodType(_reset, database)
diff --git a/src/mailman/testing/layers.py b/src/mailman/testing/layers.py
index eb51e309f..3d2a50782 100644
--- a/src/mailman/testing/layers.py
+++ b/src/mailman/testing/layers.py
@@ -190,6 +190,10 @@ class ConfigLayer(MockAndMonkeyLayer):
     @classmethod
     def tearDown(cls):
         assert cls.var_dir is not None, 'Layer not set up'
+        # Reset the test database after the tests are done so that there
+        # is no data incase the tests are rerun with a database layer like
+        # mysql or postgresql which are not deleted in teardown.
+        reset_the_world()
         config.pop('test config')
         shutil.rmtree(cls.var_dir)
         cls.var_dir = None
-- 
cgit v1.2.3-70-g09d2


From 7d8c36da200d87a37250d4300b50a98602130614 Mon Sep 17 00:00:00 2001
From: Abhilash Raj
Date: Wed, 24 Sep 2014 23:15:48 +0530
Subject: added license block for the new file

---
 src/mailman/database/alembic/README |  1 -
 src/mailman/database/alembic/env.py | 18 ++++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)
 delete mode 100644 src/mailman/database/alembic/README

(limited to 'src')

diff --git a/src/mailman/database/alembic/README b/src/mailman/database/alembic/README
deleted file mode 100644
index 98e4f9c44..000000000
--- a/src/mailman/database/alembic/README
+++ /dev/null
@@ -1 +0,0 @@
-Generic single-database configuration.
\ No newline at end of file
diff --git a/src/mailman/database/alembic/env.py b/src/mailman/database/alembic/env.py
index 8c5e0b342..68d967b7e 100644
--- a/src/mailman/database/alembic/env.py
+++ b/src/mailman/database/alembic/env.py
@@ -1,4 +1,22 @@
+# Copyright (C) 2006-2014 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
 from __future__ import with_statement
+
 from alembic import context
 from sqlalchemy import create_engine, pool
 from logging.config import fileConfig
-- 
cgit v1.2.3-70-g09d2


From e0db04cafc3e3d357cb6ce40fcfc7b5d59c1f717 Mon Sep 17 00:00:00 2001
From: Abhilash Raj
Date: Wed, 24 Sep 2014 23:30:43 +0530
Subject: no need to stamp the testing db

---
 src/mailman/database/factory.py | 1 -
 1 file changed, 1 deletion(-)

(limited to 'src')

diff --git a/src/mailman/database/factory.py b/src/mailman/database/factory.py
index 01a1357dd..d4857866a 100644
--- a/src/mailman/database/factory.py
+++ b/src/mailman/database/factory.py
@@ -87,7 +87,6 @@ class DatabaseTestingFactory:
         verifyObject(IDatabase, database)
         database.initialize()
         Model.metadata.create_all(database.engine)
-        command.stamp(alembic_cfg, "head")
         database.commit()
         # Make _reset() a bound method of the database instance.
         database._reset = types.MethodType(_reset, database)
-- 
cgit v1.2.3-70-g09d2


From 32d118329488df775cd74dad2907ed496022f757 Mon Sep 17 00:00:00 2001
From: Abhilash Raj
Date: Thu, 25 Sep 2014 00:47:08 +0530
Subject: add new command `mailman migrate` to migrate the new schema on the
 old database

---
 src/mailman/commands/cli_migrate.py      | 51 ++++++++++++++++++++++++++++++++
 src/mailman/config/schema.cfg            |  3 ++
 src/mailman/database/alembic/__init__.py |  0
 src/mailman/database/alembic/env.py      | 25 +++++-----------
 4 files changed, 61 insertions(+), 18 deletions(-)
 create mode 100644 src/mailman/commands/cli_migrate.py
 create mode 100644 src/mailman/database/alembic/__init__.py

(limited to 'src')

diff --git a/src/mailman/commands/cli_migrate.py b/src/mailman/commands/cli_migrate.py
new file mode 100644
index 000000000..6593ea832
--- /dev/null
+++ b/src/mailman/commands/cli_migrate.py
@@ -0,0 +1,51 @@
+# Copyright (C) 2010-2014 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""bin/mailman migrate."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    'Migrate',
+]
+
+from alembic import command
+from alembic.config import Config
+from zope.interface import implementer
+
+from mailman.config import config
+from mailman.core.i18n import _
+from mailman.interfaces.command import ICLISubCommand
+
+
+@implementer(ICLISubCommand)
+class Migrate:
+    """Migrate the mailman database to the schema."""
+
+    name = 'migrate'
+
+    def add(self, parser, comman_parser):
+        """See `ICLISubCommand`."""
+        pass
+
+    def process(self, args):
+        alembic_cfg= Config()
+        alembic_cfg.set_main_option(
+            "script_location", config.alembic['script_location'])
+        command.upgrade(alembic_cfg, "head")
+        print("Updated the database schema.")
diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg
index d7508a533..23382721c 100644
--- a/src/mailman/config/schema.cfg
+++ b/src/mailman/config/schema.cfg
@@ -640,3 +640,6 @@ rewrite_duplicate_headers:
     CC X-Original-CC
     Content-Transfer-Encoding X-Original-Content-Transfer-Encoding
     MIME-Version X-MIME-Version
+
+[alembic]
+script_location:  src/mailman/database/alembic
diff --git a/src/mailman/database/alembic/__init__.py b/src/mailman/database/alembic/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/mailman/database/alembic/env.py b/src/mailman/database/alembic/env.py
index 68d967b7e..002b11aa1 100644
--- a/src/mailman/database/alembic/env.py
+++ b/src/mailman/database/alembic/env.py
@@ -18,30 +18,15 @@
 from __future__ import with_statement
 
 from alembic import context
+from alembic.config import Config
 from sqlalchemy import create_engine, pool
 from logging.config import fileConfig
 
 from mailman.config import config
 from mailman.utilities.string import expand
+from mailman.database.model import Model
 
-# this is the Alembic Config object, which provides
-# access to the values within the .ini file in use.
-alembic_config = context.config
-
-# Interpret the config file for Python logging.
-# This line sets up loggers basically.
-fileConfig(alembic_config.config_file_name)
-
-# add your model's MetaData object here
-# for 'autogenerate' support
-# from myapp import mymodel
-# target_metadata = mymodel.Base.metadata
-target_metadata = None
-
-# other values from the config, defined by the needs of env.py,
-# can be acquired:
-# my_important_option = config.get_main_option("my_important_option")
-# ... etc.
+target_metadata = Model.metadata
 
 
 def run_migrations_offline():
@@ -70,6 +55,10 @@ def run_migrations_online():
     and associate a connection with the context.
 
     """
+    alembic_cfg= Config()
+    alembic_cfg.set_main_option(
+        "script_location", config.alembic['script_location'])
+
     url = expand(config.database.url, config.paths)
     engine = create_engine(url)
     connection = engine.connect()
-- 
cgit v1.2.3-70-g09d2


From 03647b16eb75cc841bb15c3c48ac5f18f77118b8 Mon Sep 17 00:00:00 2001
From: Abhilash Raj
Date: Thu, 25 Sep 2014 02:53:09 +0530
Subject: add autogenerate switch that generates to create migration scripts
 automatically

---
 src/mailman/commands/cli_migrate.py | 13 ++++++++++---
 src/mailman/database/alembic/env.py |  3 +--
 2 files changed, 11 insertions(+), 5 deletions(-)

(limited to 'src')

diff --git a/src/mailman/commands/cli_migrate.py b/src/mailman/commands/cli_migrate.py
index 6593ea832..85fd07bd4 100644
--- a/src/mailman/commands/cli_migrate.py
+++ b/src/mailman/commands/cli_migrate.py
@@ -39,13 +39,20 @@ class Migrate:
 
     name = 'migrate'
 
-    def add(self, parser, comman_parser):
+    def add(self, parser, command_parser):
         """See `ICLISubCommand`."""
+        command_parser.add_argument(
+            '-a', '--autogenerate',
+            action="store_true", help=_("""\
+            Autogenerate the migration script using alembic"""))
         pass
 
     def process(self, args):
         alembic_cfg= Config()
         alembic_cfg.set_main_option(
             "script_location", config.alembic['script_location'])
-        command.upgrade(alembic_cfg, "head")
-        print("Updated the database schema.")
+        if args.autogenerate:
+            command.revision(alembic_cfg, autogenerate=True)
+        else:
+            command.upgrade(alembic_cfg, "head")
+            print("Updated the database schema.")
diff --git a/src/mailman/database/alembic/env.py b/src/mailman/database/alembic/env.py
index 002b11aa1..11ea8f6da 100644
--- a/src/mailman/database/alembic/env.py
+++ b/src/mailman/database/alembic/env.py
@@ -20,7 +20,6 @@ from __future__ import with_statement
 from alembic import context
 from alembic.config import Config
 from sqlalchemy import create_engine, pool
-from logging.config import fileConfig
 
 from mailman.config import config
 from mailman.utilities.string import expand
@@ -58,9 +57,9 @@ def run_migrations_online():
     alembic_cfg= Config()
     alembic_cfg.set_main_option(
         "script_location", config.alembic['script_location'])
-
     url = expand(config.database.url, config.paths)
     engine = create_engine(url)
+
     connection = engine.connect()
     context.configure(
         connection=connection,
-- 
cgit v1.2.3-70-g09d2