Merge Abilash's branch

6 files changed, 123 insertions, 148 deletions

diff --git a/src/mailman/database/base.py b/src/mailman/database/base.py
index cbf88a4ff..f379b3124 100644
--- a/src/mailman/database/base.py
+++ b/src/mailman/database/base.py
@@ -29,8 +29,9 @@ import logging
 
 from lazr.config import as_boolean
 from pkg_resources import resource_listdir, resource_string
-from storm.cache import GenerationalCache
-from storm.locals import create_database, Store
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.orm.session import Session
 from zope.interface import implementer
 
 from mailman.config import config
@@ -45,23 +46,24 @@ NL = '\n'
 
 
 @implementer(IDatabase)
-class StormBaseDatabase:
-    """The database base class for use with the Storm ORM.
+class SABaseDatabase:
+    """The database base class for use with SQLAlchemy.
 
-    Use this as a base class for your DB-specific derived classes.
+    Use this as a base class for your DB-Specific derived classes.
     """
-
     # Tag used to distinguish the database being used.  Override this in base
     # classes.
+
     TAG = ''
 
     def __init__(self):
         self.url = None
         self.store = None
+        self.transaction = None
 
     def begin(self):
         """See `IDatabase`."""
-        # Storm takes care of this for us.
+        # SA does this for us.
         pass
 
     def commit(self):
@@ -100,18 +102,9 @@ class StormBaseDatabase:
         """
         pass
 
-    def _prepare(self, url):
-        """Prepare the database for creation.
-
-        Some database backends need to do so me prep work before letting Storm
-        create the database.  For example, we have to touch the SQLite .db
-        file first so that it has the proper file modes.
-        """
-        pass
-
     def initialize(self, debug=None):
-        """See `IDatabase`."""
-        # Calculate the engine url.
+        """See `IDatabase`"""
+        # Calculate the engine url
         url = expand(config.database.url, config.paths)
         log.debug('Database url: %s', url)
         # XXX By design of SQLite, database file creation does not honor
@@ -129,13 +122,10 @@ class StormBaseDatabase:
         # engines, and yes, we could have chmod'd the file after the fact, but
         # half dozen and all...
         self.url = url
-        self._prepare(url)
-        database = create_database(url)
-        store = Store(database, GenerationalCache())
-        database.DEBUG = (as_boolean(config.database.debug)
-                          if debug is None else debug)
-        self.store = store
-        store.commit()
+        self.engine = create_engine(url)
+        session = sessionmaker(bind=self.engine)
+        self.store = session()
+        self.store.commit()
 
     def load_migrations(self, until=None):
         """Load schema migrations.
@@ -144,45 +134,8 @@ class StormBaseDatabase:
             With default value of None, load all migrations.
         :type until: string
         """
-        migrations_path = config.database.migrations_path
-        if '.' in migrations_path:
-            parent, dot, child = migrations_path.rpartition('.')
-        else:
-            parent = migrations_path
-            child = ''
-        # If the database does not yet exist, load the base schema.
-        filenames = sorted(resource_listdir(parent, child))
-        # Find out which schema migrations have already been loaded.
-        if self._database_exists(self.store):
-            versions = set(version.version for version in
-                           self.store.find(Version, component='schema'))
-        else:
-            versions = set()
-        for filename in filenames:
-            module_fn, extension = os.path.splitext(filename)
-            if extension != '.py':
-                continue
-            parts = module_fn.split('_')
-            if len(parts) < 2:
-                continue
-            version = parts[1].strip()
-            if len(version) == 0:
-                # Not a schema migration file.
-                continue
-            if version in versions:
-                log.debug('already migrated to %s', version)
-                continue
-            if until is not None and version > until:
-                # We're done.
-                break
-            module_path = migrations_path + '.' + module_fn
-            __import__(module_path)
-            upgrade = getattr(sys.modules[module_path], 'upgrade', None)
-            if upgrade is None:
-                continue
-            log.debug('migrating db to %s: %s', version, module_path)
-            upgrade(self, self.store, version, module_path)
-        self.commit()
+        from mailman.database.model import Model
+        Model.metadata.create_all(self.engine)
 
     def load_sql(self, store, sql):
         """Load the given SQL into the store.
@@ -200,29 +153,6 @@ class StormBaseDatabase:
             if statement.strip() != '':
                 store.execute(statement + ';')
 
-    def load_schema(self, store, version, filename, module_path):
-        """Load the schema from a file.
-
-        This is a helper method for migration classes to call.
-
-        :param store: The Storm store to load the schema into.
-        :type store: storm.locals.Store`
-        :param version: The schema version identifier of the form
-            YYYYMMDDHHMMSS.
-        :type version: string
-        :param filename: The file name containing the schema to load.  Pass
-            `None` if there is no schema file to load.
-        :type filename: string
-        :param module_path: The fully qualified Python module path to the
-            migration module being loaded.  This is used to record information
-            for use by the test suite.
-        :type module_path: string
-        """
-        if filename is not None:
-            contents = resource_string('mailman.database.schema', filename)
-            self.load_sql(store, contents)
-        # Add a marker that indicates the migration version being applied.
-        store.add(Version(component='schema', version=version))
 
     @staticmethod
     def _make_temporary():
diff --git a/src/mailman/database/factory.py b/src/mailman/database/factory.py
index db453ea41..64fcc242c 100644
--- a/src/mailman/database/factory.py
+++ b/src/mailman/database/factory.py
@@ -62,10 +62,10 @@ class DatabaseFactory:
 
 def _reset(self):
     """See `IDatabase`."""
-    from mailman.database.model import ModelMeta
+    from mailman.database.model import Model
     self.store.rollback()
     self._pre_reset(self.store)
-    ModelMeta._reset(self.store)
+    Model._reset(self)
     self._post_reset(self.store)
     self.store.commit()
 
diff --git a/src/mailman/database/model.py b/src/mailman/database/model.py
index ba2d39213..d86ebb80e 100644
--- a/src/mailman/database/model.py
+++ b/src/mailman/database/model.py
@@ -25,44 +25,24 @@ __all__ = [
     ]
 
 
+import contextlib
 from operator import attrgetter
 
-from storm.properties import PropertyPublisherMeta
+from sqlalchemy.ext.declarative import declarative_base
 
+from mailman.config import config
 
-
-class ModelMeta(PropertyPublisherMeta):
+class ModelMeta(object):
     """Do more magic on table classes."""
 
-    _class_registry = set()
-
-    def __init__(self, name, bases, dict):
-        # Before we let the base class do it's thing, force an __storm_table__
-        # property to enforce our table naming convention.
-        self.__storm_table__ = name.lower()
-        super(ModelMeta, self).__init__(name, bases, dict)
-        # Register the model class so that it can be more easily cleared.
-        # This is required by the test framework so that the corresponding
-        # table can be reset between tests.
-        #
-        # The PRESERVE flag indicates whether the table should be reset or
-        # not.  We have to handle the actual Model base class explicitly
-        # because it does not correspond to a table in the database.
-        if not getattr(self, 'PRESERVE', False) and name != 'Model':
-            ModelMeta._class_registry.add(self)
-
     @staticmethod
-    def _reset(store):
-        from mailman.config import config
-        config.db._pre_reset(store)
-        # Make sure this is deterministic, by sorting on the storm table name.
-        classes = sorted(ModelMeta._class_registry,
-                         key=attrgetter('__storm_table__'))
-        for model_class in classes:
-            store.find(model_class).remove()
-
+    def _reset(db):
+        meta = Model.metadata
+        engine = config.db.engine
+        with contextlib.closing(engine.connect()) as con:
+            trans = con.begin()
+            for table in reversed(meta.sorted_tables):
+                con.execute(table.delete())
+            trans.commit()
 
-
-class Model:
-    """Like Storm's `Storm` subclass, but with a bit extra."""
-    __metaclass__ = ModelMeta
+Model = declarative_base(cls=ModelMeta)
diff --git a/src/mailman/database/postgresql.py b/src/mailman/database/postgresql.py
index 48c68a937..1ee454074 100644
--- a/src/mailman/database/postgresql.py
+++ b/src/mailman/database/postgresql.py
@@ -32,12 +32,12 @@ from functools import partial
 from operator import attrgetter
 from urlparse import urlsplit, urlunsplit
 
-from mailman.database.base import StormBaseDatabase
+from mailman.database.base import SABaseDatabase
 from mailman.testing.helpers import configuration
 
 
 
-class PostgreSQLDatabase(StormBaseDatabase):
+class PostgreSQLDatabase(SABaseDatabase):
     """Database class for PostgreSQL."""
 
     TAG = 'postgres'
diff --git a/src/mailman/database/sqlite.py b/src/mailman/database/sqlite.py
index 15629615f..0594d9091 100644
--- a/src/mailman/database/sqlite.py
+++ b/src/mailman/database/sqlite.py
@@ -34,12 +34,12 @@ import tempfile
 from functools import partial
 from urlparse import urlparse
 
-from mailman.database.base import StormBaseDatabase
+from mailman.database.base import SABaseDatabase
 from mailman.testing.helpers import configuration
 
 
 
-class SQLiteDatabase(StormBaseDatabase):
+class SQLiteDatabase(SABaseDatabase):
     """Database class for SQLite."""
 
     TAG = 'sqlite'
@@ -72,7 +72,7 @@ def _cleanup(self, tempdir):
 def make_temporary(database):
     """Adapts by monkey patching an existing SQLite IDatabase."""
     tempdir = tempfile.mkdtemp()
-    url = 'sqlite:///' + os.path.join(tempdir, 'mailman.db')
+    url = 'sqlite:///'   + os.path.join(tempdir, 'mailman.db')
     with configuration('database', url=url):
         database.initialize()
     database._cleanup = types.MethodType(
diff --git a/src/mailman/database/types.py b/src/mailman/database/types.py
index ba3d92df4..a6f0b32ca 100644
--- a/src/mailman/database/types.py
+++ b/src/mailman/database/types.py
@@ -23,43 +23,108 @@ from __future__ import absolute_import, print_function, unicode_literals
 __metaclass__ = type
 __all__ = [
     'Enum',
+    'UUID',
     ]
 
+import uuid
 
-from storm.properties import SimpleProperty
-from storm.variables import Variable
+from sqlalchemy import Integer
+from sqlalchemy.types import TypeDecorator, BINARY, CHAR
+from sqlalchemy.dialects import postgresql
 
 
 
-class _EnumVariable(Variable):
-    """Storm variable for supporting enum types.
-
-    To use this, make the database column a INTEGER.
+class Enum(TypeDecorator):
+    """
+    Stores an integer-based Enum as an integer in the database, and converts it
+    on-the-fly.
     """
 
-    def __init__(self, *args, **kws):
-        self._enum = kws.pop('enum')
-        super(_EnumVariable, self).__init__(*args, **kws)
+    impl = Integer
 
-    def parse_set(self, value, from_db):
+    def __init__(self, *args, **kw):
+        self.enum = kw.pop("enum")
+        TypeDecorator.__init__(self, *args, **kw)
+
+    def process_bind_param(self, value, dialect):
         if value is None:
             return None
-        if not from_db:
-            return value
-        return self._enum(value)
 
-    def parse_get(self, value, to_db):
+        return value.value
+
+
+    def process_result_value(self, value, dialect):
         if value is None:
             return None
-        if not to_db:
+        return self.enum(value)
+
+
+
+class UUID(TypeDecorator):
+    """
+    Stores a UUID in the database natively when it can and falls back to
+    a BINARY(16) or a CHAR(32) when it can't.
+
+    ::
+
+        from sqlalchemy_utils import UUIDType
+        import uuid
+
+        class User(Base):
+            __tablename__ = 'user'
+
+            # Pass `binary=False` to fallback to CHAR instead of BINARY
+            id = sa.Column(UUIDType(binary=False), primary_key=True)
+    """
+    impl = BINARY(16)
+
+    python_type = uuid.UUID
+
+    def __init__(self, binary=True, native=True):
+        """
+        :param binary: Whether to use a BINARY(16) or CHAR(32) fallback.
+        """
+        self.binary = binary
+        self.native = native
+
+    def load_dialect_impl(self, dialect):
+        if dialect.name == 'postgresql' and self.native:
+            # Use the native UUID type.
+            return dialect.type_descriptor(postgresql.UUID())
+
+        else:
+            # Fallback to either a BINARY or a CHAR.
+            kind = self.impl if self.binary else CHAR(32)
+            return dialect.type_descriptor(kind)
+
+    @staticmethod
+    def _coerce(value):
+        if value and not isinstance(value, uuid.UUID):
+            try:
+                value = uuid.UUID(value)
+
+            except (TypeError, ValueError):
+                value = uuid.UUID(bytes=value)
+
+        return value
+
+    def process_bind_param(self, value, dialect):
+        if value is None:
             return value
-        return value.value
 
+        if not isinstance(value, uuid.UUID):
+            value = self._coerce(value)
+
+        if self.native and dialect.name == 'postgresql':
+            return str(value)
 
-class Enum(SimpleProperty):
-    """Custom type for Storm supporting enums."""
+        return value.bytes if self.binary else value.hex
+
+    def process_result_value(self, value, dialect):
+        if value is None:
+            return value
 
-    variable_class = _EnumVariable
+        if self.native and dialect.name == 'postgresql':
+            return uuid.UUID(value)
 
-    def __init__(self, enum=None):
-        super(Enum, self).__init__(enum=enum)
+        return uuid.UUID(bytes=value) if self.binary else uuid.UUID(value)