summaryrefslogtreecommitdiff
path: root/src/mailman/database
diff options
context:
space:
mode:
authorBarry Warsaw2014-11-01 12:49:15 -0400
committerBarry Warsaw2014-11-01 12:49:15 -0400
commit8ab9c5111a05277e185b5e038bf12e13cd6df15e (patch)
tree9307b9f2fb65a90bc4d61a2c97478b582a96de87 /src/mailman/database
parentb6bc505e45a2f1f4f99d7dd2cdd868d533270ee9 (diff)
parentfb38e482aa42edd4032a23e7c1f727066991fa62 (diff)
downloadmailman-8ab9c5111a05277e185b5e038bf12e13cd6df15e.tar.gz
mailman-8ab9c5111a05277e185b5e038bf12e13cd6df15e.tar.zst
mailman-8ab9c5111a05277e185b5e038bf12e13cd6df15e.zip
Diffstat (limited to 'src/mailman/database')
-rw-r--r--src/mailman/database/alembic/__init__.py (renamed from src/mailman/database/schema/mm_00000000000000_base.py)15
-rw-r--r--src/mailman/database/alembic/env.py75
-rw-r--r--src/mailman/database/alembic/script.py.mako22
-rw-r--r--src/mailman/database/alembic/versions/51b7f92bd06c_initial.py66
-rw-r--r--src/mailman/database/base.py143
-rw-r--r--src/mailman/database/docs/__init__.py0
-rw-r--r--src/mailman/database/docs/migration.rst207
-rw-r--r--src/mailman/database/factory.py88
-rw-r--r--src/mailman/database/model.py57
-rw-r--r--src/mailman/database/postgresql.py65
-rw-r--r--src/mailman/database/schema/__init__.py0
-rw-r--r--src/mailman/database/schema/helpers.py43
-rw-r--r--src/mailman/database/schema/mm_20120407000000.py212
-rw-r--r--src/mailman/database/schema/mm_20121015000000.py95
-rw-r--r--src/mailman/database/schema/mm_20130406000000.py65
-rw-r--r--src/mailman/database/schema/postgres.sql349
-rw-r--r--src/mailman/database/schema/sqlite.sql327
-rw-r--r--src/mailman/database/schema/sqlite_20120407000000_01.sql280
-rw-r--r--src/mailman/database/schema/sqlite_20121015000000_01.sql230
-rw-r--r--src/mailman/database/schema/sqlite_20130406000000_01.sql46
-rw-r--r--src/mailman/database/sqlite.py46
-rw-r--r--src/mailman/database/tests/data/__init__.py0
-rw-r--r--src/mailman/database/tests/data/mailman_01.dbbin48128 -> 0 bytes
-rw-r--r--src/mailman/database/tests/data/migration_postgres_1.sql133
-rw-r--r--src/mailman/database/tests/data/migration_sqlite_1.sql133
-rw-r--r--src/mailman/database/tests/test_factory.py160
-rw-r--r--src/mailman/database/tests/test_migrations.py506
-rw-r--r--src/mailman/database/types.py69
28 files changed, 494 insertions, 2938 deletions
diff --git a/src/mailman/database/schema/mm_00000000000000_base.py b/src/mailman/database/alembic/__init__.py
index ad085427f..ffd3af6df 100644
--- a/src/mailman/database/schema/mm_00000000000000_base.py
+++ b/src/mailman/database/alembic/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2014 by the Free Software Foundation, Inc.
+# Copyright (C) 2014 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
@@ -15,21 +15,18 @@
# You should have received a copy of the GNU General Public License along with
# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
-"""Load the base schema."""
+"""Alembic configuration initization."""
from __future__ import absolute_import, print_function, unicode_literals
__metaclass__ = type
__all__ = [
- 'upgrade',
+ 'alembic_cfg',
]
-VERSION = '00000000000000'
-_helper = None
+from alembic.config import Config
+from mailman.utilities.modules import expand_path
-
-def upgrade(database, store, version, module_path):
- filename = '{0}.sql'.format(database.TAG)
- database.load_schema(store, version, filename, module_path)
+alembic_cfg = Config(expand_path('python:mailman.config.alembic'))
diff --git a/src/mailman/database/alembic/env.py b/src/mailman/database/alembic/env.py
new file mode 100644
index 000000000..125868566
--- /dev/null
+++ b/src/mailman/database/alembic/env.py
@@ -0,0 +1,75 @@
+# Copyright (C) 2014 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
+
+"""Alembic migration environment."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+ 'run_migrations_offline',
+ 'run_migrations_online',
+ ]
+
+
+from alembic import context
+from contextlib import closing
+from sqlalchemy import create_engine
+
+from mailman.config import config
+from mailman.database.model import Model
+from mailman.utilities.string import expand
+
+
+
+def run_migrations_offline():
+ """Run migrations in 'offline' mode.
+
+ This configures the context with just a URL and not an Engine,
+ though an Engine is acceptable here as well. By skipping the Engine
+ creation we don't even need a DBAPI to be available.
+
+ Calls to context.execute() here emit the given string to the script
+ output.
+ """
+ url = expand(config.database.url, config.paths)
+ context.configure(url=url, target_metadata=Model.metadata)
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+def run_migrations_online():
+ """Run migrations in 'online' mode.
+
+ In this scenario we need to create an Engine and associate a
+ connection with the context.
+ """
+ url = expand(config.database.url, config.paths)
+ engine = create_engine(url)
+
+ connection = engine.connect()
+ with closing(connection):
+ context.configure(
+ connection=connection, target_metadata=Model.metadata)
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+if context.is_offline_mode():
+ run_migrations_offline()
+else:
+ run_migrations_online()
diff --git a/src/mailman/database/alembic/script.py.mako b/src/mailman/database/alembic/script.py.mako
new file mode 100644
index 000000000..95702017e
--- /dev/null
+++ b/src/mailman/database/alembic/script.py.mako
@@ -0,0 +1,22 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+def upgrade():
+ ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+ ${downgrades if downgrades else "pass"}
diff --git a/src/mailman/database/alembic/versions/51b7f92bd06c_initial.py b/src/mailman/database/alembic/versions/51b7f92bd06c_initial.py
new file mode 100644
index 000000000..3feb24fff
--- /dev/null
+++ b/src/mailman/database/alembic/versions/51b7f92bd06c_initial.py
@@ -0,0 +1,66 @@
+# Copyright (C) 2014 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
+
+"""Initial migration.
+
+This empty migration file makes sure there is always an alembic_version
+in the database. As a consequence, if the database version is reported
+as None, it means the database needs to be created from scratch with
+SQLAlchemy itself.
+
+It also removes schema items left over from Storm.
+
+Revision ID: 51b7f92bd06c
+Revises: None
+Create Date: 2014-10-10 09:53:35.624472
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+ 'downgrade',
+ 'upgrade',
+ ]
+
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# Revision identifiers, used by Alembic.
+revision = '51b7f92bd06c'
+down_revision = None
+
+
+def upgrade():
+ op.drop_table('version')
+ if op.get_bind().dialect.name != 'sqlite':
+ # SQLite does not support dropping columns.
+ op.drop_column('mailinglist', 'acceptable_aliases_id')
+ op.create_index(op.f('ix_user__user_id'), 'user',
+ ['_user_id'], unique=False)
+ op.drop_index('ix_user_user_id', table_name='user')
+
+
+def downgrade():
+ op.create_table('version')
+ op.create_index('ix_user_user_id', 'user', ['_user_id'], unique=False)
+ op.drop_index(op.f('ix_user__user_id'), table_name='user')
+ op.add_column(
+ 'mailinglist',
+ sa.Column('acceptable_aliases_id', sa.INTEGER(), nullable=True))
diff --git a/src/mailman/database/base.py b/src/mailman/database/base.py
index cbf88a4ff..2b86899bc 100644
--- a/src/mailman/database/base.py
+++ b/src/mailman/database/base.py
@@ -19,49 +19,39 @@ from __future__ import absolute_import, print_function, unicode_literals
__metaclass__ = type
__all__ = [
- 'StormBaseDatabase',
+ 'SABaseDatabase',
]
-import os
-import sys
import logging
-from lazr.config import as_boolean
-from pkg_resources import resource_listdir, resource_string
-from storm.cache import GenerationalCache
-from storm.locals import create_database, Store
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
from zope.interface import implementer
from mailman.config import config
from mailman.interfaces.database import IDatabase
-from mailman.model.version import Version
from mailman.utilities.string import expand
-log = logging.getLogger('mailman.config')
+log = logging.getLogger('mailman.database')
NL = '\n'
@implementer(IDatabase)
-class StormBaseDatabase:
- """The database base class for use with the Storm ORM.
+class SABaseDatabase:
+ """The database base class for use with SQLAlchemy.
- Use this as a base class for your DB-specific derived classes.
+ Use this as a base class for your DB-Specific derived classes.
"""
-
- # Tag used to distinguish the database being used. Override this in base
- # classes.
- TAG = ''
-
def __init__(self):
self.url = None
self.store = None
def begin(self):
"""See `IDatabase`."""
- # Storm takes care of this for us.
+ # SQLAlchemy does this for us.
pass
def commit(self):
@@ -72,16 +62,6 @@ class StormBaseDatabase:
"""See `IDatabase`."""
self.store.rollback()
- def _database_exists(self):
- """Return True if the database exists and is initialized.
-
- Return False when Mailman needs to create and initialize the
- underlying database schema.
-
- Base classes *must* override this.
- """
- raise NotImplementedError
-
def _pre_reset(self, store):
"""Clean up method for testing.
@@ -113,6 +93,7 @@ class StormBaseDatabase:
"""See `IDatabase`."""
# Calculate the engine url.
url = expand(config.database.url, config.paths)
+ self._prepare(url)
log.debug('Database url: %s', url)
# XXX By design of SQLite, database file creation does not honor
# umask. See their ticket #1193:
@@ -129,101 +110,13 @@ class StormBaseDatabase:
# engines, and yes, we could have chmod'd the file after the fact, but
# half dozen and all...
self.url = url
- self._prepare(url)
- database = create_database(url)
- store = Store(database, GenerationalCache())
- database.DEBUG = (as_boolean(config.database.debug)
- if debug is None else debug)
- self.store = store
- store.commit()
-
- def load_migrations(self, until=None):
- """Load schema migrations.
-
- :param until: Load only the migrations up to the specified timestamp.
- With default value of None, load all migrations.
- :type until: string
- """
- migrations_path = config.database.migrations_path
- if '.' in migrations_path:
- parent, dot, child = migrations_path.rpartition('.')
- else:
- parent = migrations_path
- child = ''
- # If the database does not yet exist, load the base schema.
- filenames = sorted(resource_listdir(parent, child))
- # Find out which schema migrations have already been loaded.
- if self._database_exists(self.store):
- versions = set(version.version for version in
- self.store.find(Version, component='schema'))
- else:
- versions = set()
- for filename in filenames:
- module_fn, extension = os.path.splitext(filename)
- if extension != '.py':
- continue
- parts = module_fn.split('_')
- if len(parts) < 2:
- continue
- version = parts[1].strip()
- if len(version) == 0:
- # Not a schema migration file.
- continue
- if version in versions:
- log.debug('already migrated to %s', version)
- continue
- if until is not None and version > until:
- # We're done.
- break
- module_path = migrations_path + '.' + module_fn
- __import__(module_path)
- upgrade = getattr(sys.modules[module_path], 'upgrade', None)
- if upgrade is None:
- continue
- log.debug('migrating db to %s: %s', version, module_path)
- upgrade(self, self.store, version, module_path)
- self.commit()
-
- def load_sql(self, store, sql):
- """Load the given SQL into the store.
-
- :param store: The Storm store to load the schema into.
- :type store: storm.locals.Store`
- :param sql: The possibly multi-line SQL to load.
- :type sql: string
- """
- # Discard all blank and comment lines.
- lines = (line for line in sql.splitlines()
- if line.strip() != '' and line.strip()[:2] != '--')
- sql = NL.join(lines)
- for statement in sql.split(';'):
- if statement.strip() != '':
- store.execute(statement + ';')
-
- def load_schema(self, store, version, filename, module_path):
- """Load the schema from a file.
-
- This is a helper method for migration classes to call.
-
- :param store: The Storm store to load the schema into.
- :type store: storm.locals.Store`
- :param version: The schema version identifier of the form
- YYYYMMDDHHMMSS.
- :type version: string
- :param filename: The file name containing the schema to load. Pass
- `None` if there is no schema file to load.
- :type filename: string
- :param module_path: The fully qualified Python module path to the
- migration module being loaded. This is used to record information
- for use by the test suite.
- :type module_path: string
- """
- if filename is not None:
- contents = resource_string('mailman.database.schema', filename)
- self.load_sql(store, contents)
- # Add a marker that indicates the migration version being applied.
- store.add(Version(component='schema', version=version))
+ self.engine = create_engine(url)
+ session = sessionmaker(bind=self.engine)
+ self.store = session()
+ self.store.commit()
- @staticmethod
- def _make_temporary():
- raise NotImplementedError
+ # XXX BAW Why doesn't model.py _reset() do this?
+ def destroy(self):
+ """Drop all database tables"""
+ from mailman.database.model import Model
+ Model.metadata.drop_all(self.engine)
diff --git a/src/mailman/database/docs/__init__.py b/src/mailman/database/docs/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--- a/src/mailman/database/docs/__init__.py
+++ /dev/null
diff --git a/src/mailman/database/docs/migration.rst b/src/mailman/database/docs/migration.rst
deleted file mode 100644
index fafdfaf26..000000000
--- a/src/mailman/database/docs/migration.rst
+++ /dev/null
@@ -1,207 +0,0 @@
-=================
-Schema migrations
-=================
-
-The SQL database schema will over time require upgrading to support new
-features. This is supported via schema migration.
-
-Migrations are embodied in individual Python classes, which themselves may
-load SQL into the database. The naming scheme for migration files is:
-
- mm_YYYYMMDDHHMMSS_comment.py
-
-where `YYYYMMDDHHMMSS` is a required numeric year, month, day, hour, minute,
-and second specifier providing unique ordering for processing. Only this
-component of the file name is used to determine the ordering. The prefix is
-required due to Python module naming requirements, but it is actually
-ignored. `mm_` is reserved for Mailman's own use.
-
-The optional `comment` part of the file name can be used as a short
-description for the migration, although comments and docstrings in the
-migration files should be used for more detailed descriptions.
-
-Migrations are applied automatically when Mailman starts up, but can also be
-applied at any time by calling in the API directly. Once applied, a
-migration's version string is registered so it will not be applied again.
-
-We see that the base migration, as well as subsequent standard migrations, are
-already applied.
-
- >>> from mailman.model.version import Version
- >>> results = config.db.store.find(Version, component='schema')
- >>> results.count()
- 4
- >>> versions = sorted(result.version for result in results)
- >>> for version in versions:
- ... print(version)
- 00000000000000
- 20120407000000
- 20121015000000
- 20130406000000
-
-
-Migrations
-==========
-
-Migrations can be loaded at any time, and can be found in the migrations path
-specified in the configuration file.
-
-.. Create a temporary directory for the migrations::
-
- >>> import os, sys, tempfile
- >>> tempdir = tempfile.mkdtemp()
- >>> path = os.path.join(tempdir, 'migrations')
- >>> os.makedirs(path)
- >>> sys.path.append(tempdir)
- >>> config.push('migrations', """
- ... [database]
- ... migrations_path: migrations
- ... """)
-
-.. Clean this up at the end of the doctest.
- >>> def cleanup():
- ... import shutil
- ... from mailman.config import config
- ... config.pop('migrations')
- ... shutil.rmtree(tempdir)
- >>> cleanups.append(cleanup)
-
-Here is an example migrations module. The key part of this interface is the
-``upgrade()`` method, which takes four arguments:
-
- * `database` - The database class, as derived from `StormBaseDatabase`
- * `store` - The Storm `Store` object.
- * `version` - The version string as derived from the migrations module's file
- name. This will include only the `YYYYMMDDHHMMSS` string.
- * `module_path` - The dotted module path to the migrations module, suitable
- for lookup in `sys.modules`.
-
-This migration module just adds a marker to the `version` table.
-
- >>> with open(os.path.join(path, '__init__.py'), 'w') as fp:
- ... pass
- >>> with open(os.path.join(path, 'mm_20159999000000.py'), 'w') as fp:
- ... print("""
- ... from __future__ import unicode_literals
- ... from mailman.model.version import Version
- ... def upgrade(database, store, version, module_path):
- ... v = Version(component='test', version=version)
- ... store.add(v)
- ... database.load_schema(store, version, None, module_path)
- ... """, file=fp)
-
-This will load the new migration, since it hasn't been loaded before.
-
- >>> config.db.load_migrations()
- >>> results = config.db.store.find(Version, component='schema')
- >>> for result in sorted(result.version for result in results):
- ... print(result)
- 00000000000000
- 20120407000000
- 20121015000000
- 20130406000000
- 20159999000000
- >>> test = config.db.store.find(Version, component='test').one()
- >>> print(test.version)
- 20159999000000
-
-Migrations will only be loaded once.
-
- >>> with open(os.path.join(path, 'mm_20159999000001.py'), 'w') as fp:
- ... print("""
- ... from __future__ import unicode_literals
- ... from mailman.model.version import Version
- ... _marker = 801
- ... def upgrade(database, store, version, module_path):
- ... global _marker
- ... # Pad enough zeros on the left to reach 14 characters wide.
- ... marker = '{0:=#014d}'.format(_marker)
- ... _marker += 1
- ... v = Version(component='test', version=marker)
- ... store.add(v)
- ... database.load_schema(store, version, None, module_path)
- ... """, file=fp)
-
-The first time we load this new migration, we'll get the 801 marker.
-
- >>> config.db.load_migrations()
- >>> results = config.db.store.find(Version, component='schema')
- >>> for result in sorted(result.version for result in results):
- ... print(result)
- 00000000000000
- 20120407000000
- 20121015000000
- 20130406000000
- 20159999000000
- 20159999000001
- >>> test = config.db.store.find(Version, component='test')
- >>> for marker in sorted(marker.version for marker in test):
- ... print(marker)
- 00000000000801
- 20159999000000
-
-We do not get an 802 marker because the migration has already been loaded.
-
- >>> config.db.load_migrations()
- >>> results = config.db.store.find(Version, component='schema')
- >>> for result in sorted(result.version for result in results):
- ... print(result)
- 00000000000000
- 20120407000000
- 20121015000000
- 20130406000000
- 20159999000000
- 20159999000001
- >>> test = config.db.store.find(Version, component='test')
- >>> for marker in sorted(marker.version for marker in test):
- ... print(marker)
- 00000000000801
- 20159999000000
-
-
-Partial upgrades
-================
-
-It's possible (mostly for testing purposes) to only do a partial upgrade, by
-providing a timestamp to `load_migrations()`. To demonstrate this, we add two
-additional migrations, intended to be applied in sequential order.
-
- >>> from shutil import copyfile
- >>> from mailman.testing.helpers import chdir
- >>> with chdir(path):
- ... copyfile('mm_20159999000000.py', 'mm_20159999000002.py')
- ... copyfile('mm_20159999000000.py', 'mm_20159999000003.py')
- ... copyfile('mm_20159999000000.py', 'mm_20159999000004.py')
-
-Now, only migrate to the ...03 timestamp.
-
- >>> config.db.load_migrations('20159999000003')
-
-You'll notice that the ...04 version is not present.
-
- >>> results = config.db.store.find(Version, component='schema')
- >>> for result in sorted(result.version for result in results):
- ... print(result)
- 00000000000000
- 20120407000000
- 20121015000000
- 20130406000000
- 20159999000000
- 20159999000001
- 20159999000002
- 20159999000003
-
-
-.. cleanup:
- Because the Version table holds schema migration data, it will not be
- cleaned up by the standard test suite. This is generally not a problem
- for SQLite since each test gets a new database file, but for PostgreSQL,
- this will cause migration.rst to fail on subsequent runs. So let's just
- clean up the database explicitly.
-
- >>> if config.db.TAG != 'sqlite':
- ... results = config.db.store.execute("""
- ... DELETE FROM version WHERE version.version >= '201299990000'
- ... OR version.component = 'test';
- ... """)
- ... config.db.commit()
diff --git a/src/mailman/database/factory.py b/src/mailman/database/factory.py
index db453ea41..64174449d 100644
--- a/src/mailman/database/factory.py
+++ b/src/mailman/database/factory.py
@@ -22,25 +22,32 @@ from __future__ import absolute_import, print_function, unicode_literals
__metaclass__ = type
__all__ = [
'DatabaseFactory',
- 'DatabaseTemporaryFactory',
'DatabaseTestingFactory',
]
import os
import types
+import alembic.command
+from alembic.migration import MigrationContext
+from alembic.script import ScriptDirectory
from flufl.lock import Lock
-from zope.component import getAdapter
+from sqlalchemy import MetaData
from zope.interface import implementer
from zope.interface.verify import verifyObject
from mailman.config import config
+from mailman.database.alembic import alembic_cfg
+from mailman.database.model import Model
from mailman.interfaces.database import (
- IDatabase, IDatabaseFactory, ITemporaryDatabase)
+ DatabaseError, IDatabase, IDatabaseFactory)
from mailman.utilities.modules import call_name
+LAST_STORM_SCHEMA_VERSION = '20130406000000'
+
+
@implementer(IDatabaseFactory)
class DatabaseFactory:
@@ -54,18 +61,69 @@ class DatabaseFactory:
database = call_name(database_class)
verifyObject(IDatabase, database)
database.initialize()
- database.load_migrations()
+ SchemaManager(database).setup_database()
database.commit()
return database
+class SchemaManager:
+ "Manage schema migrations."""
+
+ def __init__(self, database):
+ self._database = database
+ self._script = ScriptDirectory.from_config(alembic_cfg)
+
+ def _get_storm_schema_version(self):
+ metadata = MetaData()
+ metadata.reflect(bind=self._database.engine)
+ if 'version' not in metadata.tables:
+ # There are no Storm artifacts left.
+ return None
+ Version = metadata.tables['version']
+ last_version = self._database.store.query(Version.c.version).filter(
+ Version.c.component == 'schema'
+ ).order_by(Version.c.version.desc()).first()
+ # Don't leave open transactions or they will block any schema change.
+ self._database.commit()
+ return last_version
+
+ def setup_database(self):
+ context = MigrationContext.configure(self._database.store.connection())
+ current_rev = context.get_current_revision()
+ head_rev = self._script.get_current_head()
+ if current_rev == head_rev:
+ # We're already at the latest revision so there's nothing to do.
+ return head_rev
+ if current_rev is None:
+ # No Alembic information is available.
+ storm_version = self._get_storm_schema_version()
+ if storm_version is None:
+ # Initial database creation.
+ Model.metadata.create_all(self._database.engine)
+ self._database.commit()
+ alembic.command.stamp(alembic_cfg, 'head')
+ else:
+ # The database was previously managed by Storm.
+ if storm_version.version < LAST_STORM_SCHEMA_VERSION:
+ raise DatabaseError(
+ 'Upgrades skipping beta versions is not supported.')
+ # Run migrations to remove the Storm-specific table and upgrade
+ # to SQLAlchemy and Alembic.
+ alembic.command.upgrade(alembic_cfg, 'head')
+ elif current_rev != head_rev:
+ alembic.command.upgrade(alembic_cfg, 'head')
+ return head_rev
+
+
+
def _reset(self):
"""See `IDatabase`."""
- from mailman.database.model import ModelMeta
+ # Avoid a circular import at module level.
+ from mailman.database.model import Model
self.store.rollback()
self._pre_reset(self.store)
- ModelMeta._reset(self.store)
+ Model._reset(self)
self._post_reset(self.store)
self.store.commit()
@@ -81,24 +139,8 @@ class DatabaseTestingFactory:
database = call_name(database_class)
verifyObject(IDatabase, database)
database.initialize()
- database.load_migrations()
+ Model.metadata.create_all(database.engine)
database.commit()
# Make _reset() a bound method of the database instance.
database._reset = types.MethodType(_reset, database)
return database
-
-
-
-@implementer(IDatabaseFactory)
-class DatabaseTemporaryFactory:
- """Create a temporary database for some of the migration tests."""
-
- @staticmethod
- def create():
- """See `IDatabaseFactory`."""
- database_class_name = config.database['class']
- database = call_name(database_class_name)
- verifyObject(IDatabase, database)
- adapted_database = getAdapter(
- database, ITemporaryDatabase, database.TAG)
- return adapted_database
diff --git a/src/mailman/database/model.py b/src/mailman/database/model.py
index ba2d39213..a6056bf63 100644
--- a/src/mailman/database/model.py
+++ b/src/mailman/database/model.py
@@ -25,44 +25,33 @@ __all__ = [
]
-from operator import attrgetter
+import contextlib
-from storm.properties import PropertyPublisherMeta
+from mailman.config import config
+from sqlalchemy.ext.declarative import declarative_base
-
-class ModelMeta(PropertyPublisherMeta):
- """Do more magic on table classes."""
-
- _class_registry = set()
-
- def __init__(self, name, bases, dict):
- # Before we let the base class do it's thing, force an __storm_table__
- # property to enforce our table naming convention.
- self.__storm_table__ = name.lower()
- super(ModelMeta, self).__init__(name, bases, dict)
- # Register the model class so that it can be more easily cleared.
- # This is required by the test framework so that the corresponding
- # table can be reset between tests.
- #
- # The PRESERVE flag indicates whether the table should be reset or
- # not. We have to handle the actual Model base class explicitly
- # because it does not correspond to a table in the database.
- if not getattr(self, 'PRESERVE', False) and name != 'Model':
- ModelMeta._class_registry.add(self)
+class ModelMeta:
+ """The custom metaclass for all model base classes.
+ This is used in the test suite to quickly reset the database after each
+ test. It works by iterating over all the tables, deleting each. The test
+ suite will then recreate the tables before each test.
+ """
@staticmethod
- def _reset(store):
- from mailman.config import config
- config.db._pre_reset(store)
- # Make sure this is deterministic, by sorting on the storm table name.
- classes = sorted(ModelMeta._class_registry,
- key=attrgetter('__storm_table__'))
- for model_class in classes:
- store.find(model_class).remove()
+ def _reset(db):
+ with contextlib.closing(config.db.engine.connect()) as connection:
+ transaction = connection.begin()
+ try:
+ # Delete all the tables in reverse foreign key dependency
+ # order. http://tinyurl.com/on8dy6f
+ for table in reversed(Model.metadata.sorted_tables):
+ connection.execute(table.delete())
+ except:
+ transaction.rollback()
+ raise
+ else:
+ transaction.commit()
-
-class Model:
- """Like Storm's `Storm` subclass, but with a bit extra."""
- __metaclass__ = ModelMeta
+Model = declarative_base(cls=ModelMeta)
diff --git a/src/mailman/database/postgresql.py b/src/mailman/database/postgresql.py
index 48c68a937..717b69dd1 100644
--- a/src/mailman/database/postgresql.py
+++ b/src/mailman/database/postgresql.py
@@ -22,34 +22,17 @@ from __future__ import absolute_import, print_function, unicode_literals
__metaclass__ = type
__all__ = [
'PostgreSQLDatabase',
- 'make_temporary',
]
-import types
-
-from functools import partial
-from operator import attrgetter
-from urlparse import urlsplit, urlunsplit
-
-from mailman.database.base import StormBaseDatabase
-from mailman.testing.helpers import configuration
+from mailman.database.base import SABaseDatabase
+from mailman.database.model import Model
-class PostgreSQLDatabase(StormBaseDatabase):
+class PostgreSQLDatabase(SABaseDatabase):
"""Database class for PostgreSQL."""
- TAG = 'postgres'
-
- def _database_exists(self, store):
- """See `BaseDatabase`."""
- table_query = ('SELECT table_name FROM information_schema.tables '
- "WHERE table_schema = 'public'")
- results = store.execute(table_query)
- table_names = set(item[0] for item in results)
- return 'version' in table_names
-
def _post_reset(self, store):
"""PostgreSQL-specific test suite cleanup.
@@ -57,49 +40,13 @@ class PostgreSQLDatabase(StormBaseDatabase):
restart from zero for new tests.
"""
super(PostgreSQLDatabase, self)._post_reset(store)
- from mailman.database.model import ModelMeta
- classes = sorted(ModelMeta._class_registry,
- key=attrgetter('__storm_table__'))
+ tables = reversed(Model.metadata.sorted_tables)
# Recipe adapted from
# http://stackoverflow.com/questions/544791/
# django-postgresql-how-to-reset-primary-key
- for model_class in classes:
+ for table in tables:
store.execute("""\
SELECT setval('"{0}_id_seq"', coalesce(max("id"), 1),
max("id") IS NOT null)
FROM "{0}";
- """.format(model_class.__storm_table__))
-
-
-
-# Test suite adapter for ITemporaryDatabase.
-
-def _cleanup(self, store, tempdb_name):
- from mailman.config import config
- store.rollback()
- store.close()
- # From the original database connection, drop the now unused database.
- config.db.store.execute('DROP DATABASE {0}'.format(tempdb_name))
-
-
-def make_temporary(database):
- """Adapts by monkey patching an existing PostgreSQL IDatabase."""
- from mailman.config import config
- parts = urlsplit(config.database.url)
- assert parts.scheme == 'postgres'
- new_parts = list(parts)
- new_parts[2] = '/mmtest'
- url = urlunsplit(new_parts)
- # Use the existing database connection to create a new testing
- # database.
- config.db.store.execute('ABORT;')
- config.db.store.execute('CREATE DATABASE mmtest;')
- with configuration('database', url=url):
- database.initialize()
- database._cleanup = types.MethodType(
- partial(_cleanup, store=database.store, tempdb_name='mmtest'),
- database)
- # bool column values in PostgreSQL.
- database.FALSE = 'False'
- database.TRUE = 'True'
- return database
+ """.format(table))
diff --git a/src/mailman/database/schema/__init__.py b/src/mailman/database/schema/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--- a/src/mailman/database/schema/__init__.py
+++ /dev/null
diff --git a/src/mailman/database/schema/helpers.py b/src/mailman/database/schema/helpers.py
deleted file mode 100644
index 827e6cc96..000000000
--- a/src/mailman/database/schema/helpers.py
+++ /dev/null
@@ -1,43 +0,0 @@
-# Copyright (C) 2013-2014 by the Free Software Foundation, Inc.
-#
-# This file is part of GNU Mailman.
-#
-# GNU Mailman is free software: you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by the Free
-# Software Foundation, either version 3 of the License, or (at your option)
-# any later version.
-#
-# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-# more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
-
-"""Schema migration helpers."""
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-__metaclass__ = type
-__all__ = [
- 'make_listid',
- ]
-
-
-
-def make_listid(fqdn_listname):
- """Turn a FQDN list name into a List-ID."""
- list_name, at, mail_host = fqdn_listname.partition('@')
- if at == '':
- # If there is no @ sign in the value, assume it already contains the
- # list-id.
- return fqdn_listname
- return '{0}.{1}'.format(list_name, mail_host)
-
-
-
-def pivot(store, table_name):
- """Pivot a backup table into the real table name."""
- store.execute('DROP TABLE {}'.format(table_name))
- store.execute('ALTER TABLE {0}_backup RENAME TO {0}'.format(table_name))
diff --git a/src/mailman/database/schema/mm_20120407000000.py b/src/mailman/database/schema/mm_20120407000000.py
deleted file mode 100644
index 1d798ea96..000000000
--- a/src/mailman/database/schema/mm_20120407000000.py
+++ /dev/null
@@ -1,212 +0,0 @@
-# Copyright (C) 2012-2014 by the Free Software Foundation, Inc.
-#
-# This file is part of GNU Mailman.
-#
-# GNU Mailman is free software: you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by the Free
-# Software Foundation, either version 3 of the License, or (at your option)
-# any later version.
-#
-# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-# more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
-
-"""3.0b1 -> 3.0b2 schema migrations.
-
-All column changes are in the `mailinglist` table.
-
-* Renames:
- - news_prefix_subject_too -> nntp_prefix_subject_too
- - news_moderation -> newsgroup_moderation
-
-* Collapsing:
- - archive, archive_private -> archive_policy
-
-* Remove:
- - archive_volume_frequency
- - generic_nonmember_action
- - nntp_host
-
-* Added:
- - list_id
-
-* Changes:
- member.mailing_list holds the list_id not the fqdn_listname
-
-See https://bugs.launchpad.net/mailman/+bug/971013 for details.
-"""
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-__metaclass__ = type
-__all__ = [
- 'upgrade',
- ]
-
-
-from mailman.database.schema.helpers import pivot
-from mailman.interfaces.archiver import ArchivePolicy
-
-
-VERSION = '20120407000000'
-
-
-
-def upgrade(database, store, version, module_path):
- if database.TAG == 'sqlite':
- upgrade_sqlite(database, store, version, module_path)
- else:
- upgrade_postgres(database, store, version, module_path)
-
-
-
-def archive_policy(archive, archive_private):
- """Convert archive and archive_private to archive_policy."""
- if archive == 0:
- return ArchivePolicy.never.value
- elif archive_private == 1:
- return ArchivePolicy.private.value
- else:
- return ArchivePolicy.public.value
-
-
-
-def upgrade_sqlite(database, store, version, module_path):
- # Load the first part of the migration. This creates a temporary table to
- # hold the new mailinglist table columns. The problem is that some of the
- # changes must be performed in Python, so after the first part is loaded,
- # we do the Python changes, drop the old mailing list table, and then
- # rename the temporary table to its place.
- database.load_schema(
- store, version, 'sqlite_{0}_01.sql'.format(version), module_path)
- results = store.execute("""
- SELECT id, include_list_post_header,
- news_prefix_subject_too, news_moderation,
- archive, archive_private, list_name, mail_host
- FROM mailinglist;
- """)
- for value in results:
- (id, list_post,
- news_prefix, news_moderation,
- archive, archive_private,
- list_name, mail_host) = value
- # Figure out what the new archive_policy column value should be.
- list_id = '{0}.{1}'.format(list_name, mail_host)
- fqdn_listname = '{0}@{1}'.format(list_name, mail_host)
- store.execute("""
- UPDATE mailinglist_backup SET
- allow_list_posts = {0},
- newsgroup_moderation = {1},
- nntp_prefix_subject_too = {2},
- archive_policy = {3},
- list_id = '{4}'
- WHERE id = {5};
- """.format(
- list_post,
- news_moderation,
- news_prefix,
- archive_policy(archive, archive_private),
- list_id,
- id))
- # Also update the member.mailing_list column to hold the list_id
- # instead of the fqdn_listname.
- store.execute("""
- UPDATE member SET
- mailing_list = '{0}'
- WHERE mailing_list = '{1}';
- """.format(list_id, fqdn_listname))
- # Pivot the backup table to the real thing.
- pivot(store, 'mailinglist')
- # Now add some indexes that were previously missing.
- store.execute(
- 'CREATE INDEX ix_mailinglist_list_id ON mailinglist (list_id);')
- store.execute(
- 'CREATE INDEX ix_mailinglist_fqdn_listname '
- 'ON mailinglist (list_name, mail_host);')
- # Now, do the member table.
- results = store.execute('SELECT id, mailing_list FROM member;')
- for id, mailing_list in results:
- list_name, at, mail_host = mailing_list.partition('@')
- if at == '':
- list_id = mailing_list
- else:
- list_id = '{0}.{1}'.format(list_name, mail_host)
- store.execute("""
- UPDATE member_backup SET list_id = '{0}'
- WHERE id = {1};
- """.format(list_id, id))
- # Pivot the backup table to the real thing.
- pivot(store, 'member')
-
-
-
-def upgrade_postgres(database, store, version, module_path):
- # Get the old values from the mailinglist table.
- results = store.execute("""
- SELECT id, archive, archive_private, list_name, mail_host
- FROM mailinglist;
- """)
- # Do the simple renames first.
- store.execute("""
- ALTER TABLE mailinglist
- RENAME COLUMN news_prefix_subject_too TO nntp_prefix_subject_too;
- """)
- store.execute("""
- ALTER TABLE mailinglist
- RENAME COLUMN news_moderation TO newsgroup_moderation;
- """)
- store.execute("""
- ALTER TABLE mailinglist
- RENAME COLUMN include_list_post_header TO allow_list_posts;
- """)
- # Do the easy column drops next.
- for column in ('archive_volume_frequency',
- 'generic_nonmember_action',
- 'nntp_host'):
- store.execute(
- 'ALTER TABLE mailinglist DROP COLUMN {0};'.format(column))
- # Now do the trickier collapsing of values. Add the new columns.
- store.execute('ALTER TABLE mailinglist ADD COLUMN archive_policy INTEGER;')
- store.execute('ALTER TABLE mailinglist ADD COLUMN list_id TEXT;')
- # Query the database for the old values of archive and archive_private in
- # each column. Then loop through all the results and update the new
- # archive_policy from the old values.
- for value in results:
- id, archive, archive_private, list_name, mail_host = value
- list_id = '{0}.{1}'.format(list_name, mail_host)
- store.execute("""
- UPDATE mailinglist SET
- archive_policy = {0},
- list_id = '{1}'
- WHERE id = {2};
- """.format(archive_policy(archive, archive_private), list_id, id))
- # Now drop the old columns.
- for column in ('archive', 'archive_private'):
- store.execute(
- 'ALTER TABLE mailinglist DROP COLUMN {0};'.format(column))
- # Now add some indexes that were previously missing.
- store.execute(
- 'CREATE INDEX ix_mailinglist_list_id ON mailinglist (list_id);')
- store.execute(
- 'CREATE INDEX ix_mailinglist_fqdn_listname '
- 'ON mailinglist (list_name, mail_host);')
- # Now, do the member table.
- results = store.execute('SELECT id, mailing_list FROM member;')
- store.execute('ALTER TABLE member ADD COLUMN list_id TEXT;')
- for id, mailing_list in results:
- list_name, at, mail_host = mailing_list.partition('@')
- if at == '':
- list_id = mailing_list
- else:
- list_id = '{0}.{1}'.format(list_name, mail_host)
- store.execute("""
- UPDATE member SET list_id = '{0}'
- WHERE id = {1};
- """.format(list_id, id))
- store.execute('ALTER TABLE member DROP COLUMN mailing_list;')
- # Record the migration in the version table.
- database.load_schema(store, version, None, module_path)
diff --git a/src/mailman/database/schema/mm_20121015000000.py b/src/mailman/database/schema/mm_20121015000000.py
deleted file mode 100644
index 84510ff57..000000000
--- a/src/mailman/database/schema/mm_20121015000000.py
+++ /dev/null
@@ -1,95 +0,0 @@
-# Copyright (C) 2012-2014 by the Free Software Foundation, Inc.
-#
-# This file is part of GNU Mailman.
-#
-# GNU Mailman is free software: you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by the Free
-# Software Foundation, either version 3 of the License, or (at your option)
-# any later version.
-#
-# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-# more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
-
-"""3.0b2 -> 3.0b3 schema migrations.
-
-Renamed:
- * bans.mailing_list -> bans.list_id
-
-Removed:
- * mailinglist.new_member_options
- * mailinglist.send_remindersn
-"""
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-__metaclass__ = type
-__all__ = [
- 'upgrade',
- ]
-
-
-from mailman.database.schema.helpers import make_listid, pivot
-
-
-VERSION = '20121015000000'
-
-
-
-def upgrade(database, store, version, module_path):
- if database.TAG == 'sqlite':
- upgrade_sqlite(database, store, version, module_path)
- else:
- upgrade_postgres(database, store, version, module_path)
-
-
-
-def upgrade_sqlite(database, store, version, module_path):
- database.load_schema(
- store, version, 'sqlite_{}_01.sql'.format(version), module_path)
- results = store.execute("""
- SELECT id, mailing_list
- FROM ban;
- """)
- for id, mailing_list in results:
- # Skip global bans since there's nothing to update.
- if mailing_list is None:
- continue
- store.execute("""
- UPDATE ban_backup SET list_id = '{}'
- WHERE id = {};
- """.format(make_listid(mailing_list), id))
- # Pivot the bans backup table to the real thing.
- pivot(store, 'ban')
- pivot(store, 'mailinglist')
-
-
-
-def upgrade_postgres(database, store, version, module_path):
- # Get the old values from the ban table.
- results = store.execute('SELECT id, mailing_list FROM ban;')
- store.execute('ALTER TABLE ban ADD COLUMN list_id TEXT;')
- for id, mailing_list in results:
- # Skip global bans since there's nothing to update.
- if mailing_list is None:
- continue
- store.execute("""
- UPDATE ban SET list_id = '{0}'
- WHERE id = {1};
- """.format(make_listid(mailing_list), id))
- store.execute('ALTER TABLE ban DROP COLUMN mailing_list;')
- store.execute('ALTER TABLE mailinglist DROP COLUMN new_member_options;')
- store.execute('ALTER TABLE mailinglist DROP COLUMN send_reminders;')
- store.execute('ALTER TABLE mailinglist DROP COLUMN subscribe_policy;')
- store.execute('ALTER TABLE mailinglist DROP COLUMN unsubscribe_policy;')
- store.execute(
- 'ALTER TABLE mailinglist DROP COLUMN subscribe_auto_approval;')
- store.execute('ALTER TABLE mailinglist DROP COLUMN private_roster;')
- store.execute(
- 'ALTER TABLE mailinglist DROP COLUMN admin_member_chunksize;')
- # Record the migration in the version table.
- database.load_schema(store, version, None, module_path)
diff --git a/src/mailman/database/schema/mm_20130406000000.py b/src/mailman/database/schema/mm_20130406000000.py
deleted file mode 100644
index 8d38dbab0..000000000
--- a/src/mailman/database/schema/mm_20130406000000.py
+++ /dev/null
@@ -1,65 +0,0 @@
-# Copyright (C) 2013-2014 by the Free Software Foundation, Inc.
-#
-# This file is part of GNU Mailman.
-#
-# GNU Mailman is free software: you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by the Free
-# Software Foundation, either version 3 of the License, or (at your option)
-# any later version.
-#
-# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-# more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
-
-"""3.0b3 -> 3.0b4 schema migrations.
-
-Renamed:
- * bounceevent.list_name -> bounceevent.list_id
-"""
-
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-__metaclass__ = type
-__all__ = [
- 'upgrade'
- ]
-
-
-from mailman.database.schema.helpers import make_listid, pivot
-
-
-VERSION = '20130406000000'
-
-
-
-def upgrade(database, store, version, module_path):
- if database.TAG == 'sqlite':
- upgrade_sqlite(database, store, version, module_path)
- else:
- upgrade_postgres(database, store, version, module_path)
-
-
-
-def upgrade_sqlite(database, store, version, module_path):
- database.load_schema(
- store, version, 'sqlite_{}_01.sql'.format(version), module_path)
- results = store.execute("""
- SELECT id, list_name
- FROM bounceevent;
- """)
- for id, list_name in results:
- store.execute("""
- UPDATE bounceevent_backup SET list_id = '{}'
- WHERE id = {};
- """.format(make_listid(list_name), id))
- pivot(store, 'bounceevent')
-
-
-
-def upgrade_postgres(database, store, version, module_path):
- pass
diff --git a/src/mailman/database/schema/postgres.sql b/src/mailman/database/schema/postgres.sql
deleted file mode 100644
index 0e97a4332..000000000
--- a/src/mailman/database/schema/postgres.sql
+++ /dev/null
@@ -1,349 +0,0 @@
-CREATE TABLE mailinglist (
- id SERIAL NOT NULL,
- -- List identity
- list_name TEXT,
- mail_host TEXT,
- include_list_post_header BOOLEAN,
- include_rfc2369_headers BOOLEAN,
- -- Attributes not directly modifiable via the web u/i
- created_at TIMESTAMP,
- admin_member_chunksize INTEGER,
- next_request_id INTEGER,
- next_digest_number INTEGER,
- digest_last_sent_at TIMESTAMP,
- volume INTEGER,
- last_post_at TIMESTAMP,
- accept_these_nonmembers BYTEA,
- acceptable_aliases_id INTEGER,
- admin_immed_notify BOOLEAN,
- admin_notify_mchanges BOOLEAN,
- administrivia BOOLEAN,
- advertised BOOLEAN,
- anonymous_list BOOLEAN,
- archive BOOLEAN,
- archive_private BOOLEAN,
- archive_volume_frequency INTEGER,
- -- Automatic responses.
- autorespond_owner INTEGER,
- autoresponse_owner_text TEXT,
- autorespond_postings INTEGER,
- autoresponse_postings_text TEXT,
- autorespond_requests INTEGER,
- autoresponse_request_text TEXT,
- autoresponse_grace_period TEXT,
- -- Bounces.
- forward_unrecognized_bounces_to INTEGER,
- process_bounces BOOLEAN,
- bounce_info_stale_after TEXT,
- bounce_matching_headers TEXT,
- bounce_notify_owner_on_disable BOOLEAN,
- bounce_notify_owner_on_removal BOOLEAN,
- bounce_score_threshold INTEGER,
- bounce_you_are_disabled_warnings INTEGER,
- bounce_you_are_disabled_warnings_interval TEXT,
- -- Content filtering.
- filter_action INTEGER,
- filter_content BOOLEAN,
- collapse_alternatives BOOLEAN,
- convert_html_to_plaintext BOOLEAN,
- default_member_action INTEGER,
- default_nonmember_action INTEGER,
- description TEXT,
- digest_footer_uri TEXT,
- digest_header_uri TEXT,
- digest_is_default BOOLEAN,
- digest_send_periodic BOOLEAN,
- digest_size_threshold REAL,
- digest_volume_frequency INTEGER,
- digestable BOOLEAN,
- discard_these_nonmembers BYTEA,
- emergency BOOLEAN,
- encode_ascii_prefixes BOOLEAN,
- first_strip_reply_to BOOLEAN,
- footer_uri TEXT,
- forward_auto_discards BOOLEAN,
- gateway_to_mail BOOLEAN,
- gateway_to_news BOOLEAN,
- generic_nonmember_action INTEGER,
- goodbye_message_uri TEXT,
- header_matches BYTEA,
- header_uri TEXT,
- hold_these_nonmembers BYTEA,
- info TEXT,
- linked_newsgroup TEXT,
- max_days_to_hold INTEGER,
- max_message_size INTEGER,
- max_num_recipients INTEGER,
- member_moderation_notice TEXT,
- mime_is_default_digest BOOLEAN,
- moderator_password TEXT,
- new_member_options INTEGER,
- news_moderation INTEGER,
- news_prefix_subject_too BOOLEAN,
- nntp_host TEXT,
- nondigestable BOOLEAN,
- nonmember_rejection_notice TEXT,
- obscure_addresses BOOLEAN,
- owner_chain TEXT,
- owner_pipeline TEXT,
- personalize INTEGER,
- post_id INTEGER,
- posting_chain TEXT,
- posting_pipeline TEXT,
- preferred_language TEXT,
- private_roster BOOLEAN,
- display_name TEXT,
- reject_these_nonmembers BYTEA,
- reply_goes_to_list INTEGER,
- reply_to_address TEXT,
- require_explicit_destination BOOLEAN,
- respond_to_post_requests BOOLEAN,
- scrub_nondigest BOOLEAN,
- send_goodbye_message BOOLEAN,
- send_reminders BOOLEAN,
- send_welcome_message BOOLEAN,
- subject_prefix TEXT,
- subscribe_auto_approval BYTEA,
- subscribe_policy INTEGER,
- topics BYTEA,
- topics_bodylines_limit INTEGER,
- topics_enabled BOOLEAN,
- unsubscribe_policy INTEGER,
- welcome_message_uri TEXT,
- -- This was accidentally added by the PostgreSQL porter.
- -- moderation_callback TEXT,
- PRIMARY KEY (id)
- );
-
-CREATE TABLE _request (
- id SERIAL NOT NULL,
- "key" TEXT,
- request_type INTEGER,
- data_hash BYTEA,
- mailing_list_id INTEGER,
- PRIMARY KEY (id)
- -- XXX: config.db_reset() triggers IntegrityError
- -- ,
- -- CONSTRAINT _request_mailing_list_id_fk
- -- FOREIGN KEY (mailing_list_id) REFERENCES mailinglist (id)
- );
-
-CREATE TABLE acceptablealias (
- id SERIAL NOT NULL,
- "alias" TEXT NOT NULL,
- mailing_list_id INTEGER NOT NULL,
- PRIMARY KEY (id)
- -- XXX: config.db_reset() triggers IntegrityError
- -- ,
- -- CONSTRAINT acceptablealias_mailing_list_id_fk
- -- FOREIGN KEY (mailing_list_id) REFERENCES mailinglist (id)
- );
-CREATE INDEX ix_acceptablealias_mailing_list_id
- ON acceptablealias (mailing_list_id);
-CREATE INDEX ix_acceptablealias_alias ON acceptablealias ("alias");
-
-CREATE TABLE preferences (
- id SERIAL NOT NULL,
- acknowledge_posts BOOLEAN,
- hide_address BOOLEAN,
- preferred_language TEXT,
- receive_list_copy BOOLEAN,
- receive_own_postings BOOLEAN,
- delivery_mode INTEGER,
- delivery_status INTEGER,
- PRIMARY KEY (id)
- );
-
-CREATE TABLE address (
- id SERIAL NOT NULL,
- email TEXT,
- _original TEXT,
- display_name TEXT,
- verified_on TIMESTAMP,
- registered_on TIMESTAMP,
- user_id INTEGER,
- preferences_id INTEGER,
- PRIMARY KEY (id)
- -- XXX: config.db_reset() triggers IntegrityError
- -- ,
- -- CONSTRAINT address_preferences_id_fk
- -- FOREIGN KEY (preferences_id) REFERENCES preferences (id)
- );
-
-CREATE TABLE "user" (
- id SERIAL NOT NULL,
- display_name TEXT,
- password BYTEA,
- _user_id UUID,
- _created_on TIMESTAMP,
- _preferred_address_id INTEGER,
- preferences_id INTEGER,
- PRIMARY KEY (id)
- -- XXX: config.db_reset() triggers IntegrityError
- -- ,
- -- CONSTRAINT user_preferences_id_fk
- -- FOREIGN KEY (preferences_id) REFERENCES preferences (id),
- -- XXX: config.db_reset() triggers IntegrityError
- -- CONSTRAINT _preferred_address_id_fk
- -- FOREIGN KEY (_preferred_address_id) REFERENCES address (id)
- );
-CREATE INDEX ix_user_user_id ON "user" (_user_id);
-
--- since user and address have circular foreign key refs, the
--- constraint on the address table has to be added after
--- the user table is created
---
--- XXX: users.rst triggers an IntegrityError
--- ALTER TABLE address ADD
--- CONSTRAINT address_user_id_fk
--- FOREIGN KEY (user_id) REFERENCES "user" (id);
-
-CREATE TABLE autoresponserecord (
- id SERIAL NOT NULL,
- address_id INTEGER,
- mailing_list_id INTEGER,
- response_type INTEGER,
- date_sent TIMESTAMP,
- PRIMARY KEY (id)
- -- XXX: config.db_reset() triggers IntegrityError
- -- ,
- -- CONSTRAINT autoresponserecord_address_id_fk
- -- FOREIGN KEY (address_id) REFERENCES address (id)
- -- XXX: config.db_reset() triggers IntegrityError
- -- ,
- -- CONSTRAINT autoresponserecord_mailing_list_id
- -- FOREIGN KEY (mailing_list_id) REFERENCES mailinglist (id)
- );
-CREATE INDEX ix_autoresponserecord_address_id
- ON autoresponserecord (address_id);
-CREATE INDEX ix_autoresponserecord_mailing_list_id
- ON autoresponserecord (mailing_list_id);
-
-CREATE TABLE bounceevent (
- id SERIAL NOT NULL,
- list_name TEXT,
- email TEXT,
- "timestamp" TIMESTAMP,
- message_id TEXT,
- context INTEGER,
- processed BOOLEAN,
- PRIMARY KEY (id)
- );
-
-CREATE TABLE contentfilter (
- id SERIAL NOT NULL,
- mailing_list_id INTEGER,
- filter_pattern TEXT,
- filter_type INTEGER,
- PRIMARY KEY (id),
- CONSTRAINT contentfilter_mailing_list_id
- FOREIGN KEY (mailing_list_id) REFERENCES mailinglist (id)
- );
-CREATE INDEX ix_contentfilter_mailing_list_id
- ON contentfilter (mailing_list_id);
-
-CREATE TABLE domain (
- id SERIAL NOT NULL,
- mail_host TEXT,
- base_url TEXT,
- description TEXT,
- contact_address TEXT,
- PRIMARY KEY (id)
- );
-
-CREATE TABLE language (
- id SERIAL NOT NULL,
- code TEXT,
- PRIMARY KEY (id)
- );
-
-CREATE TABLE member (
- id SERIAL NOT NULL,
- _member_id UUID,
- role INTEGER,
- mailing_list TEXT,
- moderation_action INTEGER,
- address_id INTEGER,
- preferences_id INTEGER,
- user_id INTEGER,
- PRIMARY KEY (id)
- -- XXX: config.db_reset() triggers IntegrityError
- -- ,
- -- CONSTRAINT member_address_id_fk
- -- FOREIGN KEY (address_id) REFERENCES address (id),
- -- XXX: config.db_reset() triggers IntegrityError
- -- CONSTRAINT member_preferences_id_fk
- -- FOREIGN KEY (preferences_id) REFERENCES preferences (id),
- -- CONSTRAINT member_user_id_fk
- -- FOREIGN KEY (user_id) REFERENCES "user" (id)
- );
-CREATE INDEX ix_member__member_id ON member (_member_id);
-CREATE INDEX ix_member_address_id ON member (address_id);
-CREATE INDEX ix_member_preferences_id ON member (preferences_id);
-
-CREATE TABLE message (
- id SERIAL NOT NULL,
- message_id_hash BYTEA,
- path BYTEA,
- message_id TEXT,
- PRIMARY KEY (id)
- );
-
-CREATE TABLE onelastdigest (
- id SERIAL NOT NULL,
- mailing_list_id INTEGER,
- address_id INTEGER,
- delivery_mode INTEGER,
- PRIMARY KEY (id),
- CONSTRAINT onelastdigest_mailing_list_id_fk
- FOREIGN KEY (mailing_list_id) REFERENCES mailinglist(id),
- CONSTRAINT onelastdigest_address_id_fk
- FOREIGN KEY (address_id) REFERENCES address(id)
- );
-
-CREATE TABLE pended (
- id SERIAL NOT NULL,
- token BYTEA,
- expiration_date TIMESTAMP,
- PRIMARY KEY (id)
- );
-
-CREATE TABLE pendedkeyvalue (
- id SERIAL NOT NULL,
- "key" TEXT,
- value TEXT,
- pended_id INTEGER,
- PRIMARY KEY (id)
- -- ,
- -- XXX: config.db_reset() triggers IntegrityError
- -- CONSTRAINT pendedkeyvalue_pended_id_fk
- -- FOREIGN KEY (pended_id) REFERENCES pended (id)
- );
-
-CREATE TABLE version (
- id SERIAL NOT NULL,
- component TEXT,
- version TEXT,
- PRIMARY KEY (id)
- );
-
-CREATE INDEX ix__request_mailing_list_id ON _request (mailing_list_id);
-CREATE INDEX ix_address_preferences_id ON address (preferences_id);
-CREATE INDEX ix_address_user_id ON address (user_id);
-CREATE INDEX ix_pendedkeyvalue_pended_id ON pendedkeyvalue (pended_id);
-CREATE INDEX ix_user_preferences_id ON "user" (preferences_id);
-
-CREATE TABLE ban (
- id SERIAL NOT NULL,
- email TEXT,
- mailing_list TEXT,
- PRIMARY KEY (id)
- );
-
-CREATE TABLE uid (
- -- Keep track of all assigned unique ids to prevent re-use.
- id SERIAL NOT NULL,
- uid UUID,
- PRIMARY KEY (id)
- );
-CREATE INDEX ix_uid_uid ON uid (uid);
diff --git a/src/mailman/database/schema/sqlite.sql b/src/mailman/database/schema/sqlite.sql
deleted file mode 100644
index e2b2d3814..000000000
--- a/src/mailman/database/schema/sqlite.sql
+++ /dev/null
@@ -1,327 +0,0 @@
--- THIS FILE HAS BEEN FROZEN AS OF 3.0b1
--- SEE THE SCHEMA MIGRATIONS FOR DIFFERENCES.
-
-PRAGMA foreign_keys = ON;
-
-CREATE TABLE _request (
- id INTEGER NOT NULL,
- "key" TEXT,
- request_type INTEGER,
- data_hash TEXT,
- mailing_list_id INTEGER,
- PRIMARY KEY (id),
- CONSTRAINT _request_mailing_list_id_fk
- FOREIGN KEY (mailing_list_id) REFERENCES mailinglist (id)
- );
-
-CREATE TABLE acceptablealias (
- id INTEGER NOT NULL,
- "alias" TEXT NOT NULL,
- mailing_list_id INTEGER NOT NULL,
- PRIMARY KEY (id),
- CONSTRAINT acceptablealias_mailing_list_id_fk
- FOREIGN KEY (mailing_list_id) REFERENCES mailinglist (id)
- );
-CREATE INDEX ix_acceptablealias_mailing_list_id
- ON acceptablealias (mailing_list_id);
-CREATE INDEX ix_acceptablealias_alias ON acceptablealias ("alias");
-
-CREATE TABLE address (
- id INTEGER NOT NULL,
- email TEXT,
- _original TEXT,
- display_name TEXT,
- verified_on TIMESTAMP,
- registered_on TIMESTAMP,
- user_id INTEGER,
- preferences_id INTEGER,
- PRIMARY KEY (id),
- CONSTRAINT address_user_id_fk
- FOREIGN KEY (user_id) REFERENCES user (id),
- CONSTRAINT address_preferences_id_fk
- FOREIGN KEY (preferences_id) REFERENCES preferences (id)
- );
-
-CREATE TABLE autoresponserecord (
- id INTEGER NOT NULL,
- address_id INTEGER,
- mailing_list_id INTEGER,
- response_type INTEGER,
- date_sent TIMESTAMP,
- PRIMARY KEY (id),
- CONSTRAINT autoresponserecord_address_id_fk
- FOREIGN KEY (address_id) REFERENCES address (id),
- CONSTRAINT autoresponserecord_mailing_list_id
- FOREIGN KEY (mailing_list_id) REFERENCES mailinglist (id)
- );
-CREATE INDEX ix_autoresponserecord_address_id
- ON autoresponserecord (address_id);
-CREATE INDEX ix_autoresponserecord_mailing_list_id
- ON autoresponserecord (mailing_list_id);
-
-CREATE TABLE bounceevent (
- id INTEGER NOT NULL,
- list_name TEXT,
- email TEXT,
- 'timestamp' TIMESTAMP,
- message_id TEXT,
- context INTEGER,
- processed BOOLEAN,
- PRIMARY KEY (id)
- );
-
-CREATE TABLE contentfilter (
- id INTEGER NOT NULL,
- mailing_list_id INTEGER,
- filter_pattern TEXT,
- filter_type INTEGER,
- PRIMARY KEY (id),
- CONSTRAINT contentfilter_mailing_list_id
- FOREIGN KEY (mailing_list_id) REFERENCES mailinglist (id)
- );
-CREATE INDEX ix_contentfilter_mailing_list_id
- ON contentfilter (mailing_list_id);
-
-CREATE TABLE domain (
- id INTEGER NOT NULL,
- mail_host TEXT,
- base_url TEXT,
- description TEXT,
- contact_address TEXT,
- PRIMARY KEY (id)
- );
-
-CREATE TABLE language (
- id INTEGER NOT NULL,
- code TEXT,
- PRIMARY KEY (id)
- );
-
-CREATE TABLE mailinglist (
- id INTEGER NOT NULL,
- -- List identity
- list_name TEXT,
- mail_host TEXT,
- include_list_post_header BOOLEAN,
- include_rfc2369_headers BOOLEAN,
- -- Attributes not directly modifiable via the web u/i
- created_at TIMESTAMP,
- admin_member_chunksize INTEGER,
- next_request_id INTEGER,
- next_digest_number INTEGER,
- digest_last_sent_at TIMESTAMP,
- volume INTEGER,
- last_post_at TIMESTAMP,
- accept_these_nonmembers BLOB,
- acceptable_aliases_id INTEGER,
- admin_immed_notify BOOLEAN,
- admin_notify_mchanges BOOLEAN,
- administrivia BOOLEAN,
- advertised BOOLEAN,
- anonymous_list BOOLEAN,
- archive BOOLEAN,
- archive_private BOOLEAN,
- archive_volume_frequency INTEGER,
- -- Automatic responses.
- autorespond_owner INTEGER,
- autoresponse_owner_text TEXT,
- autorespond_postings INTEGER,
- autoresponse_postings_text TEXT,
- autorespond_requests INTEGER,
- autoresponse_request_text TEXT,
- autoresponse_grace_period TEXT,
- -- Bounces.
- forward_unrecognized_bounces_to INTEGER,
- process_bounces BOOLEAN,
- bounce_info_stale_after TEXT,
- bounce_matching_headers TEXT,
- bounce_notify_owner_on_disable BOOLEAN,
- bounce_notify_owner_on_removal BOOLEAN,
- bounce_score_threshold INTEGER,
- bounce_you_are_disabled_warnings INTEGER,
- bounce_you_are_disabled_warnings_interval TEXT,
- -- Content filtering.
- filter_action INTEGER,
- filter_content BOOLEAN,
- collapse_alternatives BOOLEAN,
- convert_html_to_plaintext BOOLEAN,
- default_member_action INTEGER,
- default_nonmember_action INTEGER,
- description TEXT,
- digest_footer_uri TEXT,
- digest_header_uri TEXT,
- digest_is_default BOOLEAN,
- digest_send_periodic BOOLEAN,
- digest_size_threshold FLOAT,
- digest_volume_frequency INTEGER,
- digestable BOOLEAN,
- discard_these_nonmembers BLOB,
- emergency BOOLEAN,
- encode_ascii_prefixes BOOLEAN,
- first_strip_reply_to BOOLEAN,
- footer_uri TEXT,
- forward_auto_discards BOOLEAN,
- gateway_to_mail BOOLEAN,
- gateway_to_news BOOLEAN,
- generic_nonmember_action INTEGER,
- goodbye_message_uri TEXT,
- header_matches BLOB,
- header_uri TEXT,
- hold_these_nonmembers BLOB,
- info TEXT,
- linked_newsgroup TEXT,
- max_days_to_hold INTEGER,
- max_message_size INTEGER,
- max_num_recipients INTEGER,
- member_moderation_notice TEXT,
- mime_is_default_digest BOOLEAN,
- moderator_password TEXT,
- new_member_options INTEGER,
- news_moderation INTEGER,
- news_prefix_subject_too BOOLEAN,
- nntp_host TEXT,
- nondigestable BOOLEAN,
- nonmember_rejection_notice TEXT,
- obscure_addresses BOOLEAN,
- owner_chain TEXT,
- owner_pipeline TEXT,
- personalize INTEGER,
- post_id INTEGER,
- posting_chain TEXT,
- posting_pipeline TEXT,
- preferred_language TEXT,
- private_roster BOOLEAN,
- display_name TEXT,
- reject_these_nonmembers BLOB,
- reply_goes_to_list INTEGER,
- reply_to_address TEXT,
- require_explicit_destination BOOLEAN,
- respond_to_post_requests BOOLEAN,
- scrub_nondigest BOOLEAN,
- send_goodbye_message BOOLEAN,
- send_reminders BOOLEAN,
- send_welcome_message BOOLEAN,
- subject_prefix TEXT,
- subscribe_auto_approval BLOB,
- subscribe_policy INTEGER,
- topics BLOB,
- topics_bodylines_limit INTEGER,
- topics_enabled BOOLEAN,
- unsubscribe_policy INTEGER,
- welcome_message_uri TEXT,
- PRIMARY KEY (id)
- );
-
-CREATE TABLE member (
- id INTEGER NOT NULL,
- _member_id TEXT,
- role INTEGER,
- mailing_list TEXT,
- moderation_action INTEGER,
- address_id INTEGER,
- preferences_id INTEGER,
- user_id INTEGER,
- PRIMARY KEY (id),
- CONSTRAINT member_address_id_fk
- FOREIGN KEY (address_id) REFERENCES address (id),
- CONSTRAINT member_preferences_id_fk
- FOREIGN KEY (preferences_id) REFERENCES preferences (id)
- CONSTRAINT member_user_id_fk
- FOREIGN KEY (user_id) REFERENCES user (id)
- );
-CREATE INDEX ix_member__member_id ON member (_member_id);
-CREATE INDEX ix_member_address_id ON member (address_id);
-CREATE INDEX ix_member_preferences_id ON member (preferences_id);
-
-CREATE TABLE message (
- id INTEGER NOT NULL,
- message_id_hash TEXT,
- path TEXT,
- message_id TEXT,
- PRIMARY KEY (id)
- );
-
-CREATE TABLE onelastdigest (
- id INTEGER NOT NULL,
- mailing_list_id INTEGER,
- address_id INTEGER,
- delivery_mode INTEGER,
- PRIMARY KEY (id),
- CONSTRAINT onelastdigest_mailing_list_id_fk
- FOREIGN KEY (mailing_list_id) REFERENCES mailinglist(id),
- CONSTRAINT onelastdigest_address_id_fk
- FOREIGN KEY (address_id) REFERENCES address(id)
- );
-
-CREATE TABLE pended (
- id INTEGER NOT NULL,
- token TEXT,
- expiration_date TIMESTAMP,
- PRIMARY KEY (id)
- );
-
-CREATE TABLE pendedkeyvalue (
- id INTEGER NOT NULL,
- "key" TEXT,
- value TEXT,
- pended_id INTEGER,
- PRIMARY KEY (id),
- CONSTRAINT pendedkeyvalue_pended_id_fk
- FOREIGN KEY (pended_id) REFERENCES pended (id)
- );
-
-CREATE TABLE preferences (
- id INTEGER NOT NULL,
- acknowledge_posts BOOLEAN,
- hide_address BOOLEAN,
- preferred_language TEXT,
- receive_list_copy BOOLEAN,
- receive_own_postings BOOLEAN,
- delivery_mode INTEGER,
- delivery_status INTEGER,
- PRIMARY KEY (id)
- );
-
-CREATE TABLE user (
- id INTEGER NOT NULL,
- display_name TEXT,
- password BINARY,
- _user_id TEXT,
- _created_on TIMESTAMP,
- _preferred_address_id INTEGER,
- preferences_id INTEGER,
- PRIMARY KEY (id),
- CONSTRAINT user_preferences_id_fk
- FOREIGN KEY (preferences_id) REFERENCES preferences (id),
- CONSTRAINT _preferred_address_id_fk
- FOREIGN KEY (_preferred_address_id) REFERENCES address (id)
- );
-CREATE INDEX ix_user_user_id ON user (_user_id);
-
-CREATE TABLE version (
- id INTEGER NOT NULL,
- component TEXT,
- version TEXT,
- PRIMARY KEY (id)
- );
-
-CREATE INDEX ix__request_mailing_list_id ON _request (mailing_list_id);
-CREATE INDEX ix_address_preferences_id ON address (preferences_id);
-CREATE INDEX ix_address_user_id ON address (user_id);
-CREATE INDEX ix_pendedkeyvalue_pended_id ON pendedkeyvalue (pended_id);
-CREATE INDEX ix_user_preferences_id ON user (preferences_id);
-
-CREATE TABLE ban (
- id INTEGER NOT NULL,
- email TEXT,
- mailing_list TEXT,
- PRIMARY KEY (id)
- );
-
-CREATE TABLE uid (
- -- Keep track of all assigned unique ids to prevent re-use.
- id INTEGER NOT NULL,
- uid TEXT,
- PRIMARY KEY (id)
- );
-CREATE INDEX ix_uid_uid ON uid (uid);
diff --git a/src/mailman/database/schema/sqlite_20120407000000_01.sql b/src/mailman/database/schema/sqlite_20120407000000_01.sql
deleted file mode 100644
index a8db75be9..000000000
--- a/src/mailman/database/schema/sqlite_20120407000000_01.sql
+++ /dev/null
@@ -1,280 +0,0 @@
--- This file contains the sqlite3 schema migration from
--- 3.0b1 TO 3.0b2
---
--- 3.0b2 has been released thus you MAY NOT edit this file.
-
--- For SQLite3 migration strategy, see
--- http://sqlite.org/faq.html#q11
-
--- REMOVALS from the mailinglist table:
--- REM archive
--- REM archive_private
--- REM archive_volume_frequency
--- REM include_list_post_header
--- REM news_moderation
--- REM news_prefix_subject_too
--- REM nntp_host
---
--- ADDS to the mailing list table:
--- ADD allow_list_posts
--- ADD archive_policy
--- ADD list_id
--- ADD newsgroup_moderation
--- ADD nntp_prefix_subject_too
-
--- LP: #971013
--- LP: #967238
-
--- REMOVALS from the member table:
--- REM mailing_list
-
--- ADDS to the member table:
--- ADD list_id
-
--- LP: #1024509
-
-
-CREATE TABLE mailinglist_backup (
- id INTEGER NOT NULL,
- -- List identity
- list_name TEXT,
- mail_host TEXT,
- allow_list_posts BOOLEAN,
- include_rfc2369_headers BOOLEAN,
- -- Attributes not directly modifiable via the web u/i
- created_at TIMESTAMP,
- admin_member_chunksize INTEGER,
- next_request_id INTEGER,
- next_digest_number INTEGER,
- digest_last_sent_at TIMESTAMP,
- volume INTEGER,
- last_post_at TIMESTAMP,
- accept_these_nonmembers BLOB,
- acceptable_aliases_id INTEGER,
- admin_immed_notify BOOLEAN,
- admin_notify_mchanges BOOLEAN,
- administrivia BOOLEAN,
- advertised BOOLEAN,
- anonymous_list BOOLEAN,
- -- Automatic responses.
- autorespond_owner INTEGER,
- autoresponse_owner_text TEXT,
- autorespond_postings INTEGER,
- autoresponse_postings_text TEXT,
- autorespond_requests INTEGER,
- autoresponse_request_text TEXT,
- autoresponse_grace_period TEXT,
- -- Bounces.
- forward_unrecognized_bounces_to INTEGER,
- process_bounces BOOLEAN,
- bounce_info_stale_after TEXT,
- bounce_matching_headers TEXT,
- bounce_notify_owner_on_disable BOOLEAN,
- bounce_notify_owner_on_removal BOOLEAN,
- bounce_score_threshold INTEGER,
- bounce_you_are_disabled_warnings INTEGER,
- bounce_you_are_disabled_warnings_interval TEXT,
- -- Content filtering.
- filter_action INTEGER,
- filter_content BOOLEAN,
- collapse_alternatives BOOLEAN,
- convert_html_to_plaintext BOOLEAN,
- default_member_action INTEGER,
- default_nonmember_action INTEGER,
- description TEXT,
- digest_footer_uri TEXT,
- digest_header_uri TEXT,
- digest_is_default BOOLEAN,
- digest_send_periodic BOOLEAN,
- digest_size_threshold FLOAT,
- digest_volume_frequency INTEGER,
- digestable BOOLEAN,
- discard_these_nonmembers BLOB,
- emergency BOOLEAN,
- encode_ascii_prefixes BOOLEAN,
- first_strip_reply_to BOOLEAN,
- footer_uri TEXT,
- forward_auto_discards BOOLEAN,
- gateway_to_mail BOOLEAN,
- gateway_to_news BOOLEAN,
- goodbye_message_uri TEXT,
- header_matches BLOB,
- header_uri TEXT,
- hold_these_nonmembers BLOB,
- info TEXT,
- linked_newsgroup TEXT,
- max_days_to_hold INTEGER,
- max_message_size INTEGER,
- max_num_recipients INTEGER,
- member_moderation_notice TEXT,
- mime_is_default_digest BOOLEAN,
- moderator_password TEXT,
- new_member_options INTEGER,
- nondigestable BOOLEAN,
- nonmember_rejection_notice TEXT,
- obscure_addresses BOOLEAN,
- owner_chain TEXT,
- owner_pipeline TEXT,
- personalize INTEGER,
- post_id INTEGER,
- posting_chain TEXT,
- posting_pipeline TEXT,
- preferred_language TEXT,
- private_roster BOOLEAN,
- display_name TEXT,
- reject_these_nonmembers BLOB,
- reply_goes_to_list INTEGER,
- reply_to_address TEXT,
- require_explicit_destination BOOLEAN,
- respond_to_post_requests BOOLEAN,
- scrub_nondigest BOOLEAN,
- send_goodbye_message BOOLEAN,
- send_reminders BOOLEAN,
- send_welcome_message BOOLEAN,
- subject_prefix TEXT,
- subscribe_auto_approval BLOB,
- subscribe_policy INTEGER,
- topics BLOB,
- topics_bodylines_limit INTEGER,
- topics_enabled BOOLEAN,
- unsubscribe_policy INTEGER,
- welcome_message_uri TEXT,
- PRIMARY KEY (id)
- );
-
-INSERT INTO mailinglist_backup SELECT
- id,
- -- List identity
- list_name,
- mail_host,
- include_list_post_header,
- include_rfc2369_headers,
- -- Attributes not directly modifiable via the web u/i
- created_at,
- admin_member_chunksize,
- next_request_id,
- next_digest_number,
- digest_last_sent_at,
- volume,
- last_post_at,
- accept_these_nonmembers,
- acceptable_aliases_id,
- admin_immed_notify,
- admin_notify_mchanges,
- administrivia,
- advertised,
- anonymous_list,
- -- Automatic responses.
- autorespond_owner,
- autoresponse_owner_text,
- autorespond_postings,
- autoresponse_postings_text,
- autorespond_requests,
- autoresponse_request_text,
- autoresponse_grace_period,
- -- Bounces.
- forward_unrecognized_bounces_to,
- process_bounces,
- bounce_info_stale_after,
- bounce_matching_headers,
- bounce_notify_owner_on_disable,
- bounce_notify_owner_on_removal,
- bounce_score_threshold,
- bounce_you_are_disabled_warnings,
- bounce_you_are_disabled_warnings_interval,
- -- Content filtering.
- filter_action,
- filter_content,
- collapse_alternatives,
- convert_html_to_plaintext,
- default_member_action,
- default_nonmember_action,
- description,
- digest_footer_uri,
- digest_header_uri,
- digest_is_default,
- digest_send_periodic,
- digest_size_threshold,
- digest_volume_frequency,
- digestable,
- discard_these_nonmembers,
- emergency,
- encode_ascii_prefixes,
- first_strip_reply_to,
- footer_uri,
- forward_auto_discards,
- gateway_to_mail,
- gateway_to_news,
- goodbye_message_uri,
- header_matches,
- header_uri,
- hold_these_nonmembers,
- info,
- linked_newsgroup,
- max_days_to_hold,
- max_message_size,
- max_num_recipients,
- member_moderation_notice,
- mime_is_default_digest,
- moderator_password,
- new_member_options,
- nondigestable,
- nonmember_rejection_notice,
- obscure_addresses,
- owner_chain,
- owner_pipeline,
- personalize,
- post_id,
- posting_chain,
- posting_pipeline,
- preferred_language,
- private_roster,
- display_name,
- reject_these_nonmembers,
- reply_goes_to_list,
- reply_to_address,
- require_explicit_destination,
- respond_to_post_requests,
- scrub_nondigest,
- send_goodbye_message,
- send_reminders,
- send_welcome_message,
- subject_prefix,
- subscribe_auto_approval,
- subscribe_policy,
- topics,
- topics_bodylines_limit,
- topics_enabled,
- unsubscribe_policy,
- welcome_message_uri
- FROM mailinglist;
-
-CREATE TABLE member_backup(
- id INTEGER NOT NULL,
- _member_id TEXT,
- role INTEGER,
- moderation_action INTEGER,
- address_id INTEGER,
- preferences_id INTEGER,
- user_id INTEGER,
- PRIMARY KEY (id)
- );
-
-INSERT INTO member_backup SELECT
- id,
- _member_id,
- role,
- moderation_action,
- address_id,
- preferences_id,
- user_id
- FROM member;
-
-
--- Add the new columns. They'll get inserted at the Python layer.
-ALTER TABLE mailinglist_backup ADD COLUMN archive_policy INTEGER;
-ALTER TABLE mailinglist_backup ADD COLUMN list_id TEXT;
-ALTER TABLE mailinglist_backup ADD COLUMN nntp_prefix_subject_too INTEGER;
-ALTER TABLE mailinglist_backup ADD COLUMN newsgroup_moderation INTEGER;
-
-ALTER TABLE member_backup ADD COLUMN list_id TEXT;
diff --git a/src/mailman/database/schema/sqlite_20121015000000_01.sql b/src/mailman/database/schema/sqlite_20121015000000_01.sql
deleted file mode 100644
index a80dc03df..000000000
--- a/src/mailman/database/schema/sqlite_20121015000000_01.sql
+++ /dev/null
@@ -1,230 +0,0 @@
--- This file contains the sqlite3 schema migration from
--- 3.0b2 TO 3.0b3
---
--- 3.0b3 has been released thus you MAY NOT edit this file.
-
--- REMOVALS from the ban table:
--- REM mailing_list
-
--- ADDS to the ban table:
--- ADD list_id
-
-CREATE TABLE ban_backup (
- id INTEGER NOT NULL,
- email TEXT,
- PRIMARY KEY (id)
- );
-
-INSERT INTO ban_backup SELECT
- id, email
- FROM ban;
-
-ALTER TABLE ban_backup ADD COLUMN list_id TEXT;
-
--- REMOVALS from the mailinglist table.
--- REM new_member_options
--- REM send_reminders
--- REM subscribe_policy
--- REM unsubscribe_policy
--- REM subscribe_auto_approval
--- REM private_roster
--- REM admin_member_chunksize
-
-CREATE TABLE mailinglist_backup (
- id INTEGER NOT NULL,
- list_name TEXT,
- mail_host TEXT,
- allow_list_posts BOOLEAN,
- include_rfc2369_headers BOOLEAN,
- created_at TIMESTAMP,
- next_request_id INTEGER,
- next_digest_number INTEGER,
- digest_last_sent_at TIMESTAMP,
- volume INTEGER,
- last_post_at TIMESTAMP,
- accept_these_nonmembers BLOB,
- acceptable_aliases_id INTEGER,
- admin_immed_notify BOOLEAN,
- admin_notify_mchanges BOOLEAN,
- administrivia BOOLEAN,
- advertised BOOLEAN,
- anonymous_list BOOLEAN,
- autorespond_owner INTEGER,
- autoresponse_owner_text TEXT,
- autorespond_postings INTEGER,
- autoresponse_postings_text TEXT,
- autorespond_requests INTEGER,
- autoresponse_request_text TEXT,
- autoresponse_grace_period TEXT,
- forward_unrecognized_bounces_to INTEGER,
- process_bounces BOOLEAN,
- bounce_info_stale_after TEXT,
- bounce_matching_headers TEXT,
- bounce_notify_owner_on_disable BOOLEAN,
- bounce_notify_owner_on_removal BOOLEAN,
- bounce_score_threshold INTEGER,
- bounce_you_are_disabled_warnings INTEGER,
- bounce_you_are_disabled_warnings_interval TEXT,
- filter_action INTEGER,
- filter_content BOOLEAN,
- collapse_alternatives BOOLEAN,
- convert_html_to_plaintext BOOLEAN,
- default_member_action INTEGER,
- default_nonmember_action INTEGER,
- description TEXT,
- digest_footer_uri TEXT,
- digest_header_uri TEXT,
- digest_is_default BOOLEAN,
- digest_send_periodic BOOLEAN,
- digest_size_threshold FLOAT,
- digest_volume_frequency INTEGER,
- digestable BOOLEAN,
- discard_these_nonmembers BLOB,
- emergency BOOLEAN,
- encode_ascii_prefixes BOOLEAN,
- first_strip_reply_to BOOLEAN,
- footer_uri TEXT,
- forward_auto_discards BOOLEAN,
- gateway_to_mail BOOLEAN,
- gateway_to_news BOOLEAN,
- goodbye_message_uri TEXT,
- header_matches BLOB,
- header_uri TEXT,
- hold_these_nonmembers BLOB,
- info TEXT,
- linked_newsgroup TEXT,
- max_days_to_hold INTEGER,
- max_message_size INTEGER,
- max_num_recipients INTEGER,
- member_moderation_notice TEXT,
- mime_is_default_digest BOOLEAN,
- moderator_password TEXT,
- nondigestable BOOLEAN,
- nonmember_rejection_notice TEXT,
- obscure_addresses BOOLEAN,
- owner_chain TEXT,
- owner_pipeline TEXT,
- personalize INTEGER,
- post_id INTEGER,
- posting_chain TEXT,
- posting_pipeline TEXT,
- preferred_language TEXT,
- display_name TEXT,
- reject_these_nonmembers BLOB,
- reply_goes_to_list INTEGER,
- reply_to_address TEXT,
- require_explicit_destination BOOLEAN,
- respond_to_post_requests BOOLEAN,
- scrub_nondigest BOOLEAN,
- send_goodbye_message BOOLEAN,
- send_welcome_message BOOLEAN,
- subject_prefix TEXT,
- topics BLOB,
- topics_bodylines_limit INTEGER,
- topics_enabled BOOLEAN,
- welcome_message_uri TEXT,
- archive_policy INTEGER,
- list_id TEXT,
- nntp_prefix_subject_too INTEGER,
- newsgroup_moderation INTEGER,
- PRIMARY KEY (id)
- );
-
-INSERT INTO mailinglist_backup SELECT
- id,
- list_name,
- mail_host,
- allow_list_posts,
- include_rfc2369_headers,
- created_at,
- next_request_id,
- next_digest_number,
- digest_last_sent_at,
- volume,
- last_post_at,
- accept_these_nonmembers,
- acceptable_aliases_id,
- admin_immed_notify,
- admin_notify_mchanges,
- administrivia,
- advertised,
- anonymous_list,
- autorespond_owner,
- autoresponse_owner_text,
- autorespond_postings,
- autoresponse_postings_text,
- autorespond_requests,
- autoresponse_request_text,
- autoresponse_grace_period,
- forward_unrecognized_bounces_to,
- process_bounces,
- bounce_info_stale_after,
- bounce_matching_headers,
- bounce_notify_owner_on_disable,
- bounce_notify_owner_on_removal,
- bounce_score_threshold,
- bounce_you_are_disabled_warnings,
- bounce_you_are_disabled_warnings_interval,
- filter_action,
- filter_content,
- collapse_alternatives,
- convert_html_to_plaintext,
- default_member_action,
- default_nonmember_action,
- description,
- digest_footer_uri,
- digest_header_uri,
- digest_is_default,
- digest_send_periodic,
- digest_size_threshold,
- digest_volume_frequency,
- digestable,
- discard_these_nonmembers,
- emergency,
- encode_ascii_prefixes,
- first_strip_reply_to,
- footer_uri,
- forward_auto_discards,
- gateway_to_mail,
- gateway_to_news,
- goodbye_message_uri,
- header_matches,
- header_uri,
- hold_these_nonmembers,
- info,
- linked_newsgroup,
- max_days_to_hold,
- max_message_size,
- max_num_recipients,
- member_moderation_notice,
- mime_is_default_digest,
- moderator_password,
- nondigestable,
- nonmember_rejection_notice,
- obscure_addresses,
- owner_chain,
- owner_pipeline,
- personalize,
- post_id,
- posting_chain,
- posting_pipeline,
- preferred_language,
- display_name,
- reject_these_nonmembers,
- reply_goes_to_list,
- reply_to_address,
- require_explicit_destination,
- respond_to_post_requests,
- scrub_nondigest,
- send_goodbye_message,
- send_welcome_message,
- subject_prefix,
- topics,
- topics_bodylines_limit,
- topics_enabled,
- welcome_message_uri,
- archive_policy,
- list_id,
- nntp_prefix_subject_too,
- newsgroup_moderation
- FROM mailinglist;
diff --git a/src/mailman/database/schema/sqlite_20130406000000_01.sql b/src/mailman/database/schema/sqlite_20130406000000_01.sql
deleted file mode 100644
index fe30ed247..000000000
--- a/src/mailman/database/schema/sqlite_20130406000000_01.sql
+++ /dev/null
@@ -1,46 +0,0 @@
--- This file contains the SQLite schema migration from
--- 3.0b3 to 3.0b4
---
--- After 3.0b4 is released you may not edit this file.
-
--- For SQLite3 migration strategy, see
--- http://sqlite.org/faq.html#q11
-
--- ADD listarchiver table.
-
--- REMOVALs from the bounceevent table:
--- REM list_name
-
--- ADDs to the bounceevent table:
--- ADD list_id
-
--- ADDs to the mailinglist table:
--- ADD archiver_id
-
-CREATE TABLE bounceevent_backup (
- id INTEGER NOT NULL,
- email TEXT,
- 'timestamp' TIMESTAMP,
- message_id TEXT,
- context INTEGER,
- processed BOOLEAN,
- PRIMARY KEY (id)
- );
-
-INSERT INTO bounceevent_backup SELECT
- id, email, "timestamp", message_id,
- context, processed
- FROM bounceevent;
-
-ALTER TABLE bounceevent_backup ADD COLUMN list_id TEXT;
-
-CREATE TABLE listarchiver (
- id INTEGER NOT NULL,
- mailing_list_id INTEGER NOT NULL,
- name TEXT NOT NULL,
- _is_enabled BOOLEAN,
- PRIMARY KEY (id)
- );
-
-CREATE INDEX ix_listarchiver_mailing_list_id
- ON listarchiver(mailing_list_id);
diff --git a/src/mailman/database/sqlite.py b/src/mailman/database/sqlite.py
index 15629615f..db7860390 100644
--- a/src/mailman/database/sqlite.py
+++ b/src/mailman/database/sqlite.py
@@ -22,63 +22,27 @@ from __future__ import absolute_import, print_function, unicode_literals
__metaclass__ = type
__all__ = [
'SQLiteDatabase',
- 'make_temporary',
]
import os
-import types
-import shutil
-import tempfile
-from functools import partial
+from mailman.database.base import SABaseDatabase
from urlparse import urlparse
-from mailman.database.base import StormBaseDatabase
-from mailman.testing.helpers import configuration
-
-class SQLiteDatabase(StormBaseDatabase):
+class SQLiteDatabase(SABaseDatabase):
"""Database class for SQLite."""
- TAG = 'sqlite'
-
- def _database_exists(self, store):
- """See `BaseDatabase`."""
- table_query = 'select tbl_name from sqlite_master;'
- table_names = set(item[0] for item in
- store.execute(table_query))
- return 'version' in table_names
-
def _prepare(self, url):
parts = urlparse(url)
assert parts.scheme == 'sqlite', (
'Database url mismatch (expected sqlite prefix): {0}'.format(url))
+ # Ensure that the SQLite database file has the proper permissions,
+ # since SQLite doesn't play nice with umask.
path = os.path.normpath(parts.path)
- fd = os.open(path, os.O_WRONLY | os.O_NONBLOCK | os.O_CREAT, 0666)
+ fd = os.open(path, os.O_WRONLY | os.O_NONBLOCK | os.O_CREAT, 0o666)
# Ignore errors
if fd > 0:
os.close(fd)
-
-
-
-# Test suite adapter for ITemporaryDatabase.
-
-def _cleanup(self, tempdir):
- shutil.rmtree(tempdir)
-
-
-def make_temporary(database):
- """Adapts by monkey patching an existing SQLite IDatabase."""
- tempdir = tempfile.mkdtemp()
- url = 'sqlite:///' + os.path.join(tempdir, 'mailman.db')
- with configuration('database', url=url):
- database.initialize()
- database._cleanup = types.MethodType(
- partial(_cleanup, tempdir=tempdir),
- database)
- # bool column values in SQLite must be integers.
- database.FALSE = 0
- database.TRUE = 1
- return database
diff --git a/src/mailman/database/tests/data/__init__.py b/src/mailman/database/tests/data/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--- a/src/mailman/database/tests/data/__init__.py
+++ /dev/null
diff --git a/src/mailman/database/tests/data/mailman_01.db b/src/mailman/database/tests/data/mailman_01.db
deleted file mode 100644
index 1ff8d8343..000000000
--- a/src/mailman/database/tests/data/mailman_01.db
+++ /dev/null
Binary files differ
diff --git a/src/mailman/database/tests/data/migration_postgres_1.sql b/src/mailman/database/tests/data/migration_postgres_1.sql
deleted file mode 100644
index b82ecf6e4..000000000
--- a/src/mailman/database/tests/data/migration_postgres_1.sql
+++ /dev/null
@@ -1,133 +0,0 @@
-INSERT INTO "acceptablealias" VALUES(1,'foo@example.com',1);
-INSERT INTO "acceptablealias" VALUES(2,'bar@example.com',1);
-
-INSERT INTO "address" VALUES(
- 1,'anne@example.com',NULL,'Anne Person',
- '2012-04-19 00:52:24.826432','2012-04-19 00:49:42.373769',1,2);
-INSERT INTO "address" VALUES(
- 2,'bart@example.com',NULL,'Bart Person',
- '2012-04-19 00:53:25.878800','2012-04-19 00:49:52.882050',2,4);
-
-INSERT INTO "domain" VALUES(
- 1,'example.com','http://example.com',NULL,'postmaster@example.com');
-
-INSERT INTO "mailinglist" VALUES(
- -- id,list_name,mail_host,include_list_post_header,include_rfc2369_headers
- 1,'test','example.com',True,True,
- -- created_at,admin_member_chunksize,next_request_id,next_digest_number
- '2012-04-19 00:46:13.173844',30,1,1,
- -- digest_last_sent_at,volume,last_post_at,accept_these_nonmembers
- NULL,1,NULL,E'\\x80025D71012E',
- -- acceptable_aliases_id,admin_immed_notify,admin_notify_mchanges
- NULL,True,False,
- -- administrivia,advertised,anonymous_list,archive,archive_private
- True,True,False,True,False,
- -- archive_volume_frequency
- 1,
- --autorespond_owner,autoresponse_owner_text
- 0,'',
- -- autorespond_postings,autoresponse_postings_text
- 0,'',
- -- autorespond_requests,authoresponse_requests_text
- 0,'',
- -- autoresponse_grace_period
- '90 days, 0:00:00',
- -- forward_unrecognized_bounces_to,process_bounces
- 1,True,
- -- bounce_info_stale_after,bounce_matching_headers
- '7 days, 0:00:00','
-# Lines that *start* with a ''#'' are comments.
-to: friend@public.com
-message-id: relay.comanche.denmark.eu
-from: list@listme.com
-from: .*@uplinkpro.com
-',
- -- bounce_notify_owner_on_disable,bounce_notify_owner_on_removal
- True,True,
- -- bounce_score_threshold,bounce_you_are_disabled_warnings
- 5,3,
- -- bounce_you_are_disabled_warnings_interval
- '7 days, 0:00:00',
- -- filter_action,filter_content,collapse_alternatives
- 2,False,True,
- -- convert_html_to_plaintext,default_member_action,default_nonmember_action
- False,4,0,
- -- description
- '',
- -- digest_footer_uri
- 'mailman:///$listname/$language/footer-generic.txt',
- -- digest_header_uri
- NULL,
- -- digest_is_default,digest_send_periodic,digest_size_threshold
- False,True,30.0,
- -- digest_volume_frequency,digestable,discard_these_nonmembers
- 1,True,E'\\x80025D71012E',
- -- emergency,encode_ascii_prefixes,first_strip_reply_to
- False,False,False,
- -- footer_uri
- 'mailman:///$listname/$language/footer-generic.txt',
- -- forward_auto_discards,gateway_to_mail,gateway_to_news
- True,False,FAlse,
- -- generic_nonmember_action,goodby_message_uri
- 1,'',
- -- header_matches,header_uri,hold_these_nonmembers,info,linked_newsgroup
- E'\\x80025D71012E',NULL,E'\\x80025D71012E','','',
- -- max_days_to_hold,max_message_size,max_num_recipients
- 0,40,10,
- -- member_moderation_notice,mime_is_default_digest,moderator_password
- '',False,NULL,
- -- new_member_options,news_moderation,news_prefix_subject_too
- 256,0,True,
- -- nntp_host,nondigestable,nonmember_rejection_notice,obscure_addresses
- '',True,'',True,
- -- owner_chain,owner_pipeline,personalize,post_id
- 'default-owner-chain','default-owner-pipeline',0,1,
- -- posting_chain,posting_pipeline,preferred_language,private_roster
- 'default-posting-chain','default-posting-pipeline','en',True,
- -- display_name,reject_these_nonmembers
- 'Test',E'\\x80025D71012E',
- -- reply_goes_to_list,reply_to_address
- 0,'',
- -- require_explicit_destination,respond_to_post_requests
- True,True,
- -- scrub_nondigest,send_goodbye_message,send_reminders,send_welcome_message
- False,True,True,True,
- -- subject_prefix,subscribe_auto_approval
- '[Test] ',E'\\x80025D71012E',
- -- subscribe_policy,topics,topics_bodylines_limit,topics_enabled
- 1,E'\\x80025D71012E',5,False,
- -- unsubscribe_policy,welcome_message_uri
- 0,'mailman:///welcome.txt');
-
-INSERT INTO "member" VALUES(
- 1,'d1243f4d-e604-4f6b-af52-98d0a7bce0f1',1,'test@example.com',4,NULL,5,1);
-INSERT INTO "member" VALUES(
- 2,'dccc3851-fdfb-4afa-90cf-bdcbf80ad0fd',2,'test@example.com',3,NULL,6,1);
-INSERT INTO "member" VALUES(
- 3,'479be431-45f2-473d-bc3c-7eac614030ac',3,'test@example.com',3,NULL,7,2);
-INSERT INTO "member" VALUES(
- 4,'e2dc604c-d93a-4b91-b5a8-749e3caade36',1,'test@example.com',4,NULL,8,2);
-
-INSERT INTO "preferences" VALUES(1,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
-INSERT INTO "preferences" VALUES(2,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
-INSERT INTO "preferences" VALUES(3,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
-INSERT INTO "preferences" VALUES(4,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
-INSERT INTO "preferences" VALUES(5,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
-INSERT INTO "preferences" VALUES(6,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
-INSERT INTO "preferences" VALUES(7,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
-INSERT INTO "preferences" VALUES(8,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
-
-INSERT INTO "user" VALUES(
- 1,'Anne Person',NULL,'0adf3caa-6f26-46f8-a11d-5256c8148592',
- '2012-04-19 00:49:42.370493',1,1);
-INSERT INTO "user" VALUES(
- 2,'Bart Person',NULL,'63f5d1a2-e533-4055-afe4-475dec3b1163',
- '2012-04-19 00:49:52.868746',2,3);
-
-INSERT INTO "uid" VALUES(1,'8bf9a615-f23e-4980-b7d1-90ac0203c66f');
-INSERT INTO "uid" VALUES(2,'0adf3caa-6f26-46f8-a11d-5256c8148592');
-INSERT INTO "uid" VALUES(3,'63f5d1a2-e533-4055-afe4-475dec3b1163');
-INSERT INTO "uid" VALUES(4,'d1243f4d-e604-4f6b-af52-98d0a7bce0f1');
-INSERT INTO "uid" VALUES(5,'dccc3851-fdfb-4afa-90cf-bdcbf80ad0fd');
-INSERT INTO "uid" VALUES(6,'479be431-45f2-473d-bc3c-7eac614030ac');
-INSERT INTO "uid" VALUES(7,'e2dc604c-d93a-4b91-b5a8-749e3caade36');
diff --git a/src/mailman/database/tests/data/migration_sqlite_1.sql b/src/mailman/database/tests/data/migration_sqlite_1.sql
deleted file mode 100644
index a5ac96dfa..000000000
--- a/src/mailman/database/tests/data/migration_sqlite_1.sql
+++ /dev/null
@@ -1,133 +0,0 @@
-INSERT INTO "acceptablealias" VALUES(1,'foo@example.com',1);
-INSERT INTO "acceptablealias" VALUES(2,'bar@example.com',1);
-
-INSERT INTO "address" VALUES(
- 1,'anne@example.com',NULL,'Anne Person',
- '2012-04-19 00:52:24.826432','2012-04-19 00:49:42.373769',1,2);
-INSERT INTO "address" VALUES(
- 2,'bart@example.com',NULL,'Bart Person',
- '2012-04-19 00:53:25.878800','2012-04-19 00:49:52.882050',2,4);
-
-INSERT INTO "domain" VALUES(
- 1,'example.com','http://example.com',NULL,'postmaster@example.com');
-
-INSERT INTO "mailinglist" VALUES(
- -- id,list_name,mail_host,include_list_post_header,include_rfc2369_headers
- 1,'test','example.com',1,1,
- -- created_at,admin_member_chunksize,next_request_id,next_digest_number
- '2012-04-19 00:46:13.173844',30,1,1,
- -- digest_last_sent_at,volume,last_post_at,accept_these_nonmembers
- NULL,1,NULL,X'80025D71012E',
- -- acceptable_aliases_id,admin_immed_notify,admin_notify_mchanges
- NULL,1,0,
- -- administrivia,advertised,anonymous_list,archive,archive_private
- 1,1,0,1,0,
- -- archive_volume_frequency
- 1,
- --autorespond_owner,autoresponse_owner_text
- 0,'',
- -- autorespond_postings,autoresponse_postings_text
- 0,'',
- -- autorespond_requests,authoresponse_requests_text
- 0,'',
- -- autoresponse_grace_period
- '90 days, 0:00:00',
- -- forward_unrecognized_bounces_to,process_bounces
- 1,1,
- -- bounce_info_stale_after,bounce_matching_headers
- '7 days, 0:00:00','
-# Lines that *start* with a ''#'' are comments.
-to: friend@public.com
-message-id: relay.comanche.denmark.eu
-from: list@listme.com
-from: .*@uplinkpro.com
-',
- -- bounce_notify_owner_on_disable,bounce_notify_owner_on_removal
- 1,1,
- -- bounce_score_threshold,bounce_you_are_disabled_warnings
- 5,3,
- -- bounce_you_are_disabled_warnings_interval
- '7 days, 0:00:00',
- -- filter_action,filter_content,collapse_alternatives
- 2,0,1,
- -- convert_html_to_plaintext,default_member_action,default_nonmember_action
- 0,4,0,
- -- description
- '',
- -- digest_footer_uri
- 'mailman:///$listname/$language/footer-generic.txt',
- -- digest_header_uri
- NULL,
- -- digest_is_default,digest_send_periodic,digest_size_threshold
- 0,1,30.0,
- -- digest_volume_frequency,digestable,discard_these_nonmembers
- 1,1,X'80025D71012E',
- -- emergency,encode_ascii_prefixes,first_strip_reply_to
- 0,0,0,
- -- footer_uri
- 'mailman:///$listname/$language/footer-generic.txt',
- -- forward_auto_discards,gateway_to_mail,gateway_to_news
- 1,0,0,
- -- generic_nonmember_action,goodby_message_uri
- 1,'',
- -- header_matches,header_uri,hold_these_nonmembers,info,linked_newsgroup
- X'80025D71012E',NULL,X'80025D71012E','','',
- -- max_days_to_hold,max_message_size,max_num_recipients
- 0,40,10,
- -- member_moderation_notice,mime_is_default_digest,moderator_password
- '',0,NULL,
- -- new_member_options,news_moderation,news_prefix_subject_too
- 256,0,1,
- -- nntp_host,nondigestable,nonmember_rejection_notice,obscure_addresses
- '',1,'',1,
- -- owner_chain,owner_pipeline,personalize,post_id
- 'default-owner-chain','default-owner-pipeline',0,1,
- -- posting_chain,posting_pipeline,preferred_language,private_roster
- 'default-posting-chain','default-posting-pipeline','en',1,
- -- display_name,reject_these_nonmembers
- 'Test',X'80025D71012E',
- -- reply_goes_to_list,reply_to_address
- 0,'',
- -- require_explicit_destination,respond_to_post_requests
- 1,1,
- -- scrub_nondigest,send_goodbye_message,send_reminders,send_welcome_message
- 0,1,1,1,
- -- subject_prefix,subscribe_auto_approval
- '[Test] ',X'80025D71012E',
- -- subscribe_policy,topics,topics_bodylines_limit,topics_enabled
- 1,X'80025D71012E',5,0,
- -- unsubscribe_policy,welcome_message_uri
- 0,'mailman:///welcome.txt');
-
-INSERT INTO "member" VALUES(
- 1,'d1243f4d-e604-4f6b-af52-98d0a7bce0f1',1,'test@example.com',4,NULL,5,1);
-INSERT INTO "member" VALUES(
- 2,'dccc3851-fdfb-4afa-90cf-bdcbf80ad0fd',2,'test@example.com',3,NULL,6,1);
-INSERT INTO "member" VALUES(
- 3,'479be431-45f2-473d-bc3c-7eac614030ac',3,'test@example.com',3,NULL,7,2);
-INSERT INTO "member" VALUES(
- 4,'e2dc604c-d93a-4b91-b5a8-749e3caade36',1,'test@example.com',4,NULL,8,2);
-
-INSERT INTO "preferences" VALUES(1,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
-INSERT INTO "preferences" VALUES(2,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
-INSERT INTO "preferences" VALUES(3,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
-INSERT INTO "preferences" VALUES(4,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
-INSERT INTO "preferences" VALUES(5,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
-INSERT INTO "preferences" VALUES(6,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
-INSERT INTO "preferences" VALUES(7,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
-INSERT INTO "preferences" VALUES(8,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
-
-INSERT INTO "user" VALUES(
- 1,'Anne Person',NULL,'0adf3caa-6f26-46f8-a11d-5256c8148592',
- '2012-04-19 00:49:42.370493',1,1);
-INSERT INTO "user" VALUES(
- 2,'Bart Person',NULL,'63f5d1a2-e533-4055-afe4-475dec3b1163',
- '2012-04-19 00:49:52.868746',2,3);
-
-INSERT INTO "uid" VALUES(1,'8bf9a615-f23e-4980-b7d1-90ac0203c66f');
-INSERT INTO "uid" VALUES(2,'0adf3caa-6f26-46f8-a11d-5256c8148592');
-INSERT INTO "uid" VALUES(3,'63f5d1a2-e533-4055-afe4-475dec3b1163');
-INSERT INTO "uid" VALUES(4,'d1243f4d-e604-4f6b-af52-98d0a7bce0f1');
-INSERT INTO "uid" VALUES(5,'dccc3851-fdfb-4afa-90cf-bdcbf80ad0fd');
-INSERT INTO "uid" VALUES(6,'479be431-45f2-473d-bc3c-7eac614030ac');
-INSERT INTO "uid" VALUES(7,'e2dc604c-d93a-4b91-b5a8-749e3caade36');
diff --git a/src/mailman/database/tests/test_factory.py b/src/mailman/database/tests/test_factory.py
new file mode 100644
index 000000000..d7c4d8503
--- /dev/null
+++ b/src/mailman/database/tests/test_factory.py
@@ -0,0 +1,160 @@
+# Copyright (C) 2013-2014 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
+
+"""Test database schema migrations"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+ 'TestSchemaManager',
+ ]
+
+
+import unittest
+import alembic.command
+
+from mock import patch
+from sqlalchemy import MetaData, Table, Column, Integer, Unicode
+from sqlalchemy.exc import ProgrammingError, OperationalError
+from sqlalchemy.schema import Index
+
+from mailman.config import config
+from mailman.database.alembic import alembic_cfg
+from mailman.database.factory import LAST_STORM_SCHEMA_VERSION, SchemaManager
+from mailman.database.model import Model
+from mailman.interfaces.database import DatabaseError
+from mailman.testing.layers import ConfigLayer
+
+
+
+class TestSchemaManager(unittest.TestCase):
+
+ layer = ConfigLayer
+
+ def setUp(self):
+ # Drop the existing database.
+ Model.metadata.drop_all(config.db.engine)
+ md = MetaData()
+ md.reflect(bind=config.db.engine)
+ for tablename in ('alembic_version', 'version'):
+ if tablename in md.tables:
+ md.tables[tablename].drop(config.db.engine)
+ self.schema_mgr = SchemaManager(config.db)
+
+ def tearDown(self):
+ self._drop_storm_database()
+ # Restore a virgin database.
+ Model.metadata.create_all(config.db.engine)
+
+ def _table_exists(self, tablename):
+ md = MetaData()
+ md.reflect(bind=config.db.engine)
+ return tablename in md.tables
+
+ def _create_storm_database(self, revision):
+ version_table = Table(
+ 'version', Model.metadata,
+ Column('id', Integer, primary_key=True),
+ Column('component', Unicode),
+ Column('version', Unicode),
+ )
+ version_table.create(config.db.engine)
+ config.db.store.execute(version_table.insert().values(
+ component='schema', version=revision))
+ config.db.commit()
+ # Other Storm specific changes, those SQL statements hopefully work on
+ # all DB engines...
+ config.db.engine.execute(
+ 'ALTER TABLE mailinglist ADD COLUMN acceptable_aliases_id INT')
+ Index('ix_user__user_id').drop(bind=config.db.engine)
+ # Don't pollute our main metadata object, create a new one.
+ md = MetaData()
+ user_table = Model.metadata.tables['user'].tometadata(md)
+ Index('ix_user_user_id', user_table.c._user_id).create(
+ bind=config.db.engine)
+ config.db.commit()
+
+ def _drop_storm_database(self):
+ """Remove the leftovers from a Storm DB.
+
+ A drop_all() must be issued afterwards.
+ """
+ if 'version' in Model.metadata.tables:
+ version = Model.metadata.tables['version']
+ version.drop(config.db.engine, checkfirst=True)
+ Model.metadata.remove(version)
+ try:
+ Index('ix_user_user_id').drop(bind=config.db.engine)
+ except (ProgrammingError, OperationalError):
+ # Nonexistent. PostgreSQL raises a ProgrammingError, while SQLite
+ # raises an OperationalError.
+ pass
+ config.db.commit()
+
+ def test_current_database(self):
+ # The database is already at the latest version.
+ alembic.command.stamp(alembic_cfg, 'head')
+ with patch('alembic.command') as alembic_command:
+ self.schema_mgr.setup_database()
+ self.assertFalse(alembic_command.stamp.called)
+ self.assertFalse(alembic_command.upgrade.called)
+
+ @patch('alembic.command')
+ def test_initial(self, alembic_command):
+ # No existing database.
+ self.assertFalse(self._table_exists('mailinglist'))
+ self.assertFalse(self._table_exists('alembic_version'))
+ self.schema_mgr.setup_database()
+ self.assertFalse(alembic_command.upgrade.called)
+ self.assertTrue(self._table_exists('mailinglist'))
+ self.assertTrue(self._table_exists('alembic_version'))
+
+ @patch('alembic.command.stamp')
+ def test_storm(self, alembic_command_stamp):
+ # Existing Storm database.
+ Model.metadata.create_all(config.db.engine)
+ self._create_storm_database(LAST_STORM_SCHEMA_VERSION)
+ self.schema_mgr.setup_database()
+ self.assertFalse(alembic_command_stamp.called)
+ self.assertTrue(
+ self._table_exists('mailinglist')
+ and self._table_exists('alembic_version')
+ and not self._table_exists('version'))
+
+ @patch('alembic.command')
+ def test_old_storm(self, alembic_command):
+ # Existing Storm database in an old version.
+ Model.metadata.create_all(config.db.engine)
+ self._create_storm_database('001')
+ self.assertRaises(DatabaseError, self.schema_mgr.setup_database)
+ self.assertFalse(alembic_command.stamp.called)
+ self.assertFalse(alembic_command.upgrade.called)
+
+ def test_old_db(self):
+ # The database is in an old revision, must upgrade.
+ alembic.command.stamp(alembic_cfg, 'head')
+ md = MetaData()
+ md.reflect(bind=config.db.engine)
+ config.db.store.execute(md.tables['alembic_version'].delete())
+ config.db.store.execute(md.tables['alembic_version'].insert().values(
+ version_num='dummyrevision'))
+ config.db.commit()
+ with patch('alembic.command') as alembic_command:
+ self.schema_mgr.setup_database()
+ self.assertFalse(alembic_command.stamp.called)
+ self.assertTrue(alembic_command.upgrade.called)
diff --git a/src/mailman/database/tests/test_migrations.py b/src/mailman/database/tests/test_migrations.py
deleted file mode 100644
index 9619b80a4..000000000
--- a/src/mailman/database/tests/test_migrations.py
+++ /dev/null
@@ -1,506 +0,0 @@
-# Copyright (C) 2012-2014 by the Free Software Foundation, Inc.
-#
-# This file is part of GNU Mailman.
-#
-# GNU Mailman is free software: you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by the Free
-# Software Foundation, either version 3 of the License, or (at your option)
-# any later version.
-#
-# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-# more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
-
-"""Test schema migrations."""
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-__metaclass__ = type
-__all__ = [
- 'TestMigration20120407MigratedData',
- 'TestMigration20120407Schema',
- 'TestMigration20120407UnchangedData',
- 'TestMigration20121015MigratedData',
- 'TestMigration20121015Schema',
- 'TestMigration20130406MigratedData',
- 'TestMigration20130406Schema',
- ]
-
-
-import unittest
-
-from datetime import datetime
-from operator import attrgetter
-from pkg_resources import resource_string
-from sqlite3 import OperationalError
-from storm.exceptions import DatabaseError
-from zope.component import getUtility
-
-from mailman.interfaces.database import IDatabaseFactory
-from mailman.interfaces.domain import IDomainManager
-from mailman.interfaces.archiver import ArchivePolicy
-from mailman.interfaces.bounce import BounceContext
-from mailman.interfaces.listmanager import IListManager
-from mailman.interfaces.mailinglist import IAcceptableAliasSet
-from mailman.interfaces.nntp import NewsgroupModeration
-from mailman.interfaces.subscriptions import ISubscriptionService
-from mailman.model.bans import Ban
-from mailman.model.bounce import BounceEvent
-from mailman.testing.helpers import temporary_db
-from mailman.testing.layers import ConfigLayer
-
-
-
-class MigrationTestBase(unittest.TestCase):
- """Test database migrations."""
-
- layer = ConfigLayer
-
- def setUp(self):
- self._database = getUtility(IDatabaseFactory, 'temporary').create()
-
- def tearDown(self):
- self._database._cleanup()
-
- def _table_missing_present(self, migrations, missing, present):
- """The appropriate migrations leave some tables missing and present.
-
- :param migrations: Sequence of migrations to load.
- :param missing: Tables which should be missing.
- :param present: Tables which should be present.
- """
- for migration in migrations:
- self._database.load_migrations(migration)
- self._database.store.commit()
- for table in missing:
- self.assertRaises(OperationalError,
- self._database.store.execute,
- 'select * from {};'.format(table))
- for table in present:
- self._database.store.execute('select * from {};'.format(table))
-
- def _missing_present(self, table, migrations, missing, present):
- """The appropriate migrations leave columns missing and present.
-
- :param table: The table to test columns from.
- :param migrations: Sequence of migrations to load.
- :param missing: Set of columns which should be missing after the
- migrations are loaded.
- :param present: Set of columns which should be present after the
- migrations are loaded.
- """
- for migration in migrations:
- self._database.load_migrations(migration)
- self._database.store.commit()
- for column in missing:
- self.assertRaises(DatabaseError,
- self._database.store.execute,
- 'select {0} from {1};'.format(column, table))
- self._database.store.rollback()
- for column in present:
- # This should not produce an exception. Is there some better test
- # that we can perform?
- self._database.store.execute(
- 'select {0} from {1};'.format(column, table))
-
-
-
-class TestMigration20120407Schema(MigrationTestBase):
- """Test column migrations."""
-
- def test_pre_upgrade_columns_migration(self):
- # Test that before the migration, the old table columns are present
- # and the new database columns are not.
- self._missing_present('mailinglist',
- ['20120406999999'],
- # New columns are missing.
- ('allow_list_posts',
- 'archive_policy',
- 'list_id',
- 'nntp_prefix_subject_too'),
- # Old columns are present.
- ('archive',
- 'archive_private',
- 'archive_volume_frequency',
- 'generic_nonmember_action',
- 'include_list_post_header',
- 'news_moderation',
- 'news_prefix_subject_too',
- 'nntp_host'))
- self._missing_present('member',
- ['20120406999999'],
- ('list_id',),
- ('mailing_list',))
-
- def test_post_upgrade_columns_migration(self):
- # Test that after the migration, the old table columns are missing
- # and the new database columns are present.
- self._missing_present('mailinglist',
- ['20120406999999',
- '20120407000000'],
- # The old columns are missing.
- ('archive',
- 'archive_private',
- 'archive_volume_frequency',
- 'generic_nonmember_action',
- 'include_list_post_header',
- 'news_moderation',
- 'news_prefix_subject_too',
- 'nntp_host'),
- # The new columns are present.
- ('allow_list_posts',
- 'archive_policy',
- 'list_id',
- 'nntp_prefix_subject_too'))
- self._missing_present('member',
- ['20120406999999',
- '20120407000000'],
- ('mailing_list',),
- ('list_id',))
-
-
-
-class TestMigration20120407UnchangedData(MigrationTestBase):
- """Test non-migrated data."""
-
- def setUp(self):
- MigrationTestBase.setUp(self)
- # Load all the migrations to just before the one we're testing.
- self._database.load_migrations('20120406999999')
- # Load the previous schema's sample data.
- sample_data = resource_string(
- 'mailman.database.tests.data',
- 'migration_{0}_1.sql'.format(self._database.TAG))
- self._database.load_sql(self._database.store, sample_data)
- # XXX 2012-12-28: We have to load the last migration defined in the
- # system, otherwise the ORM model will not match the SQL table
- # definitions and we'll get OperationalErrors from SQLite.
- self._database.load_migrations('20121015000000')
-
- def test_migration_domains(self):
- # Test that the domains table, which isn't touched, doesn't change.
- with temporary_db(self._database):
- # Check that the domains survived the migration. This table
- # was not touched so it should be fine.
- domains = list(getUtility(IDomainManager))
- self.assertEqual(len(domains), 1)
- self.assertEqual(domains[0].mail_host, 'example.com')
-
- def test_migration_mailing_lists(self):
- # Test that the mailing lists survive migration.
- with temporary_db(self._database):
- # There should be exactly one mailing list defined.
- mlists = list(getUtility(IListManager).mailing_lists)
- self.assertEqual(len(mlists), 1)
- self.assertEqual(mlists[0].fqdn_listname, 'test@example.com')
-
- def test_migration_acceptable_aliases(self):
- # Test that the mailing list's acceptable aliases survive migration.
- # This proves that foreign key references are migrated properly.
- with temporary_db(self._database):
- mlist = getUtility(IListManager).get('test@example.com')
- aliases_set = IAcceptableAliasSet(mlist)
- self.assertEqual(set(aliases_set.aliases),
- set(['foo@example.com', 'bar@example.com']))
-
- def test_migration_members(self):
- # Test that the members of a mailing list all survive migration.
- with temporary_db(self._database):
- mlist = getUtility(IListManager).get('test@example.com')
- # Test that all the members we expect are still there. Start with
- # the two list delivery members.
- addresses = set(address.email
- for address in mlist.members.addresses)
- self.assertEqual(addresses,
- set(['anne@example.com', 'bart@example.com']))
- # There is one owner.
- owners = set(address.email for address in mlist.owners.addresses)
- self.assertEqual(len(owners), 1)
- self.assertEqual(owners.pop(), 'anne@example.com')
- # There is one moderator.
- moderators = set(address.email
- for address in mlist.moderators.addresses)
- self.assertEqual(len(moderators), 1)
- self.assertEqual(moderators.pop(), 'bart@example.com')
-
-
-
-class TestMigration20120407MigratedData(MigrationTestBase):
- """Test affected migration data."""
-
- def setUp(self):
- MigrationTestBase.setUp(self)
- # Load all the migrations to just before the one we're testing.
- self._database.load_migrations('20120406999999')
- # Load the previous schema's sample data.
- sample_data = resource_string(
- 'mailman.database.tests.data',
- 'migration_{0}_1.sql'.format(self._database.TAG))
- self._database.load_sql(self._database.store, sample_data)
-
- def _upgrade(self):
- # XXX 2012-12-28: We have to load the last migration defined in the
- # system, otherwise the ORM model will not match the SQL table
- # definitions and we'll get OperationalErrors from SQLite.
- self._database.load_migrations('20121015000000')
-
- def test_migration_archive_policy_never_0(self):
- # Test that the new archive_policy value is updated correctly. In the
- # case of old column archive=0, the archive_private column is
- # ignored. This test sets it to 0 to ensure it's ignored.
- self._database.store.execute(
- 'UPDATE mailinglist SET archive = {0}, archive_private = {0} '
- 'WHERE id = 1;'.format(self._database.FALSE))
- # Complete the migration
- self._upgrade()
- with temporary_db(self._database):
- mlist = getUtility(IListManager).get('test@example.com')
- self.assertEqual(mlist.archive_policy, ArchivePolicy.never)
-
- def test_migration_archive_policy_never_1(self):
- # Test that the new archive_policy value is updated correctly. In the
- # case of old column archive=0, the archive_private column is
- # ignored. This test sets it to 1 to ensure it's ignored.
- self._database.store.execute(
- 'UPDATE mailinglist SET archive = {0}, archive_private = {1} '
- 'WHERE id = 1;'.format(self._database.FALSE,
- self._database.TRUE))
- # Complete the migration
- self._upgrade()
- with temporary_db(self._database):
- mlist = getUtility(IListManager).get('test@example.com')
- self.assertEqual(mlist.archive_policy, ArchivePolicy.never)
-
- def test_archive_policy_private(self):
- # Test that the new archive_policy value is updated correctly for
- # private archives.
- self._database.store.execute(
- 'UPDATE mailinglist SET archive = {0}, archive_private = {0} '
- 'WHERE id = 1;'.format(self._database.TRUE))
- # Complete the migration
- self._upgrade()
- with temporary_db(self._database):
- mlist = getUtility(IListManager).get('test@example.com')
- self.assertEqual(mlist.archive_policy, ArchivePolicy.private)
-
- def test_archive_policy_public(self):
- # Test that the new archive_policy value is updated correctly for
- # public archives.
- self._database.store.execute(
- 'UPDATE mailinglist SET archive = {1}, archive_private = {0} '
- 'WHERE id = 1;'.format(self._database.FALSE,
- self._database.TRUE))
- # Complete the migration
- self._upgrade()
- with temporary_db(self._database):
- mlist = getUtility(IListManager).get('test@example.com')
- self.assertEqual(mlist.archive_policy, ArchivePolicy.public)
-
- def test_list_id(self):
- # Test that the mailinglist table gets a list_id column.
- self._upgrade()
- with temporary_db(self._database):
- mlist = getUtility(IListManager).get('test@example.com')
- self.assertEqual(mlist.list_id, 'test.example.com')
-
- def test_list_id_member(self):
- # Test that the member table's mailing_list column becomes list_id.
- self._upgrade()
- with temporary_db(self._database):
- service = getUtility(ISubscriptionService)
- members = list(service.find_members(list_id='test.example.com'))
- self.assertEqual(len(members), 4)
-
- def test_news_moderation_none(self):
- # Test that news_moderation becomes newsgroup_moderation.
- self._database.store.execute(
- 'UPDATE mailinglist SET news_moderation = 0 '
- 'WHERE id = 1;')
- self._upgrade()
- with temporary_db(self._database):
- mlist = getUtility(IListManager).get('test@example.com')
- self.assertEqual(mlist.newsgroup_moderation,
- NewsgroupModeration.none)
-
- def test_news_moderation_open_moderated(self):
- # Test that news_moderation becomes newsgroup_moderation.
- self._database.store.execute(
- 'UPDATE mailinglist SET news_moderation = 1 '
- 'WHERE id = 1;')
- self._upgrade()
- with temporary_db(self._database):
- mlist = getUtility(IListManager).get('test@example.com')
- self.assertEqual(mlist.newsgroup_moderation,
- NewsgroupModeration.open_moderated)
-
- def test_news_moderation_moderated(self):
- # Test that news_moderation becomes newsgroup_moderation.
- self._database.store.execute(
- 'UPDATE mailinglist SET news_moderation = 2 '
- 'WHERE id = 1;')
- self._upgrade()
- with temporary_db(self._database):
- mlist = getUtility(IListManager).get('test@example.com')
- self.assertEqual(mlist.newsgroup_moderation,
- NewsgroupModeration.moderated)
-
- def test_nntp_prefix_subject_too_false(self):
- # Test that news_prefix_subject_too becomes nntp_prefix_subject_too.
- self._database.store.execute(
- 'UPDATE mailinglist SET news_prefix_subject_too = {0} '
- 'WHERE id = 1;'.format(self._database.FALSE))
- self._upgrade()
- with temporary_db(self._database):
- mlist = getUtility(IListManager).get('test@example.com')
- self.assertFalse(mlist.nntp_prefix_subject_too)
-
- def test_nntp_prefix_subject_too_true(self):
- # Test that news_prefix_subject_too becomes nntp_prefix_subject_too.
- self._database.store.execute(
- 'UPDATE mailinglist SET news_prefix_subject_too = {0} '
- 'WHERE id = 1;'.format(self._database.TRUE))
- self._upgrade()
- with temporary_db(self._database):
- mlist = getUtility(IListManager).get('test@example.com')
- self.assertTrue(mlist.nntp_prefix_subject_too)
-
- def test_allow_list_posts_false(self):
- # Test that include_list_post_header -> allow_list_posts.
- self._database.store.execute(
- 'UPDATE mailinglist SET include_list_post_header = {0} '
- 'WHERE id = 1;'.format(self._database.FALSE))
- self._upgrade()
- with temporary_db(self._database):
- mlist = getUtility(IListManager).get('test@example.com')
- self.assertFalse(mlist.allow_list_posts)
-
- def test_allow_list_posts_true(self):
- # Test that include_list_post_header -> allow_list_posts.
- self._database.store.execute(
- 'UPDATE mailinglist SET include_list_post_header = {0} '
- 'WHERE id = 1;'.format(self._database.TRUE))
- self._upgrade()
- with temporary_db(self._database):
- mlist = getUtility(IListManager).get('test@example.com')
- self.assertTrue(mlist.allow_list_posts)
-
-
-
-class TestMigration20121015Schema(MigrationTestBase):
- """Test column migrations."""
-
- def test_pre_upgrade_column_migrations(self):
- self._missing_present('ban',
- ['20121014999999'],
- ('list_id',),
- ('mailing_list',))
- self._missing_present('mailinglist',
- ['20121014999999'],
- (),
- ('new_member_options', 'send_reminders',
- 'subscribe_policy', 'unsubscribe_policy',
- 'subscribe_auto_approval', 'private_roster',
- 'admin_member_chunksize'),
- )
-
- def test_post_upgrade_column_migrations(self):
- self._missing_present('ban',
- ['20121014999999',
- '20121015000000'],
- ('mailing_list',),
- ('list_id',))
- self._missing_present('mailinglist',
- ['20121014999999',
- '20121015000000'],
- ('new_member_options', 'send_reminders',
- 'subscribe_policy', 'unsubscribe_policy',
- 'subscribe_auto_approval', 'private_roster',
- 'admin_member_chunksize'),
- ())
-
-
-
-class TestMigration20121015MigratedData(MigrationTestBase):
- """Test non-migrated data."""
-
- def test_migration_bans(self):
- # Load all the migrations to just before the one we're testing.
- self._database.load_migrations('20121014999999')
- # Insert a list-specific ban.
- self._database.store.execute("""
- INSERT INTO ban VALUES (
- 1, 'anne@example.com', 'test@example.com');
- """)
- # Insert a global ban.
- self._database.store.execute("""
- INSERT INTO ban VALUES (
- 2, 'bart@example.com', NULL);
- """)
- # Update to the current migration we're testing.
- self._database.load_migrations('20121015000000')
- # Now both the local and global bans should still be present.
- bans = sorted(self._database.store.find(Ban),
- key=attrgetter('email'))
- self.assertEqual(bans[0].email, 'anne@example.com')
- self.assertEqual(bans[0].list_id, 'test.example.com')
- self.assertEqual(bans[1].email, 'bart@example.com')
- self.assertEqual(bans[1].list_id, None)
-
-
-
-class TestMigration20130406Schema(MigrationTestBase):
- """Test column migrations."""
-
- def test_pre_upgrade_column_migrations(self):
- self._missing_present('bounceevent',
- ['20130405999999'],
- ('list_id',),
- ('list_name',))
-
- def test_post_upgrade_column_migrations(self):
- self._missing_present('bounceevent',
- ['20130405999999',
- '20130406000000'],
- ('list_name',),
- ('list_id',))
-
- def test_pre_listarchiver_table(self):
- self._table_missing_present(['20130405999999'], ('listarchiver',), ())
-
- def test_post_listarchiver_table(self):
- self._table_missing_present(['20130405999999',
- '20130406000000'],
- (),
- ('listarchiver',))
-
-
-
-class TestMigration20130406MigratedData(MigrationTestBase):
- """Test migrated data."""
-
- def test_migration_bounceevent(self):
- # Load all migrations to just before the one we're testing.
- self._database.load_migrations('20130405999999')
- # Insert a bounce event.
- self._database.store.execute("""
- INSERT INTO bounceevent VALUES (
- 1, 'test@example.com', 'anne@example.com',
- '2013-04-06 21:12:00', '<abc@example.com>',
- 1, 0);
- """)
- # Update to the current migration we're testing
- self._database.load_migrations('20130406000000')
- # The bounce event should exist, but with a list-id instead of a fqdn
- # list name.
- events = list(self._database.store.find(BounceEvent))
- self.assertEqual(len(events), 1)
- self.assertEqual(events[0].list_id, 'test.example.com')
- self.assertEqual(events[0].email, 'anne@example.com')
- self.assertEqual(events[0].timestamp, datetime(2013, 4, 6, 21, 12))
- self.assertEqual(events[0].message_id, '<abc@example.com>')
- self.assertEqual(events[0].context, BounceContext.normal)
- self.assertFalse(events[0].processed)
diff --git a/src/mailman/database/types.py b/src/mailman/database/types.py
index ba3d92df4..1984b08b5 100644
--- a/src/mailman/database/types.py
+++ b/src/mailman/database/types.py
@@ -23,43 +23,70 @@ from __future__ import absolute_import, print_function, unicode_literals
__metaclass__ = type
__all__ = [
'Enum',
+ 'UUID',
]
+import uuid
-from storm.properties import SimpleProperty
-from storm.variables import Variable
+from sqlalchemy import Integer
+from sqlalchemy.dialects import postgresql
+from sqlalchemy.types import TypeDecorator, CHAR
-class _EnumVariable(Variable):
- """Storm variable for supporting enum types.
+class Enum(TypeDecorator):
+ """Handle Python 3.4 style enums.
- To use this, make the database column a INTEGER.
+ Stores an integer-based Enum as an integer in the database, and
+ converts it on-the-fly.
"""
+ impl = Integer
- def __init__(self, *args, **kws):
- self._enum = kws.pop('enum')
- super(_EnumVariable, self).__init__(*args, **kws)
+ def __init__(self, enum, *args, **kw):
+ self.enum = enum
+ super(Enum, self).__init__(*args, **kw)
- def parse_set(self, value, from_db):
+ def process_bind_param(self, value, dialect):
if value is None:
return None
- if not from_db:
- return value
- return self._enum(value)
+ return value.value
- def parse_get(self, value, to_db):
+ def process_result_value(self, value, dialect):
if value is None:
return None
- if not to_db:
- return value
- return value.value
+ return self.enum(value)
-class Enum(SimpleProperty):
- """Custom type for Storm supporting enums."""
+
+class UUID(TypeDecorator):
+ """Platform-independent GUID type.
+
+ Uses Postgresql's UUID type, otherwise uses
+ CHAR(32), storing as stringified hex values.
- variable_class = _EnumVariable
+ """
+ impl = CHAR
- def __init__(self, enum=None):
- super(Enum, self).__init__(enum=enum)
+ def load_dialect_impl(self, dialect):
+ if dialect.name == 'postgresql':
+ return dialect.type_descriptor(postgresql.UUID())
+ else:
+ return dialect.type_descriptor(CHAR(32))
+
+ def process_bind_param(self, value, dialect):
+ if value is None:
+ return value
+ elif dialect.name == 'postgresql':
+ return str(value)
+ else:
+ if not isinstance(value, uuid.UUID):
+ return "%.32x" % uuid.UUID(value)
+ else:
+ # hexstring
+ return "%.32x" % value
+
+ def process_result_value(self, value, dialect):
+ if value is None:
+ return value
+ else:
+ return uuid.UUID(value)