diff options
20 files changed, 318 insertions, 69 deletions
diff --git a/README.rst b/README.rst index 736b886ba..6091ecdd0 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ Mailman - The GNU Mailing List Management System ================================================ -Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +Copyright (C) 1998-2012 by the Free Software Foundation, Inc. This is GNU Mailman, a mailing list management system distributed under the terms of the GNU General Public License (GPL) version 3 or later. The name of @@ -41,7 +41,7 @@ master_doc = 'README' # General information about the project. project = u'GNU Mailman' -copyright = u'1998-2011 by the Free Software Foundation, Inc.' +copyright = u'1998-2012 by the Free Software Foundation, Inc.' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/src/mailman/bin/mailman.py b/src/mailman/bin/mailman.py index 14d25f2a7..2a230b49f 100644 --- a/src/mailman/bin/mailman.py +++ b/src/mailman/bin/mailman.py @@ -44,7 +44,7 @@ def main(): parser = argparse.ArgumentParser( description=_("""\ The GNU Mailman mailing list management system - Copyright 1998-2011 by the Free Software Foundation, Inc. + Copyright 1998-2012 by the Free Software Foundation, Inc. http://www.list.org """), formatter_class=argparse.RawDescriptionHelpFormatter) diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg index c94a3828b..5a64db268 100644 --- a/src/mailman/config/schema.cfg +++ b/src/mailman/config/schema.cfg @@ -189,6 +189,9 @@ class: mailman.database.sqlite.SQLiteDatabase url: sqlite:///$DATA_DIR/mailman.db debug: no +# The module path to the migrations modules. +migrations_path: mailman.database.schema + [logging.template] # This defines various log settings. The options available are: # diff --git a/src/mailman/database/base.py b/src/mailman/database/base.py index e28ca1893..a69e99395 100644 --- a/src/mailman/database/base.py +++ b/src/mailman/database/base.py @@ -24,18 +24,18 @@ __all__ = [ import os +import sys import logging from flufl.lock import Lock from lazr.config import as_boolean +from pkg_resources import resource_listdir, resource_string from storm.cache import GenerationalCache from storm.locals import create_database, Store from zope.interface import implements -import mailman.version - from mailman.config import config -from mailman.interfaces.database import IDatabase, SchemaVersionMismatchError +from mailman.interfaces.database import IDatabase from mailman.model.version import Version from mailman.utilities.string import expand @@ -51,6 +51,10 @@ class StormBaseDatabase: Use this as a base class for your DB-specific derived classes. """ + # Tag used to distinguish the database being used. Override this in base + # classes. + TAG = '' + implements(IDatabase) def __init__(self): @@ -87,15 +91,6 @@ class StormBaseDatabase: """ raise NotImplementedError - def _get_schema(self): - """Return the database schema as a string. - - This will be loaded into the database when it is first created. - - Base classes *must* override this. - """ - raise NotImplementedError - def _pre_reset(self, store): """Clean up method for testing. @@ -147,34 +142,80 @@ class StormBaseDatabase: store = Store(database, GenerationalCache()) database.DEBUG = (as_boolean(config.database.debug) if debug is None else debug) - # Check the master / schema database to see if the version table - # exists. If so, then we assume the database schema is correctly - # initialized. Storm does not currently provide schema creation. - if not self._database_exists(store): - # Initialize the database. Start by getting the schema and - # discarding all blank and comment lines. - lines = self._get_schema().splitlines() - lines = (line for line in lines + self.store = store + self.load_migrations() + store.commit() + + def load_migrations(self): + """Load all not-yet loaded migrations.""" + migrations_path = config.database.migrations_path + if '.' in migrations_path: + parent, dot, child = migrations_path.rpartition('.') + else: + parent = migrations_path + child ='' + # If the database does not yet exist, load the base schema. + filenames = sorted(resource_listdir(parent, child)) + # Find out which schema migrations have already been loaded. + if self._database_exists(self.store): + versions = set(version.version for version in + self.store.find(Version, component='schema')) + else: + versions = set() + for filename in filenames: + module_fn, extension = os.path.splitext(filename) + if extension != '.py': + continue + parts = module_fn.split('_') + if len(parts) < 2: + continue + version = parts[1] + if version in versions: + # This one is already loaded. + continue + module_path = migrations_path + '.' + module_fn + __import__(module_path) + upgrade = getattr(sys.modules[module_path], 'upgrade', None) + if upgrade is None: + continue + upgrade(self, self.store, version, module_path) + + def load_schema(self, store, version, filename, module_path): + """Load the schema from a file. + + This is a helper method for migration classes to call. + + :param store: The Storm store to load the schema into. + :type store: storm.locals.Store` + :param version: The schema version identifier of the form + YYYYMMDDHHMMSS. + :type version: string + :param filename: The file name containing the schema to load. Pass + `None` if there is no schema file to load. + :type filename: string + :param module_path: The fully qualified Python module path to the + migration module being loaded. This is used to record information + for use by the test suite. + :type module_path: string + """ + if filename is not None: + contents = resource_string('mailman.database.schema', filename) + # Discard all blank and comment lines. + lines = (line for line in contents.splitlines() if line.strip() != '' and line.strip()[:2] != '--') sql = NL.join(lines) for statement in sql.split(';'): if statement.strip() != '': store.execute(statement + ';') - # Validate schema version. - v = store.find(Version, component='schema').one() - if not v: - # Database has not yet been initialized - v = Version(component='schema', - version=mailman.version.DATABASE_SCHEMA_VERSION) - store.add(v) - elif v.version <> mailman.version.DATABASE_SCHEMA_VERSION: - # XXX Update schema - raise SchemaVersionMismatchError(v.version) - self.store = store - store.commit() + # Add a marker that indicates the migration version being applied. + store.add(Version(component='schema', version=version)) + # Add a marker so that the module name can be found later. This is + # used by the test suite to reset the database between tests. + store.add(Version(component=version, version=module_path)) def _reset(self): """See `IDatabase`.""" from mailman.database.model import ModelMeta self.store.rollback() ModelMeta._reset(self.store) + self.store.commit() diff --git a/src/mailman/database/sql/__init__.py b/src/mailman/database/docs/__init__.py index e69de29bb..e69de29bb 100644 --- a/src/mailman/database/sql/__init__.py +++ b/src/mailman/database/docs/__init__.py diff --git a/src/mailman/database/docs/migration.rst b/src/mailman/database/docs/migration.rst new file mode 100644 index 000000000..930f7d245 --- /dev/null +++ b/src/mailman/database/docs/migration.rst @@ -0,0 +1,145 @@ +================= +Schema migrations +================= + +The SQL database schema will over time require upgrading to support new +features. This is supported via schema migration. + +Migrations are embodied in individual Python classes, which themselves may +load SQL into the database. The naming scheme for migration files is: + + s_YYYYMMDDHHMMSS_comment.py + +where `YYYYMMDDHHMMSS` is a required numeric year, month, day, hour, minute, +and second specifier providing unique ordering for processing. Only this +component of the file name is used to determine the ordering. (The `s_` +prefix is required due to Python module naming requirements). + +The optional `comment` part of the file name can be used as a short +description for the migration, although comments and docstrings in the +migration files should be used for more detailed descriptions. + +Migrations are applied automatically when Mailman starts up, but can also be +applied at any time by calling in the API directly. Once applied, a +migration's version string is registered so it will not be applied again. + +We see that the base migration is already applied. + + >>> from mailman.model.version import Version + >>> results = config.db.store.find(Version, component='schema') + >>> results.count() + 1 + >>> base = results.one() + >>> print base.component + schema + >>> print base.version + 00000000000000 + + +Migrations +========== + +Migrations can be loaded at any time, and can be found in the migrations path +specified in the configuration file. + +.. Create a temporary directory for the migrations:: + + >>> import os, sys, tempfile + >>> tempdir = tempfile.mkdtemp() + >>> path = os.path.join(tempdir, 'migrations') + >>> os.makedirs(path) + >>> sys.path.append(tempdir) + >>> config.push('migrations', """ + ... [database] + ... migrations_path: migrations + ... """) + +Here is an example migrations module. The key part of this interface is the +``upgrade()`` method, which takes four arguments: + + * `database` - The database class, as derived from `StormBaseDatabase` + * `store` - The Storm `Store` object. + * `version` - The version string as derived from the migrations module's file + name. This will include only the `YYYYMMDDHHMMSS` string. + * `module_path` - The dotted module path to the migrations module, suitable + for lookup in `sys.modules`. + +This migration module just adds a marker to the `version` table. + + >>> with open(os.path.join(path, '__init__.py'), 'w') as fp: + ... pass + >>> with open(os.path.join(path, 's_20120211000000.py'), 'w') as fp: + ... print >> fp, """ + ... from __future__ import unicode_literals + ... from mailman.model.version import Version + ... def upgrade(database, store, version, module_path): + ... v = Version(component='test', version=version) + ... store.add(v) + ... database.load_schema(store, version, None, module_path) + ... """ + +This will load the new migration, since it hasn't been loaded before. + + >>> config.db.load_migrations() + >>> results = config.db.store.find(Version, component='schema') + >>> for result in sorted(result.version for result in results): + ... print result + 00000000000000 + 20120211000000 + >>> test = config.db.store.find(Version, component='test').one() + >>> print test.version + 20120211000000 + +Migrations will only be loaded once. + + >>> with open(os.path.join(path, 's_20120211000001.py'), 'w') as fp: + ... print >> fp, """ + ... from __future__ import unicode_literals + ... from mailman.model.version import Version + ... _marker = 801 + ... def upgrade(database, store, version, module_path): + ... global _marker + ... # Pad enough zeros on the left to reach 14 characters wide. + ... marker = '{0:=#014d}'.format(_marker) + ... _marker += 1 + ... v = Version(component='test', version=marker) + ... store.add(v) + ... database.load_schema(store, version, None, module_path) + ... """ + +The first time we load this new migration, we'll get the 801 marker. + + >>> config.db.load_migrations() + >>> results = config.db.store.find(Version, component='schema') + >>> for result in sorted(result.version for result in results): + ... print result + 00000000000000 + 20120211000000 + 20120211000001 + >>> test = config.db.store.find(Version, component='test') + >>> for marker in sorted(marker.version for marker in test): + ... print marker + 00000000000801 + 20120211000000 + +We do not get an 802 marker because the migration has already been loaded. + + >>> config.db.load_migrations() + >>> results = config.db.store.find(Version, component='schema') + >>> for result in sorted(result.version for result in results): + ... print result + 00000000000000 + 20120211000000 + 20120211000001 + >>> test = config.db.store.find(Version, component='test') + >>> for marker in sorted(marker.version for marker in test): + ... print marker + 00000000000801 + 20120211000000 + +.. Clean up the temporary directory:: + + >>> config.pop('migrations') + >>> sys.path.remove(tempdir) + >>> import shutil + >>> shutil.rmtree(tempdir) diff --git a/src/mailman/database/model.py b/src/mailman/database/model.py index 9f6bf9845..c45517c9b 100644 --- a/src/mailman/database/model.py +++ b/src/mailman/database/model.py @@ -25,6 +25,8 @@ __all__ = [ ] +import sys + from operator import attrgetter from storm.properties import PropertyPublisherMeta @@ -50,12 +52,37 @@ class ModelMeta(PropertyPublisherMeta): @staticmethod def _reset(store): from mailman.config import config + from mailman.model.version import Version config.db._pre_reset(store) + # Give each schema migration a chance to do its pre-reset. See below + # for calling its post reset too. + versions = sorted(version.version for version in + store.find(Version, component='schema')) + migrations = {} + for version in versions: + # We have to give the migrations module that loaded this version a + # chance to do both pre- and post-reset operations. The following + # find the actual the module path for the migration. See + # StormBaseDatabase.load_schema(). + migration = store.find(Version, component=version).one() + if migration is None: + continue + migrations[version] = module_path = migration.version + module = sys.modules[module_path] + pre_reset = getattr(module, 'pre_reset', None) + if pre_reset is not None: + pre_reset(store) # Make sure this is deterministic, by sorting on the storm table name. classes = sorted(ModelMeta._class_registry, key=attrgetter('__storm_table__')) for model_class in classes: store.find(model_class).remove() + # Now give each migration a chance to do post-reset operations. + for version in versions: + module = sys.modules[migrations[version]] + post_reset = getattr(module, 'post_reset', None) + if post_reset is not None: + post_reset(store) config.db._post_reset(store) diff --git a/src/mailman/database/postgresql.py b/src/mailman/database/postgresql.py index 9ad8f74b5..988f7a1af 100644 --- a/src/mailman/database/postgresql.py +++ b/src/mailman/database/postgresql.py @@ -26,7 +26,6 @@ __all__ = [ from operator import attrgetter -from pkg_resources import resource_string from mailman.database.base import StormBaseDatabase @@ -35,6 +34,8 @@ from mailman.database.base import StormBaseDatabase class PostgreSQLDatabase(StormBaseDatabase): """Database class for PostgreSQL.""" + TAG = 'postgres' + def _database_exists(self, store): """See `BaseDatabase`.""" table_query = ('SELECT table_name FROM information_schema.tables ' @@ -43,16 +44,13 @@ class PostgreSQLDatabase(StormBaseDatabase): store.execute(table_query)) return 'version' in table_names - def _get_schema(self): - """See `BaseDatabase`.""" - return resource_string('mailman.database.sql', 'postgres.sql') - def _post_reset(self, store): """PostgreSQL-specific test suite cleanup. Reset the <tablename>_id_seq.last_value so that primary key ids restart from zero for new tests. """ + super(PostgreSQLDatabase, self)._post_reset(store) from mailman.database.model import ModelMeta classes = sorted(ModelMeta._class_registry, key=attrgetter('__storm_table__')) diff --git a/src/mailman/database/schema/__init__.py b/src/mailman/database/schema/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/mailman/database/schema/__init__.py diff --git a/src/mailman/database/sql/postgres.sql b/src/mailman/database/schema/postgres.sql index 60c05e340..bb209c4aa 100644 --- a/src/mailman/database/sql/postgres.sql +++ b/src/mailman/database/schema/postgres.sql @@ -319,7 +319,7 @@ CREATE TABLE pendedkeyvalue ( CREATE TABLE version ( id SERIAL NOT NULL, component TEXT, - version INTEGER, + version TEXT, PRIMARY KEY (id) ); diff --git a/src/mailman/database/schema/s_00000000000000_base.py b/src/mailman/database/schema/s_00000000000000_base.py new file mode 100644 index 000000000..d703088d6 --- /dev/null +++ b/src/mailman/database/schema/s_00000000000000_base.py @@ -0,0 +1,55 @@ +# Copyright (C) 2012 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Load the base schema.""" + +from __future__ import absolute_import, print_function, unicode_literals + +__metaclass__ = type +__all__ = [ + 'upgrade', + 'post_reset', + 'pre_reset', + ] + + +_migration_path = None +VERSION = '00000000000000' + + + +def upgrade(database, store, version, module_path): + filename = '{0}.sql'.format(database.TAG) + database.load_schema(store, version, filename, module_path) + + +def pre_reset(store): + global _migration_path + # Save the entry in the Version table for the test suite reset. This will + # be restored below. + from mailman.model.version import Version + result = store.find(Version, component=VERSION).one() + # Yes, we abuse this field. + _migration_path = result.version + + +def post_reset(store): + from mailman.model.version import Version + # We need to preserve the Version table entry for this migration, since + # its existence defines the fact that the tables have been loaded. + store.add(Version(component='schema', version=VERSION)) + store.add(Version(component=VERSION, version=_migration_path)) diff --git a/src/mailman/database/sql/sqlite.sql b/src/mailman/database/schema/sqlite.sql index 5987f1879..74d523cf2 100644 --- a/src/mailman/database/sql/sqlite.sql +++ b/src/mailman/database/schema/sqlite.sql @@ -295,7 +295,7 @@ CREATE INDEX ix_user_user_id ON user (_user_id); CREATE TABLE version ( id INTEGER NOT NULL, component TEXT, - version INTEGER, + version TEXT, PRIMARY KEY (id) ); diff --git a/src/mailman/database/sqlite.py b/src/mailman/database/sqlite.py index f48ea19c7..2677d0d71 100644 --- a/src/mailman/database/sqlite.py +++ b/src/mailman/database/sqlite.py @@ -27,7 +27,6 @@ __all__ = [ import os -from pkg_resources import resource_string from urlparse import urlparse from mailman.database.base import StormBaseDatabase @@ -37,6 +36,8 @@ from mailman.database.base import StormBaseDatabase class SQLiteDatabase(StormBaseDatabase): """Database class for SQLite.""" + TAG = 'sqlite' + def _database_exists(self, store): """See `BaseDatabase`.""" table_query = 'select tbl_name from sqlite_master;' @@ -53,7 +54,3 @@ class SQLiteDatabase(StormBaseDatabase): # Ignore errors if fd > 0: os.close(fd) - - def _get_schema(self): - """See `BaseDatabase`.""" - return resource_string('mailman.database.sql', 'sqlite.sql') diff --git a/src/mailman/docs/ACKNOWLEDGMENTS.rst b/src/mailman/docs/ACKNOWLEDGMENTS.rst index 36a386b57..b6c6cc16c 100644 --- a/src/mailman/docs/ACKNOWLEDGMENTS.rst +++ b/src/mailman/docs/ACKNOWLEDGMENTS.rst @@ -4,7 +4,7 @@ GNU Mailman Acknowledgments =========================== -Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +Copyright (C) 1998-2012 by the Free Software Foundation, Inc. Core Developers diff --git a/src/mailman/docs/INTRODUCTION.rst b/src/mailman/docs/INTRODUCTION.rst index e67439613..fd64efb42 100644 --- a/src/mailman/docs/INTRODUCTION.rst +++ b/src/mailman/docs/INTRODUCTION.rst @@ -18,7 +18,7 @@ Learn more about GNU Mailman in the `Getting Started`_ documentation. Copyright ========= -Copyright 1998-2011 by the Free Software Foundation, Inc. +Copyright 1998-2012 by the Free Software Foundation, Inc. This file is part of GNU Mailman. diff --git a/src/mailman/docs/NEWS.rst b/src/mailman/docs/NEWS.rst index f101cae06..cdd81d49e 100644 --- a/src/mailman/docs/NEWS.rst +++ b/src/mailman/docs/NEWS.rst @@ -2,7 +2,7 @@ Mailman - The GNU Mailing List Management System ================================================ -Copyright (C) 1998-2011 by the Free Software Foundation, Inc. +Copyright (C) 1998-2012 by the Free Software Foundation, Inc. 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA Here is a history of user visible changes to Mailman. @@ -14,6 +14,7 @@ Here is a history of user visible changes to Mailman. Architecture ------------ + * Schema migrations have been implemented. * Implement the style manager as a utility instead of an attribute hanging off the `mailman.config.config` object. * PostgreSQL support contributed by Stephen A. Goss. (LP: #860159) diff --git a/src/mailman/interfaces/database.py b/src/mailman/interfaces/database.py index e628354b1..0530f83b9 100644 --- a/src/mailman/interfaces/database.py +++ b/src/mailman/interfaces/database.py @@ -23,14 +23,12 @@ __metaclass__ = type __all__ = [ 'DatabaseError', 'IDatabase', - 'SchemaVersionMismatchError', ] from zope.interface import Interface from mailman.interfaces.errors import MailmanError -from mailman.version import DATABASE_SCHEMA_VERSION @@ -38,19 +36,6 @@ class DatabaseError(MailmanError): """A problem with the database occurred.""" -class SchemaVersionMismatchError(DatabaseError): - """The database schema version number did not match what was expected.""" - - def __init__(self, got): - super(SchemaVersionMismatchError, self).__init__() - self._got = got - - def __str__(self): - return ('Incompatible database schema version ' - '(got: {0}, expected: {1})'.format( - self._got, DATABASE_SCHEMA_VERSION)) - - class IDatabase(Interface): """Database layer interface.""" diff --git a/src/mailman/model/version.py b/src/mailman/model/version.py index 8d198a670..d6a4f3938 100644 --- a/src/mailman/model/version.py +++ b/src/mailman/model/version.py @@ -32,7 +32,7 @@ from mailman.database.model import Model class Version(Model): id = Int(primary=True) component = Unicode() - version = Int() + version = Unicode() def __init__(self, component, version): super(Version, self).__init__() diff --git a/src/mailman/version.py b/src/mailman/version.py index 079f3dd8d..00c5b6c7a 100644 --- a/src/mailman/version.py +++ b/src/mailman/version.py @@ -40,9 +40,6 @@ HEX_VERSION = ((MAJOR_REV << 24) | (MINOR_REV << 16) | (MICRO_REV << 8) | (REL_LEVEL << 4) | (REL_SERIAL << 0)) -# SQL database schema version -DATABASE_SCHEMA_VERSION = 1 - # qfile/*.db schema version number QFILE_SCHEMA_VERSION = 3 |