diff options
Diffstat (limited to 'src/mailman/database')
| -rw-r--r-- | src/mailman/database/alembic/env.py | 4 | ||||
| -rw-r--r-- | src/mailman/database/alembic/versions/fa0d96e28631_template_manager.py | 178 | ||||
| -rw-r--r-- | src/mailman/database/base.py | 2 | ||||
| -rw-r--r-- | src/mailman/database/tests/test_migrations.py | 156 |
4 files changed, 337 insertions, 3 deletions
diff --git a/src/mailman/database/alembic/env.py b/src/mailman/database/alembic/env.py index e97b05d5c..c7a5e151d 100644 --- a/src/mailman/database/alembic/env.py +++ b/src/mailman/database/alembic/env.py @@ -28,11 +28,11 @@ from sqlalchemy import create_engine try: - url = expand(config.database.url, config.paths) + url = expand(config.database.url, None, config.paths) except AttributeError: # Initialize config object for external alembic calls initialize_1() - url = expand(config.database.url, config.paths) + url = expand(config.database.url, None, config.paths) @public diff --git a/src/mailman/database/alembic/versions/fa0d96e28631_template_manager.py b/src/mailman/database/alembic/versions/fa0d96e28631_template_manager.py new file mode 100644 index 000000000..5920dd4e4 --- /dev/null +++ b/src/mailman/database/alembic/versions/fa0d96e28631_template_manager.py @@ -0,0 +1,178 @@ +"""File cache and template manager. + +Revision ID: fa0d96e28631 +Revises: bfda02ab3a9b +Create Date: 2016-02-21 16:21:48.277654 +""" + +import os +import shutil +import sqlalchemy as sa + +from alembic import op +from mailman.config import config +from mailman.database.helpers import exists_in_db + + +# revision identifiers, used by Alembic. +revision = 'fa0d96e28631' +down_revision = '7b254d88f122' + + +CONVERSION_MAPPING = dict( + digest_footer_uri='list:digest:footer', + digest_header_uri='list:digest:header', + footer_uri='list:regular:footer', + goodbye_message_uri='user:ack:goodbye', + header_uri='list:regular:header', + welcome_message_uri='user:ack:welcome', + ) + +REVERSE_MAPPING = {value: key for key, value in CONVERSION_MAPPING.items()} + + +def upgrade(): + op.create_table( + 'file_cache', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('key', sa.Unicode(), nullable=False), + sa.Column('file_id', sa.Unicode(), nullable=True), + sa.Column('is_bytes', sa.Boolean(), nullable=False), + sa.Column('created_on', sa.DateTime(), nullable=False), + sa.Column('expires_on', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + template_table = op.create_table( + 'template', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.Unicode(), nullable=False), + sa.Column('context', sa.Unicode(), nullable=True), + sa.Column('uri', sa.Unicode(), nullable=False), + sa.Column('username', sa.Unicode(), nullable=True), + sa.Column('password', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + connection = op.get_bind() + # For all existing mailing lists, turn the *_uri attributes into entries + # in the template cache. Don't import the table definition from the + # models, it may break this migration when the model is updated in the + # future (see the Alembic doc). + mlist_table = sa.sql.table( + 'mailinglist', + sa.sql.column('id', sa.Integer), + sa.sql.column('list_id', sa.Unicode), + sa.sql.column('digest_footer_uri', sa.Unicode), + sa.sql.column('digest_header_uri', sa.Unicode), + sa.sql.column('footer_uri', sa.Unicode), + sa.sql.column('header_uri', sa.Unicode), + sa.sql.column('goodbye_message_uri', sa.Unicode), + sa.sql.column('welcome_message_uri', sa.Unicode), + ) + for (mlist_id, list_id, + digest_footer_uri, digest_header_uri, + nondigest_footer_uri, nondigest_header_uri, + goodbye_uri, welcome_uri + ) in connection.execute(mlist_table.select()): + inserts = [] + if digest_footer_uri is not None: + entry = dict( + name=CONVERSION_MAPPING['digest_footer_uri'], + uri=digest_footer_uri, + ) + inserts.append(entry) + if digest_header_uri is not None: + entry = dict( + name=CONVERSION_MAPPING['digest_header_uri'], + uri=digest_header_uri, + ) + inserts.append(entry) + if nondigest_footer_uri is not None: + entry = dict( + name=CONVERSION_MAPPING['footer_uri'], + uri=nondigest_footer_uri, + ) + inserts.append(entry) + if nondigest_header_uri is not None: + entry = dict( + name=CONVERSION_MAPPING['header_uri'], + uri=nondigest_header_uri, + ) + inserts.append(entry) + if goodbye_uri is not None: + entry = dict( + name=CONVERSION_MAPPING['goodbye_message_uri'], + uri=goodbye_uri, + ) + inserts.append(entry) + if welcome_uri is not None: + entry = dict( + name=CONVERSION_MAPPING['welcome_message_uri'], + uri=welcome_uri, + ) + inserts.append(entry) + for entry in inserts: + # In the source tree, footer-generic.txt was renamed. + entry['context'] = list_id + connection.execute(template_table.insert().values(**entry)) + with op.batch_alter_table('mailinglist') as batch_op: + batch_op.drop_column('digest_footer_uri') + batch_op.drop_column('digest_header_uri') + batch_op.drop_column('footer_uri') + batch_op.drop_column('header_uri') + batch_op.drop_column('goodbye_message_uri') + batch_op.drop_column('welcome_message_uri') + with op.batch_alter_table('domain') as batch_op: + batch_op.drop_column('base_url') + + +def downgrade(): + # Add back the original columns to the mailinglist table. + for column in CONVERSION_MAPPING: + if not exists_in_db(op.get_bind(), 'mailinglist', column): + op.add_column( + 'mailinglist', + sa.Column(column, sa.Unicode, nullable=True)) + op.add_column('domain', sa.Column('base_url', sa.Unicode)) + # Put all the templates with a context mapping the list-id back into the + # mailinglist table. No other contexts are supported, so just throw those + # away. + template_table = sa.sql.table( + 'template', + sa.sql.column('id', sa.Integer), + sa.sql.column('name', sa.Unicode), + sa.sql.column('context', sa.Unicode), + sa.sql.column('uri', sa.Unicode), + sa.sql.column('username', sa.Unicode), + sa.sql.column('password', sa.Unicode), + ) + mlist_table = sa.sql.table( + 'mailinglist', + sa.sql.column('id', sa.Integer), + sa.sql.column('list_id', sa.Unicode), + sa.sql.column('digest_footer_uri', sa.Unicode), + sa.sql.column('digest_header_uri', sa.Unicode), + sa.sql.column('footer_uri', sa.Unicode), + sa.sql.column('header_uri', sa.Unicode), + sa.sql.column('goodbye_message_uri', sa.Unicode), + sa.sql.column('welcome_message_uri', sa.Unicode), + ) + connection = op.get_bind() + for (table_id, name, context, uri, username, password + ) in connection.execute(template_table.select()).fetchall(): + mlist = connection.execute(mlist_table.select().where( + mlist_table.c.list_id == context)).fetchone() + if mlist is None: + continue + attribute = REVERSE_MAPPING.get(name) + if attribute is not None: + connection.execute(mlist_table.update().where( + mlist_table.c.list_id == context).values( + **{attribute: uri})) + op.drop_table('file_cache') + op.drop_table('template') + # Also delete the file cache directories. Don't delete the cache + # directory itself though. + for path in os.listdir(config.CACHE_DIR): + full_path = os.path.join(config.CACHE_DIR, path) + if os.path.isdir(full_path): + shutil.rmtree(full_path) diff --git a/src/mailman/database/base.py b/src/mailman/database/base.py index 5cc582e56..dda3665af 100644 --- a/src/mailman/database/base.py +++ b/src/mailman/database/base.py @@ -85,7 +85,7 @@ class SABaseDatabase: def initialize(self, debug=None): """See `IDatabase`.""" # Calculate the engine url. - url = expand(config.database.url, config.paths) + url = expand(config.database.url, None, config.paths) self._prepare(url) log.debug('Database url: %s', url) # XXX By design of SQLite, database file creation does not honor diff --git a/src/mailman/database/tests/test_migrations.py b/src/mailman/database/tests/test_migrations.py index 1994c3854..d61154ee2 100644 --- a/src/mailman/database/tests/test_migrations.py +++ b/src/mailman/database/tests/test_migrations.py @@ -30,7 +30,9 @@ from mailman.database.model import Model from mailman.database.transaction import transaction from mailman.database.types import Enum from mailman.interfaces.action import Action +from mailman.interfaces.cache import ICacheManager from mailman.interfaces.member import MemberRole +from mailman.interfaces.template import ITemplateManager from mailman.interfaces.usermanager import IUserManager from mailman.testing.layers import ConfigLayer from zope.component import getUtility @@ -296,3 +298,157 @@ class TestMigrations(unittest.TestCase): (cris.id, Action.defer), (dana.id, Action.hold), ]) + + def test_fa0d96e28631_upgrade_uris(self): + with transaction(): + # Start at the previous revision. + alembic.command.downgrade(alembic_cfg, '7b254d88f122') + # Create a mailing list through the standard API. + create_list('ant@example.com') + mlist_table = sa.sql.table( + 'mailinglist', + sa.sql.column('id', sa.Integer), + sa.sql.column('list_id', sa.Unicode), + sa.sql.column('digest_footer_uri', sa.Unicode), + sa.sql.column('digest_header_uri', sa.Unicode), + sa.sql.column('footer_uri', sa.Unicode), + sa.sql.column('header_uri', sa.Unicode), + sa.sql.column('goodbye_message_uri', sa.Unicode), + sa.sql.column('welcome_message_uri', sa.Unicode), + ) + with transaction(): + config.db.store.execute(mlist_table.update().where( + mlist_table.c.list_id == 'ant.example.com').values( + digest_footer_uri='mailman:///digest_footer.txt', + digest_header_uri='mailman:///digest_header.txt', + footer_uri='mailman:///footer.txt', + header_uri='mailman:///header.txt', + goodbye_message_uri='mailman:///goodbye.txt', + welcome_message_uri='mailman:///welcome.txt', + )) + # Now upgrade and check to see if the values got into the template + # table correctly. + alembic.command.upgrade(alembic_cfg, 'fa0d96e28631') + seen_names = [] + template_table = sa.sql.table( + 'template', + sa.sql.column('id', sa.Integer), + sa.sql.column('name', sa.Unicode), + sa.sql.column('context', sa.Unicode), + sa.sql.column('uri', sa.Unicode), + sa.sql.column('username', sa.Unicode), + sa.sql.column('password', sa.DateTime), + ) + for (table_id, name, context, uri, + username, password) in config.db.store.execute( + template_table.select()): + # This information isn't available in the old database + # version, so there's no way these can be set. + seen_names.append(name) + self.assertIsNone(username) + self.assertIsNone(password) + self.assertEqual(context, 'ant.example.com') + self.assertEqual(uri, 'mailman:///{}.txt'.format({ + 'list:digest:footer': 'digest_footer', + 'list:digest:header': 'digest_header', + 'list:regular:footer': 'footer', + 'list:regular:header': 'header', + 'user:ack:goodbye': 'goodbye', + 'user:ack:welcome': 'welcome', + }.get(name, name))) + self.assertEqual(sorted(seen_names), [ + 'list:digest:footer', + 'list:digest:header', + 'list:regular:footer', + 'list:regular:header', + 'user:ack:goodbye', + 'user:ack:welcome', + ]) + + def test_fa0d96e28631_upgrade_no_uris(self): + # None of the URL parameters are defined. + with transaction(): + # Start at the previous revision. + alembic.command.downgrade(alembic_cfg, '7b254d88f122') + # Create a mailing list through the standard API. + create_list('ant@example.com') + # Now upgrade and check to see if the values got into the template + # table correctly. + alembic.command.upgrade(alembic_cfg, 'fa0d96e28631') + template_table = sa.sql.table( + 'template', + sa.sql.column('id', sa.Integer), + ) + entries = list(config.db.store.execute(template_table.select())) + self.assertEqual(len(entries), 0) + + def test_fa0d96e28631_downgrade_uris(self): + # Create some cache directory entries. + self.assertTrue(os.path.exists(config.CACHE_DIR)) + getUtility(ICacheManager).add('abc', 'def') + self.assertNotEqual(len(os.listdir(config.CACHE_DIR)), 0) + # Set up the templates using the current API. + with transaction(): + create_list('ant@example.com') + manager = getUtility(ITemplateManager) + manager.set('list:digest:footer', + 'ant.example.com', + 'mailman:///digest_footer.txt') + manager.set('list:digest:header', + 'ant.example.com', + 'mailman:///digest_header.txt') + manager.set('list:regular:footer', + 'ant.example.com', + 'mailman:///footer.txt') + manager.set('list:regular:header', + 'ant.example.com', + 'mailman:///header.txt') + manager.set('user:ack:welcome', + 'ant.example.com', + 'mailman:///welcome.txt') + manager.set('user:ack:goodbye', + 'ant.example.com', + 'mailman:///goodbye.txt') + mlist_table = sa.sql.table( + 'mailinglist', + sa.sql.column('id', sa.Integer), + sa.sql.column('list_id', sa.Unicode), + sa.sql.column('digest_footer_uri', sa.Unicode), + sa.sql.column('digest_header_uri', sa.Unicode), + sa.sql.column('footer_uri', sa.Unicode), + sa.sql.column('header_uri', sa.Unicode), + sa.sql.column('goodbye_message_uri', sa.Unicode), + sa.sql.column('welcome_message_uri', sa.Unicode), + ) + alembic.command.downgrade(alembic_cfg, '7b254d88f122') + for (table_id, list_id, digest_footer_uri, digest_header_uri, + footer_uri, header_uri, + goodbye_message_uri, + welcome_message_uri) in config.db.store.execute( + mlist_table.select()): + self.assertEqual(list_id, 'ant.example.com') + self.assertEqual(digest_footer_uri, 'mailman:///digest_footer.txt') + self.assertEqual(digest_header_uri, 'mailman:///digest_header.txt') + self.assertEqual(footer_uri, 'mailman:///footer.txt') + self.assertEqual(header_uri, 'mailman:///header.txt') + self.assertEqual(welcome_message_uri, 'mailman:///welcome.txt') + self.assertEqual(goodbye_message_uri, 'mailman:///goodbye.txt') + # The cache directories are gone too. + self.assertEqual(len(os.listdir(config.CACHE_DIR)), 0, + os.listdir(config.CACHE_DIR)) + + def test_fa0d96e28631_downgrade_missing_list(self): + with transaction(): + manager = getUtility(ITemplateManager) + manager.set('list:regular:footer', + 'missing.example.com', + 'mailman:///missing-footer.txt') + alembic.command.downgrade(alembic_cfg, '7b254d88f122') + mlist_table = sa.sql.table( + 'mailinglist', + sa.sql.column('id', sa.Integer), + sa.sql.column('footer_uri', sa.Unicode), + ) + self.assertEqual( + len(list(config.db.store.execute(mlist_table.select()))), + 0) |