diff options
| author | Barry Warsaw | 2016-07-29 19:28:46 -0400 |
|---|---|---|
| committer | Barry Warsaw | 2016-07-29 19:28:46 -0400 |
| commit | febb5289e82c4424cdbcc2297e967bd894cbc8cf (patch) | |
| tree | bd71f4cbf6988049ac4d5dd65ceb7d5cc51902e7 /src | |
| parent | 90e84bee5f47cbcdb9e9c367c60a877e325ef3e7 (diff) | |
| download | mailman-febb5289e82c4424cdbcc2297e967bd894cbc8cf.tar.gz mailman-febb5289e82c4424cdbcc2297e967bd894cbc8cf.tar.zst mailman-febb5289e82c4424cdbcc2297e967bd894cbc8cf.zip | |
Diffstat (limited to 'src')
35 files changed, 357 insertions, 181 deletions
diff --git a/src/mailman/database/alembic/versions/2bb9b382198_workflow_state_table.py b/src/mailman/database/alembic/versions/2bb9b382198_workflow_state_table.py index 64def0253..df7a10e00 100644 --- a/src/mailman/database/alembic/versions/2bb9b382198_workflow_state_table.py +++ b/src/mailman/database/alembic/versions/2bb9b382198_workflow_state_table.py @@ -9,6 +9,7 @@ Create Date: 2015-03-25 18:09:18.338790 import sqlalchemy as sa from alembic import op +from mailman.database.types import SAUnicode # Revision identifiers, used by Alembic. @@ -19,10 +20,10 @@ down_revision = '16c2b25c7b' def upgrade(): op.create_table( 'workflowstate', - sa.Column('name', sa.Unicode(), nullable=False), - sa.Column('token', sa.Unicode(), nullable=False), - sa.Column('step', sa.Unicode(), nullable=True), - sa.Column('data', sa.Unicode(), nullable=True), + sa.Column('name', SAUnicode(), nullable=False), + sa.Column('token', SAUnicode(), nullable=False), + sa.Column('step', SAUnicode(), nullable=True), + sa.Column('data', SAUnicode(), nullable=True), sa.PrimaryKeyConstraint('name', 'token') ) diff --git a/src/mailman/database/alembic/versions/33bc0099223_add_member_indexes.py b/src/mailman/database/alembic/versions/33bc0099223_add_member_indexes.py index ce8f2f3f1..5f01f4a66 100644 --- a/src/mailman/database/alembic/versions/33bc0099223_add_member_indexes.py +++ b/src/mailman/database/alembic/versions/33bc0099223_add_member_indexes.py @@ -7,6 +7,7 @@ Create Date: 2015-11-19 23:04:42.449553 """ from alembic import op +from mailman.database.helpers import is_mysql # Revision identifiers, used by Alembic. @@ -15,22 +16,28 @@ down_revision = '42756496720' def upgrade(): - op.create_index(op.f('ix_member_address_id'), - 'member', ['address_id'], - unique=False) - op.create_index(op.f('ix_member_preferences_id'), - 'member', ['preferences_id'], - unique=False) - op.create_index(op.f('ix_member_user_id'), - 'member', ['user_id'], - unique=False) op.create_index(op.f('ix_address_email'), 'address', ['email'], unique=False) + # MySQL automatically creates the indexes for primary keys so don't need + # to do it explicitly again. + if not is_mysql(op.get_bind()): + op.create_index(op.f('ix_member_address_id'), + 'member', ['address_id'], + unique=False) + op.create_index(op.f('ix_member_preferences_id'), + 'member', ['preferences_id'], + unique=False) + op.create_index(op.f('ix_member_user_id'), + 'member', ['user_id'], + unique=False) def downgrade(): op.drop_index(op.f('ix_address_email'), table_name='address') - op.drop_index(op.f('ix_member_user_id'), table_name='member') - op.drop_index(op.f('ix_member_preferences_id'), table_name='member') - op.drop_index(op.f('ix_member_address_id'), table_name='member') + # MySQL automatically creates and removes the indexes for primary keys. + # So, you cannot drop it without removing the foreign key constraint. + if not is_mysql(op.get_bind()): + op.drop_index(op.f('ix_member_user_id'), table_name='member') + op.drop_index(op.f('ix_member_preferences_id'), table_name='member') + op.drop_index(op.f('ix_member_address_id'), table_name='member') diff --git a/src/mailman/database/alembic/versions/33e1f5f6fa8_.py b/src/mailman/database/alembic/versions/33e1f5f6fa8_.py index a8b572518..72754d3c5 100644 --- a/src/mailman/database/alembic/versions/33e1f5f6fa8_.py +++ b/src/mailman/database/alembic/versions/33e1f5f6fa8_.py @@ -29,6 +29,7 @@ import sqlalchemy as sa from alembic import op from mailman.database.helpers import is_sqlite +from mailman.database.types import SAUnicode # Revision identifiers, used by Alembic. @@ -50,7 +51,7 @@ def upgrade(): # SQLite does not support altering columns. return for table, column in COLUMNS_TO_CHANGE: - op.alter_column(table, column, type_=sa.Unicode) + op.alter_column(table, column, type_=SAUnicode) def downgrade(): diff --git a/src/mailman/database/alembic/versions/42756496720_header_matches.py b/src/mailman/database/alembic/versions/42756496720_header_matches.py index 4514fb398..d1308a134 100644 --- a/src/mailman/database/alembic/versions/42756496720_header_matches.py +++ b/src/mailman/database/alembic/versions/42756496720_header_matches.py @@ -10,7 +10,7 @@ import sqlalchemy as sa from alembic import op from mailman.database.helpers import exists_in_db, is_sqlite - +from mailman.database.types import SAUnicode # Revision identifiers, used by Alembic. revision = '42756496720' @@ -23,9 +23,9 @@ def upgrade(): 'headermatch', sa.Column('id', sa.Integer(), nullable=False), sa.Column('mailing_list_id', sa.Integer(), nullable=True), - sa.Column('header', sa.Unicode(), nullable=False), - sa.Column('pattern', sa.Unicode(), nullable=False), - sa.Column('chain', sa.Unicode(), nullable=True), + sa.Column('header', SAUnicode(), nullable=False), + sa.Column('pattern', SAUnicode(), nullable=False), + sa.Column('chain', SAUnicode(), nullable=True), sa.ForeignKeyConstraint(['mailing_list_id'], ['mailinglist.id'], ), sa.PrimaryKeyConstraint('id') ) @@ -73,8 +73,8 @@ def downgrade(): header_match_table = sa.sql.table( 'headermatch', sa.sql.column('mailing_list_id', sa.Integer), - sa.sql.column('header', sa.Unicode), - sa.sql.column('pattern', sa.Unicode), + sa.sql.column('header', SAUnicode), + sa.sql.column('pattern', SAUnicode), ) for mlist_id, header, pattern in connection.execute( header_match_table.select()).fetchall(): diff --git a/src/mailman/database/alembic/versions/46e92facee7_add_serverowner_domainowner.py b/src/mailman/database/alembic/versions/46e92facee7_add_serverowner_domainowner.py index 1c689670a..bf734934b 100644 --- a/src/mailman/database/alembic/versions/46e92facee7_add_serverowner_domainowner.py +++ b/src/mailman/database/alembic/versions/46e92facee7_add_serverowner_domainowner.py @@ -56,8 +56,9 @@ def downgrade(): if not is_sqlite(op.get_bind()): op.drop_column('user', 'is_server_owner') if not exists_in_db(op.get_bind(), 'domain', 'contact_address'): - # SQLite may not have removed it. + # SQLite may not have removed it. Add a fixed length VARCHAR for + # MySQL. op.add_column( 'domain', - sa.Column('contact_address', sa.VARCHAR(), nullable=True)) + sa.Column('contact_address', sa.VARCHAR(255), nullable=True)) op.drop_table('domain_owner') diff --git a/src/mailman/database/alembic/versions/47294d3a604_pendable_indexes.py b/src/mailman/database/alembic/versions/47294d3a604_pendable_indexes.py index 5e465af24..4aafb4613 100644 --- a/src/mailman/database/alembic/versions/47294d3a604_pendable_indexes.py +++ b/src/mailman/database/alembic/versions/47294d3a604_pendable_indexes.py @@ -13,6 +13,7 @@ import json import sqlalchemy as sa from alembic import op +from mailman.database.types import SAUnicode # revision identifiers, used by Alembic. @@ -34,8 +35,8 @@ pended_table = sa.sql.table( keyvalue_table = sa.sql.table( 'pendedkeyvalue', sa.sql.column('id', sa.Integer), - sa.sql.column('key', sa.Unicode), - sa.sql.column('value', sa.Unicode), + sa.sql.column('key', SAUnicode), + sa.sql.column('value', SAUnicode), sa.sql.column('pended_id', sa.Integer), ) diff --git a/src/mailman/database/alembic/versions/70af5a4e5790_digests.py b/src/mailman/database/alembic/versions/70af5a4e5790_digests.py index 50c87ccf3..1f9a93bcd 100644 --- a/src/mailman/database/alembic/versions/70af5a4e5790_digests.py +++ b/src/mailman/database/alembic/versions/70af5a4e5790_digests.py @@ -20,7 +20,10 @@ down_revision = '47294d3a604' def upgrade(): with op.batch_alter_table('mailinglist') as batch_op: - batch_op.alter_column('digestable', new_column_name='digests_enabled') + batch_op.alter_column('digestable', + new_column_name='digests_enabled', + existing_type=sa.Boolean) + # All column modifications require existing types for Mysql. batch_op.drop_column('nondigestable') # Non-database migration: rename the list's data-path. for dirname in os.listdir(config.LIST_DATA_DIR): @@ -34,7 +37,9 @@ def upgrade(): def downgrade(): with op.batch_alter_table('mailinglist') as batch_op: - batch_op.alter_column('digests_enabled', new_column_name='digestable') + batch_op.alter_column('digests_enabled', + new_column_name='digestable', + existing_type=sa.Boolean) # The data for this column is lost, it's not used anyway. batch_op.add_column(sa.Column('nondigestable', sa.Boolean)) for dirname in os.listdir(config.LIST_DATA_DIR): diff --git a/src/mailman/database/alembic/versions/7b254d88f122_members_and_list_moderation_action.py b/src/mailman/database/alembic/versions/7b254d88f122_members_and_list_moderation_action.py index 6b4d7bfd1..9f19e3278 100644 --- a/src/mailman/database/alembic/versions/7b254d88f122_members_and_list_moderation_action.py +++ b/src/mailman/database/alembic/versions/7b254d88f122_members_and_list_moderation_action.py @@ -13,7 +13,7 @@ fallback to the list's default. import sqlalchemy as sa from alembic import op -from mailman.database.types import Enum +from mailman.database.types import Enum, SAUnicode from mailman.interfaces.action import Action from mailman.interfaces.member import MemberRole @@ -26,7 +26,7 @@ down_revision = 'd4fbb4fd34ca' mailinglist_table = sa.sql.table( 'mailinglist', sa.sql.column('id', sa.Integer), - sa.sql.column('list_id', sa.Unicode), + sa.sql.column('list_id', SAUnicode), sa.sql.column('default_member_action', Enum(Action)), sa.sql.column('default_nonmember_action', Enum(Action)), ) @@ -35,7 +35,7 @@ mailinglist_table = sa.sql.table( member_table = sa.sql.table( 'member', sa.sql.column('id', sa.Integer), - sa.sql.column('list_id', sa.Unicode), + sa.sql.column('list_id', SAUnicode), sa.sql.column('role', Enum(MemberRole)), sa.sql.column('moderation_action', Enum(Action)), ) diff --git a/src/mailman/database/alembic/versions/d4fbb4fd34ca_header_match_order.py b/src/mailman/database/alembic/versions/d4fbb4fd34ca_header_match_order.py index 2608bb812..313963b86 100644 --- a/src/mailman/database/alembic/versions/d4fbb4fd34ca_header_match_order.py +++ b/src/mailman/database/alembic/versions/d4fbb4fd34ca_header_match_order.py @@ -9,6 +9,7 @@ Create Date: 2016-02-01 15:57:09.807678 import sqlalchemy as sa from alembic import op +from mailman.database.helpers import is_mysql # Revision identifiers, used by Alembic. @@ -20,19 +21,34 @@ def upgrade(): with op.batch_alter_table('headermatch') as batch_op: batch_op.add_column( sa.Column('position', sa.Integer(), nullable=True)) - batch_op.alter_column( - 'mailing_list_id', existing_type=sa.INTEGER(), nullable=False) batch_op.create_index( op.f('ix_headermatch_position'), ['position'], unique=False) - batch_op.create_index( - op.f('ix_headermatch_mailing_list_id'), ['mailing_list_id'], - unique=False) + if not is_mysql(op.get_bind()): + # MySQL automatically creates indexes for primary keys. + batch_op.create_index( + op.f('ix_headermatch_mailing_list_id'), ['mailing_list_id'], + unique=False) + # MySQL doesn't allow changing columns used in a foreign key + # constrains since MySQL version 5.6. We need to drop the + # constraint before changing the column. But, since the + # constraint name is auto-generated, we can't really hardcode the + # name here to use batch_op.drop_constraint(). Until we have a + # better fix for this, it should be safe to skip this. + batch_op.alter_column( + 'mailing_list_id', existing_type=sa.INTEGER(), nullable=False) def downgrade(): with op.batch_alter_table('headermatch') as batch_op: - batch_op.drop_index(op.f('ix_headermatch_mailing_list_id')) batch_op.drop_index(op.f('ix_headermatch_position')) - batch_op.alter_column( - 'mailing_list_id', existing_type=sa.INTEGER(), nullable=True) batch_op.drop_column('position') + + if not is_mysql(op.get_bind()): + # MySQL automatically creates and removes the indexes for primary + # keys. So, you cannot drop it without removing the foreign key + # constraint. + batch_op.drop_index(op.f('ix_headermatch_mailing_list_id')) + # MySQL doesn't allow changing columns used in foreign_key + # constraints. + batch_op.alter_column( + 'mailing_list_id', existing_type=sa.INTEGER(), nullable=True) diff --git a/src/mailman/database/alembic/versions/fa0d96e28631_template_manager.py b/src/mailman/database/alembic/versions/fa0d96e28631_template_manager.py index 5920dd4e4..58d688453 100644 --- a/src/mailman/database/alembic/versions/fa0d96e28631_template_manager.py +++ b/src/mailman/database/alembic/versions/fa0d96e28631_template_manager.py @@ -12,6 +12,7 @@ import sqlalchemy as sa from alembic import op from mailman.config import config from mailman.database.helpers import exists_in_db +from mailman.database.types import SAUnicode # revision identifiers, used by Alembic. @@ -35,8 +36,8 @@ def upgrade(): op.create_table( 'file_cache', sa.Column('id', sa.Integer(), nullable=False), - sa.Column('key', sa.Unicode(), nullable=False), - sa.Column('file_id', sa.Unicode(), nullable=True), + sa.Column('key', SAUnicode(), nullable=False), + sa.Column('file_id', SAUnicode(), nullable=True), sa.Column('is_bytes', sa.Boolean(), nullable=False), sa.Column('created_on', sa.DateTime(), nullable=False), sa.Column('expires_on', sa.DateTime(), nullable=False), @@ -45,10 +46,10 @@ def upgrade(): template_table = op.create_table( 'template', sa.Column('id', sa.Integer(), nullable=False), - sa.Column('name', sa.Unicode(), nullable=False), - sa.Column('context', sa.Unicode(), nullable=True), - sa.Column('uri', sa.Unicode(), nullable=False), - sa.Column('username', sa.Unicode(), nullable=True), + sa.Column('name', SAUnicode(), nullable=False), + sa.Column('context', SAUnicode(), nullable=True), + sa.Column('uri', SAUnicode(), nullable=False), + sa.Column('username', SAUnicode(), nullable=True), sa.Column('password', sa.DateTime(), nullable=True), sa.PrimaryKeyConstraint('id') ) @@ -60,13 +61,13 @@ def upgrade(): mlist_table = sa.sql.table( 'mailinglist', sa.sql.column('id', sa.Integer), - sa.sql.column('list_id', sa.Unicode), - sa.sql.column('digest_footer_uri', sa.Unicode), - sa.sql.column('digest_header_uri', sa.Unicode), - sa.sql.column('footer_uri', sa.Unicode), - sa.sql.column('header_uri', sa.Unicode), - sa.sql.column('goodbye_message_uri', sa.Unicode), - sa.sql.column('welcome_message_uri', sa.Unicode), + sa.sql.column('list_id', SAUnicode), + sa.sql.column('digest_footer_uri', SAUnicode), + sa.sql.column('digest_header_uri', SAUnicode), + sa.sql.column('footer_uri', SAUnicode), + sa.sql.column('header_uri', SAUnicode), + sa.sql.column('goodbye_message_uri', SAUnicode), + sa.sql.column('welcome_message_uri', SAUnicode), ) for (mlist_id, list_id, digest_footer_uri, digest_header_uri, @@ -131,30 +132,30 @@ def downgrade(): if not exists_in_db(op.get_bind(), 'mailinglist', column): op.add_column( 'mailinglist', - sa.Column(column, sa.Unicode, nullable=True)) - op.add_column('domain', sa.Column('base_url', sa.Unicode)) + sa.Column(column, SAUnicode, nullable=True)) + op.add_column('domain', sa.Column('base_url', SAUnicode)) # Put all the templates with a context mapping the list-id back into the # mailinglist table. No other contexts are supported, so just throw those # away. template_table = sa.sql.table( 'template', sa.sql.column('id', sa.Integer), - sa.sql.column('name', sa.Unicode), - sa.sql.column('context', sa.Unicode), - sa.sql.column('uri', sa.Unicode), - sa.sql.column('username', sa.Unicode), - sa.sql.column('password', sa.Unicode), + sa.sql.column('name', SAUnicode), + sa.sql.column('context', SAUnicode), + sa.sql.column('uri', SAUnicode), + sa.sql.column('username', SAUnicode), + sa.sql.column('password', SAUnicode), ) mlist_table = sa.sql.table( 'mailinglist', sa.sql.column('id', sa.Integer), - sa.sql.column('list_id', sa.Unicode), - sa.sql.column('digest_footer_uri', sa.Unicode), - sa.sql.column('digest_header_uri', sa.Unicode), - sa.sql.column('footer_uri', sa.Unicode), - sa.sql.column('header_uri', sa.Unicode), - sa.sql.column('goodbye_message_uri', sa.Unicode), - sa.sql.column('welcome_message_uri', sa.Unicode), + sa.sql.column('list_id', SAUnicode), + sa.sql.column('digest_footer_uri', SAUnicode), + sa.sql.column('digest_header_uri', SAUnicode), + sa.sql.column('footer_uri', SAUnicode), + sa.sql.column('header_uri', SAUnicode), + sa.sql.column('goodbye_message_uri', SAUnicode), + sa.sql.column('welcome_message_uri', SAUnicode), ) connection = op.get_bind() for (table_id, name, context, uri, username, password diff --git a/src/mailman/database/base.py b/src/mailman/database/base.py index dda3665af..f57228029 100644 --- a/src/mailman/database/base.py +++ b/src/mailman/database/base.py @@ -103,7 +103,7 @@ class SABaseDatabase: # engines, and yes, we could have chmod'd the file after the fact, but # half dozen and all... self.url = url - self.engine = create_engine(url) + self.engine = create_engine(url, isolation_level='READ UNCOMMITTED') session = sessionmaker(bind=self.engine) self.store = session() self.store.commit() diff --git a/src/mailman/database/helpers.py b/src/mailman/database/helpers.py index f58d559c5..b1dc381df 100644 --- a/src/mailman/database/helpers.py +++ b/src/mailman/database/helpers.py @@ -28,6 +28,11 @@ def is_sqlite(bind): @public +def is_mysql(bind): + return bind.dialect.name == 'mysql' + + +@public def exists_in_db(bind, tablename, columnname=None): md = sa.MetaData() md.reflect(bind=bind) diff --git a/src/mailman/database/mysql.py b/src/mailman/database/mysql.py new file mode 100644 index 000000000..0c0de54a9 --- /dev/null +++ b/src/mailman/database/mysql.py @@ -0,0 +1,34 @@ +# Copyright (C) 2016 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""MySQL database support""" + +from mailman import public +from mailman.database.base import SABaseDatabase +from mailman.database.model import Model + + +@public +class MySQLDatabase(SABaseDatabase): + """Database class for MySQL.""" + + def _post_reset(self, store): + """Reset AUTO_INCREMENT counters for all the tables.""" + super()._post_reset(store) + tables = reversed(Model.metadata.sorted_tables) + for table in tables: + store.execute('ALTER TABLE {} AUTO_INCREMENT = 1;'.format(table)) diff --git a/src/mailman/database/tests/test_factory.py b/src/mailman/database/tests/test_factory.py index 74b45e43a..b0aa89fc4 100644 --- a/src/mailman/database/tests/test_factory.py +++ b/src/mailman/database/tests/test_factory.py @@ -24,10 +24,12 @@ from contextlib import suppress from mailman.config import config from mailman.database.alembic import alembic_cfg from mailman.database.factory import LAST_STORM_SCHEMA_VERSION, SchemaManager +from mailman.database.helpers import is_mysql from mailman.database.model import Model +from mailman.database.types import SAUnicode from mailman.interfaces.database import DatabaseError from mailman.testing.layers import ConfigLayer -from sqlalchemy import Column, Integer, MetaData, Table, Unicode +from sqlalchemy import Column, Integer, MetaData, Table from sqlalchemy.exc import OperationalError, ProgrammingError from sqlalchemy.schema import Index from unittest.mock import patch @@ -60,8 +62,8 @@ class TestSchemaManager(unittest.TestCase): version_table = Table( 'version', Model.metadata, Column('id', Integer, primary_key=True), - Column('component', Unicode), - Column('version', Unicode), + Column('component', SAUnicode), + Column('version', SAUnicode), ) version_table.create(config.db.engine) config.db.store.execute(version_table.insert().values( @@ -71,12 +73,15 @@ class TestSchemaManager(unittest.TestCase): # all DB engines... config.db.engine.execute( 'ALTER TABLE mailinglist ADD COLUMN acceptable_aliases_id INT') - Index('ix_user__user_id').drop(bind=config.db.engine) - # Don't pollute our main metadata object, create a new one. - md = MetaData() - user_table = Model.metadata.tables['user'].tometadata(md) - Index('ix_user_user_id', user_table.c._user_id).create( - bind=config.db.engine) + # In case of MySQL, you cannot create/drop indexes on primary keys + # manually as it is handled automatically by MySQL. + if not is_mysql(config.db.engine): + Index('ix_user__user_id').drop(bind=config.db.engine) + # Don't pollute our main metadata object, create a new one. + md = MetaData() + user_table = Model.metadata.tables['user'].tometadata(md) + Index('ix_user_user_id', user_table.c._user_id).create( + bind=config.db.engine) config.db.commit() def _drop_storm_database(self): @@ -88,10 +93,12 @@ class TestSchemaManager(unittest.TestCase): version = Model.metadata.tables['version'] version.drop(config.db.engine, checkfirst=True) Model.metadata.remove(version) - # If it's nonexistent, PostgreSQL raises a ProgrammingError, while - # SQLite raises an OperationalError. - with suppress(ProgrammingError, OperationalError): - Index('ix_user_user_id').drop(bind=config.db.engine) + # If it's nonexistent, PostgreSQL raises a ProgrammingError while + # SQLite raises an OperationalError. Since MySQL automatically handles + # indexes for primary keys, don't try doing it with that backend. + if not is_mysql(config.db.engine): + with suppress(ProgrammingError, OperationalError): + Index('ix_user_user_id').drop(bind=config.db.engine) config.db.commit() def test_current_database(self): diff --git a/src/mailman/database/tests/test_migrations.py b/src/mailman/database/tests/test_migrations.py index 88a2a30ab..2ad41ae53 100644 --- a/src/mailman/database/tests/test_migrations.py +++ b/src/mailman/database/tests/test_migrations.py @@ -28,7 +28,7 @@ from mailman.database.alembic import alembic_cfg from mailman.database.helpers import exists_in_db from mailman.database.model import Model from mailman.database.transaction import transaction -from mailman.database.types import Enum +from mailman.database.types import Enum, SAUnicode from mailman.interfaces.action import Action from mailman.interfaces.cache import ICacheManager from mailman.interfaces.member import MemberRole @@ -83,8 +83,8 @@ class TestMigrations(unittest.TestCase): header_match_table = sa.sql.table( 'headermatch', sa.sql.column('mailing_list_id', sa.Integer), - sa.sql.column('header', sa.Unicode), - sa.sql.column('pattern', sa.Unicode), + sa.sql.column('header', SAUnicode), + sa.sql.column('pattern', SAUnicode), ) # Bring the DB to the revision that is being tested. alembic.command.downgrade(alembic_cfg, '42756496720') @@ -126,8 +126,8 @@ class TestMigrations(unittest.TestCase): keyvalue_table = sa.sql.table( 'pendedkeyvalue', sa.sql.column('id', sa.Integer), - sa.sql.column('key', sa.Unicode), - sa.sql.column('value', sa.Unicode), + sa.sql.column('key', SAUnicode), + sa.sql.column('value', SAUnicode), sa.sql.column('pended_id', sa.Integer), ) def get_from_db(): # noqa: E301 @@ -230,14 +230,14 @@ class TestMigrations(unittest.TestCase): sa.sql.table( 'mailinglist', sa.sql.column('id', sa.Integer), - sa.sql.column('list_id', sa.Unicode), + sa.sql.column('list_id', SAUnicode), sa.sql.column('default_member_action', Enum(Action)), sa.sql.column('default_nonmember_action', Enum(Action)), ) member_table = sa.sql.table( 'member', sa.sql.column('id', sa.Integer), - sa.sql.column('list_id', sa.Unicode), + sa.sql.column('list_id', SAUnicode), sa.sql.column('address_id', sa.Integer), sa.sql.column('role', Enum(MemberRole)), sa.sql.column('moderation_action', Enum(Action)), diff --git a/src/mailman/database/types.py b/src/mailman/database/types.py index 28b261514..bbd040d96 100644 --- a/src/mailman/database/types.py +++ b/src/mailman/database/types.py @@ -22,7 +22,8 @@ import uuid from mailman import public from sqlalchemy import Integer from sqlalchemy.dialects import postgresql -from sqlalchemy.types import CHAR, TypeDecorator +from sqlalchemy.ext.compiler import compiles +from sqlalchemy.types import CHAR, TypeDecorator, Unicode @public @@ -80,3 +81,46 @@ class UUID(TypeDecorator): return value else: return uuid.UUID(value) + + +@public +class SAUnicode(TypeDecorator): + """Unicode datatype to support fixed length VARCHAR in MySQL. + + This type compiles to VARCHAR(255) in case of MySQL, and in case of + other dailects defaults to the Unicode type. This was created so + that we don't have to alter the output of the default Unicode data + type and it can still be used if needed in the codebase. + """ + impl = Unicode + + +@compiles(SAUnicode) +def default_sa_unicode(element, compiler, **kw): + return compiler.visit_Unicode(element, **kw) + + +@compiles(SAUnicode, 'mysql') +def compile_sa_unicode(element, compiler, **kw): + # We hardcode the collate here to make string comparison case sensitive. + return 'VARCHAR(255) COLLATE utf8_bin' + + +@public +class SAUnicodeLarge(TypeDecorator): + """Similar to SAUnicode type, but compiles to VARCHAR(510). + + This is double size of SAUnicode defined above. + """ + impl = Unicode + + +@compiles(SAUnicodeLarge, 'mysql') +def compile_sa_unicode_large(element, compiler, **kw): + # We hardcode the collate here to make string comparison case sensitive. + return 'VARCHAR(510) COLLATE utf8_bin' + + +@compiles(SAUnicode) +def defalt_sa_unicode_large(element, compiler, **kw): + return compiler.visit_unicode(element, **kw) diff --git a/src/mailman/docs/DATABASE.rst b/src/mailman/docs/DATABASE.rst index b27b1f788..cc7d09eb9 100644 --- a/src/mailman/docs/DATABASE.rst +++ b/src/mailman/docs/DATABASE.rst @@ -7,11 +7,11 @@ relational database. By default, Mailman uses Python's built-in SQLite3_ database, however, SQLAlchemy is compatible with PostgreSQL_ and MySQL, among possibly others. -Currently, Mailman is known to work with either the default SQLite3 database, -or PostgreSQL. (Volunteers to port it to other databases are welcome!). If -you want to use SQLite3, you generally don't need to change anything, but if -you want Mailman to use PostgreSQL, you'll need to set that up first, and then -change a configuration variable in your ``/etc/mailman.cfg`` file. +Currently, Mailman is known to work with the SQLite3, PostgreSQL, and MySQL +databases. (Volunteers to port it to other databases are welcome!). If you +want to use SQLite3, you generally don't need to change anything, but if you +want Mailman to use PostgreSQL or MySQL, you'll need to set those up first, +and then change a configuration variable in your ``/etc/mailman.cfg`` file. Two configuration variables control which database Mailman uses. The first names the class implementing the database interface. The second names the URL @@ -63,6 +63,42 @@ it:: My thanks to Stephen A. Goss for his contribution of PostgreSQL support. +MySQL +===== + +First, you need to configure MySQL itself. Lets say you create the `mailman` +database in MySQL via:: + + mysql> CREATE DATABASE mailman; + +In some cases, our test suite requires the default collation of the database +to be set to `utf8_unicode_ci`. You can change it via the MySQL command line +like this:: + + mysql> ALTER DATABASE mailman DEFAULT COLLATE utf8_unicode_ci; + +You would also need the Python driver `pymysql` for MySQL.:: + + $ pip install pymysql + +You would then need to set both the `class` and `url` variables in +`mailman.cfg` like so:: + + [database] + class: mailman.database.mysql.MySQLDatabase + url: mysql+pymysql://myuser:mypassword@mymysqlhost/mailman?charset=utf8&use_unicode=1 + +The last part of the url specifies the charset that client expects from the +server and to use Unicode via the flag `use_unicode`. You can find more about +these options on the `SQLAlchemy's MySQL page`_. + +If you have any problems, you may need to delete the database and re-create +it:: + + mysql> DROP DATABASE mailman; + mysql> CREATE DATABASE mailman; + + Database Migrations =================== @@ -101,3 +137,4 @@ integer in the database. A more complex migration would be needed for .. _MySQL: http://dev.mysql.com/ .. _`Ubuntu article`: https://help.ubuntu.com/community/PostgreSQL .. _`Alembic`: https://alembic.readthedocs.org/en/latest/ +.. _`SQLAlchemy's MySQL page`: http://docs.sqlalchemy.org/en/latest/dialects/mysql.html#unicode diff --git a/src/mailman/docs/NEWS.rst b/src/mailman/docs/NEWS.rst index 93bac61ad..f2c63a8f3 100644 --- a/src/mailman/docs/NEWS.rst +++ b/src/mailman/docs/NEWS.rst @@ -111,6 +111,11 @@ Command line ``[shell]history_file`` variable in mailman.cfg. Also, many useful names are pre-populated in the namespace of the shell. (Closes: #228) +Database +-------- + + * MySQL is now an officially supported database. Given by Abhilash Raj. + Interfaces ---------- * Implement reasons for why a message is being held for moderator approval. diff --git a/src/mailman/model/address.py b/src/mailman/model/address.py index 7a3485378..a31e9a429 100644 --- a/src/mailman/model/address.py +++ b/src/mailman/model/address.py @@ -20,10 +20,11 @@ from email.utils import formataddr from mailman import public from mailman.database.model import Model +from mailman.database.types import SAUnicode from mailman.interfaces.address import ( AddressVerificationEvent, IAddress, IEmailValidator) from mailman.utilities.datetime import now -from sqlalchemy import Column, DateTime, ForeignKey, Integer, Unicode +from sqlalchemy import Column, DateTime, ForeignKey, Integer from sqlalchemy.orm import backref, relationship from zope.component import getUtility from zope.event import notify @@ -38,9 +39,9 @@ class Address(Model): __tablename__ = 'address' id = Column(Integer, primary_key=True) - email = Column(Unicode, index=True) - _original = Column(Unicode) - display_name = Column(Unicode) + email = Column(SAUnicode, index=True) + _original = Column(SAUnicode) + display_name = Column(SAUnicode) _verified_on = Column('verified_on', DateTime) registered_on = Column(DateTime) diff --git a/src/mailman/model/bans.py b/src/mailman/model/bans.py index e690145a4..e2b6490eb 100644 --- a/src/mailman/model/bans.py +++ b/src/mailman/model/bans.py @@ -22,8 +22,9 @@ import re from mailman import public from mailman.database.model import Model from mailman.database.transaction import dbconnection +from mailman.database.types import SAUnicode from mailman.interfaces.bans import IBan, IBanManager -from sqlalchemy import Column, Integer, Unicode +from sqlalchemy import Column, Integer from zope.interface import implementer @@ -35,8 +36,8 @@ class Ban(Model): __tablename__ = 'ban' id = Column(Integer, primary_key=True) - email = Column(Unicode, index=True) - list_id = Column(Unicode, index=True) + email = Column(SAUnicode, index=True) + list_id = Column(SAUnicode, index=True) def __init__(self, email, list_id): super().__init__() diff --git a/src/mailman/model/bounce.py b/src/mailman/model/bounce.py index b7f217048..856d91f20 100644 --- a/src/mailman/model/bounce.py +++ b/src/mailman/model/bounce.py @@ -20,11 +20,11 @@ from mailman import public from mailman.database.model import Model from mailman.database.transaction import dbconnection -from mailman.database.types import Enum +from mailman.database.types import Enum, SAUnicode from mailman.interfaces.bounce import ( BounceContext, IBounceEvent, IBounceProcessor) from mailman.utilities.datetime import now -from sqlalchemy import Boolean, Column, DateTime, Integer, Unicode +from sqlalchemy import Boolean, Column, DateTime, Integer from zope.interface import implementer @@ -36,10 +36,10 @@ class BounceEvent(Model): __tablename__ = 'bounceevent' id = Column(Integer, primary_key=True) - list_id = Column(Unicode) - email = Column(Unicode) + list_id = Column(SAUnicode) + email = Column(SAUnicode) timestamp = Column(DateTime) - message_id = Column(Unicode) + message_id = Column(SAUnicode) context = Column(Enum(BounceContext)) processed = Column(Boolean) diff --git a/src/mailman/model/cache.py b/src/mailman/model/cache.py index d9c0d0445..6be5fbc9d 100644 --- a/src/mailman/model/cache.py +++ b/src/mailman/model/cache.py @@ -26,9 +26,10 @@ from mailman import public from mailman.config import config from mailman.database.model import Model from mailman.database.transaction import dbconnection +from mailman.database.types import SAUnicode from mailman.interfaces.cache import ICacheManager from mailman.utilities.datetime import now -from sqlalchemy import Boolean, Column, DateTime, Integer, Unicode +from sqlalchemy import Boolean, Column, DateTime, Integer from zope.interface import implementer @@ -36,8 +37,8 @@ class CacheEntry(Model): __tablename__ = 'file_cache' id = Column(Integer, primary_key=True) - key = Column(Unicode) - file_id = Column(Unicode) + key = Column(SAUnicode) + file_id = Column(SAUnicode) is_bytes = Column(Boolean) created_on = Column(DateTime) expires_on = Column(DateTime) diff --git a/src/mailman/model/domain.py b/src/mailman/model/domain.py index 575d157cf..5bb569585 100644 --- a/src/mailman/model/domain.py +++ b/src/mailman/model/domain.py @@ -20,13 +20,14 @@ from mailman import public from mailman.database.model import Model from mailman.database.transaction import dbconnection +from mailman.database.types import SAUnicode from mailman.interfaces.domain import ( BadDomainSpecificationError, DomainCreatedEvent, DomainCreatingEvent, DomainDeletedEvent, DomainDeletingEvent, IDomain, IDomainManager) from mailman.interfaces.user import IUser from mailman.interfaces.usermanager import IUserManager from mailman.model.mailinglist import MailingList -from sqlalchemy import Column, Integer, Unicode +from sqlalchemy import Column, Integer from sqlalchemy.orm import relationship from zope.component import getUtility from zope.event import notify @@ -42,8 +43,8 @@ class Domain(Model): id = Column(Integer, primary_key=True) - mail_host = Column(Unicode) - description = Column(Unicode) + mail_host = Column(SAUnicode) + description = Column(SAUnicode) owners = relationship('User', secondary='domain_owner', backref='domains') diff --git a/src/mailman/model/language.py b/src/mailman/model/language.py index f0a3326a8..596e580e1 100644 --- a/src/mailman/model/language.py +++ b/src/mailman/model/language.py @@ -19,8 +19,9 @@ from mailman import public from mailman.database.model import Model +from mailman.database.types import SAUnicode from mailman.interfaces.languages import ILanguage -from sqlalchemy import Column, Integer, Unicode +from sqlalchemy import Column, Integer from zope.interface import implementer @@ -32,4 +33,4 @@ class Language(Model): __tablename__ = 'language' id = Column(Integer, primary_key=True) - code = Column(Unicode) + code = Column(SAUnicode) diff --git a/src/mailman/model/mailinglist.py b/src/mailman/model/mailinglist.py index 9d7d63d9b..933384797 100644 --- a/src/mailman/model/mailinglist.py +++ b/src/mailman/model/mailinglist.py @@ -23,7 +23,7 @@ from mailman import public from mailman.config import config from mailman.database.model import Model from mailman.database.transaction import dbconnection -from mailman.database.types import Enum +from mailman.database.types import Enum, SAUnicode from mailman.interfaces.action import Action, FilterAction from mailman.interfaces.address import IAddress from mailman.interfaces.archiver import ArchivePolicy @@ -51,7 +51,7 @@ from mailman.utilities.filesystem import makedirs from mailman.utilities.string import expand from sqlalchemy import ( Boolean, Column, DateTime, Float, ForeignKey, Integer, Interval, - LargeBinary, PickleType, Unicode) + LargeBinary, PickleType) from sqlalchemy.event import listen from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import relationship @@ -78,9 +78,9 @@ class MailingList(Model): # are currently missing. # List identity - list_name = Column(Unicode, index=True) - mail_host = Column(Unicode, index=True) - _list_id = Column('list_id', Unicode, index=True, unique=True) + list_name = Column(SAUnicode, index=True) + mail_host = Column(SAUnicode, index=True) + _list_id = Column('list_id', SAUnicode, index=True, unique=True) allow_list_posts = Column(Boolean) include_rfc2369_headers = Column(Boolean) advertised = Column(Boolean) @@ -106,11 +106,11 @@ class MailingList(Model): # Automatic responses. autoresponse_grace_period = Column(Interval) autorespond_owner = Column(Enum(ResponseAction)) - autoresponse_owner_text = Column(Unicode) + autoresponse_owner_text = Column(SAUnicode) autorespond_postings = Column(Enum(ResponseAction)) - autoresponse_postings_text = Column(Unicode) + autoresponse_postings_text = Column(SAUnicode) autorespond_requests = Column(Enum(ResponseAction)) - autoresponse_request_text = Column(Unicode) + autoresponse_request_text = Column(SAUnicode) # Content filters. filter_action = Column(Enum(FilterAction)) filter_content = Column(Boolean) @@ -118,7 +118,7 @@ class MailingList(Model): convert_html_to_plaintext = Column(Boolean) # Bounces. bounce_info_stale_after = Column(Interval) # XXX - bounce_matching_headers = Column(Unicode) # XXX + bounce_matching_headers = Column(SAUnicode) # XXX bounce_notify_owner_on_disable = Column(Boolean) # XXX bounce_notify_owner_on_removal = Column(Boolean) # XXX bounce_score_threshold = Column(Integer) # XXX @@ -130,7 +130,7 @@ class MailingList(Model): # Miscellaneous default_member_action = Column(Enum(Action)) default_nonmember_action = Column(Enum(Action)) - description = Column(Unicode) + description = Column(SAUnicode) digests_enabled = Column(Boolean) digest_is_default = Column(Boolean) digest_send_periodic = Column(Boolean) @@ -144,36 +144,36 @@ class MailingList(Model): gateway_to_mail = Column(Boolean) gateway_to_news = Column(Boolean) hold_these_nonmembers = Column(PickleType) - info = Column(Unicode) - linked_newsgroup = Column(Unicode) + info = Column(SAUnicode) + linked_newsgroup = Column(SAUnicode) max_days_to_hold = Column(Integer) max_message_size = Column(Integer) max_num_recipients = Column(Integer) - member_moderation_notice = Column(Unicode) + member_moderation_notice = Column(SAUnicode) mime_is_default_digest = Column(Boolean) # FIXME: There should be no moderator_password moderator_password = Column(LargeBinary) # TODO : was RawStr() newsgroup_moderation = Column(Enum(NewsgroupModeration)) nntp_prefix_subject_too = Column(Boolean) - nonmember_rejection_notice = Column(Unicode) + nonmember_rejection_notice = Column(SAUnicode) obscure_addresses = Column(Boolean) - owner_chain = Column(Unicode) - owner_pipeline = Column(Unicode) + owner_chain = Column(SAUnicode) + owner_pipeline = Column(SAUnicode) personalize = Column(Enum(Personalization)) post_id = Column(Integer) - posting_chain = Column(Unicode) - posting_pipeline = Column(Unicode) - _preferred_language = Column('preferred_language', Unicode) - display_name = Column(Unicode) + posting_chain = Column(SAUnicode) + posting_pipeline = Column(SAUnicode) + _preferred_language = Column('preferred_language', SAUnicode) + display_name = Column(SAUnicode) reject_these_nonmembers = Column(PickleType) reply_goes_to_list = Column(Enum(ReplyToMunging)) - reply_to_address = Column(Unicode) + reply_to_address = Column(SAUnicode) require_explicit_destination = Column(Boolean) respond_to_post_requests = Column(Boolean) scrub_nondigest = Column(Boolean) send_goodbye_message = Column(Boolean) send_welcome_message = Column(Boolean) - subject_prefix = Column(Unicode) + subject_prefix = Column(SAUnicode) subscription_policy = Column(Enum(SubscriptionPolicy)) topics = Column(PickleType) topics_bodylines_limit = Column(Integer) @@ -487,7 +487,7 @@ class AcceptableAlias(Model): Integer, ForeignKey('mailinglist.id'), index=True, nullable=False) mailing_list = relationship('MailingList', backref='acceptablealias') - alias = Column(Unicode, index=True, nullable=False) + alias = Column(SAUnicode, index=True, nullable=False) def __init__(self, mailing_list, alias): super().__init__() @@ -545,7 +545,7 @@ class ListArchiver(Model): index=True, nullable=False) mailing_list = relationship('MailingList') - name = Column(Unicode, nullable=False) + name = Column(SAUnicode, nullable=False) _is_enabled = Column(Boolean) def __init__(self, mailing_list, archiver_name, system_archiver): @@ -617,9 +617,9 @@ class HeaderMatch(Model): index=True, nullable=False) _position = Column('position', Integer, index=True, default=0) - header = Column(Unicode) - pattern = Column(Unicode) - chain = Column(Unicode, nullable=True) + header = Column(SAUnicode) + pattern = Column(SAUnicode) + chain = Column(SAUnicode, nullable=True) def __init__(self, **kw): position = kw.pop('position', None) diff --git a/src/mailman/model/member.py b/src/mailman/model/member.py index 3aaa5e392..aacaa73b6 100644 --- a/src/mailman/model/member.py +++ b/src/mailman/model/member.py @@ -21,7 +21,7 @@ from mailman import public from mailman.core.constants import system_preferences from mailman.database.model import Model from mailman.database.transaction import dbconnection -from mailman.database.types import Enum, UUID +from mailman.database.types import Enum, SAUnicode, UUID from mailman.interfaces.action import Action from mailman.interfaces.address import IAddress from mailman.interfaces.listmanager import IListManager @@ -30,7 +30,7 @@ from mailman.interfaces.member import ( from mailman.interfaces.user import IUser, UnverifiedAddressError from mailman.interfaces.usermanager import IUserManager from mailman.utilities.uid import UIDFactory -from sqlalchemy import Column, ForeignKey, Integer, Unicode +from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.orm import relationship from zope.component import getUtility from zope.event import notify @@ -50,7 +50,7 @@ class Member(Model): id = Column(Integer, primary_key=True) _member_id = Column(UUID) role = Column(Enum(MemberRole), index=True) - list_id = Column(Unicode, index=True) + list_id = Column(SAUnicode, index=True) moderation_action = Column(Enum(Action)) address_id = Column(Integer, ForeignKey('address.id'), index=True) diff --git a/src/mailman/model/message.py b/src/mailman/model/message.py index c4fe63f1c..576baab5c 100644 --- a/src/mailman/model/message.py +++ b/src/mailman/model/message.py @@ -20,8 +20,9 @@ from mailman import public from mailman.database.model import Model from mailman.database.transaction import dbconnection +from mailman.database.types import SAUnicode from mailman.interfaces.messages import IMessage -from sqlalchemy import Column, Integer, Unicode +from sqlalchemy import Column, Integer from zope.interface import implementer @@ -34,9 +35,9 @@ class Message(Model): id = Column(Integer, primary_key=True) # This is a Messge-ID field representation, not a database row id. - message_id = Column(Unicode) - message_id_hash = Column(Unicode) - path = Column(Unicode) + message_id = Column(SAUnicode) + message_id_hash = Column(SAUnicode) + path = Column(SAUnicode) @dbconnection def __init__(self, store, message_id, message_id_hash, path): diff --git a/src/mailman/model/mime.py b/src/mailman/model/mime.py index 849eac385..da5f11f05 100644 --- a/src/mailman/model/mime.py +++ b/src/mailman/model/mime.py @@ -19,9 +19,9 @@ from mailman import public from mailman.database.model import Model -from mailman.database.types import Enum +from mailman.database.types import Enum, SAUnicode from mailman.interfaces.mime import FilterType, IContentFilter -from sqlalchemy import Column, ForeignKey, Integer, Unicode +from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.orm import relationship from zope.interface import implementer @@ -39,7 +39,7 @@ class ContentFilter(Model): mailing_list = relationship('MailingList') filter_type = Column(Enum(FilterType)) - filter_pattern = Column(Unicode) + filter_pattern = Column(SAUnicode) def __init__(self, mailing_list, filter_pattern, filter_type): self.mailing_list = mailing_list diff --git a/src/mailman/model/pending.py b/src/mailman/model/pending.py index 0e19f0cbf..9d8315605 100644 --- a/src/mailman/model/pending.py +++ b/src/mailman/model/pending.py @@ -24,11 +24,12 @@ from mailman import public from mailman.config import config from mailman.database.model import Model from mailman.database.transaction import dbconnection +from mailman.database.types import SAUnicode from mailman.interfaces.pending import ( IPendable, IPended, IPendedKeyValue, IPendings) from mailman.utilities.datetime import now from mailman.utilities.uid import TokenFactory -from sqlalchemy import Column, DateTime, ForeignKey, Integer, Unicode, and_ +from sqlalchemy import Column, DateTime, ForeignKey, Integer, and_ from sqlalchemy.orm import aliased, relationship from zope.interface import implementer from zope.interface.verify import verifyObject @@ -45,8 +46,8 @@ class PendedKeyValue(Model): __tablename__ = 'pendedkeyvalue' id = Column(Integer, primary_key=True) - key = Column(Unicode, index=True) - value = Column(Unicode, index=True) + key = Column(SAUnicode, index=True) + value = Column(SAUnicode, index=True) pended_id = Column(Integer, ForeignKey('pended.id'), index=True) def __init__(self, key, value): @@ -62,9 +63,9 @@ class Pended(Model): __tablename__ = 'pended' id = Column(Integer, primary_key=True) - token = Column(Unicode, index=True) + token = Column(SAUnicode, index=True) expiration_date = Column(DateTime, index=True) - key_values = relationship('PendedKeyValue', cascade="all, delete-orphan") + key_values = relationship('PendedKeyValue', cascade='all, delete-orphan') @public diff --git a/src/mailman/model/preferences.py b/src/mailman/model/preferences.py index 71775117f..366ed97c2 100644 --- a/src/mailman/model/preferences.py +++ b/src/mailman/model/preferences.py @@ -20,11 +20,11 @@ from mailman import public from mailman.database.model import Model from mailman.database.transaction import dbconnection -from mailman.database.types import Enum +from mailman.database.types import Enum, SAUnicode from mailman.interfaces.languages import ILanguageManager from mailman.interfaces.member import DeliveryMode, DeliveryStatus from mailman.interfaces.preferences import IPreferences -from sqlalchemy import Boolean, Column, Integer, Unicode +from sqlalchemy import Boolean, Column, Integer from zope.component import getUtility from zope.interface import implementer @@ -39,7 +39,7 @@ class Preferences(Model): id = Column(Integer, primary_key=True) acknowledge_posts = Column(Boolean) hide_address = Column(Boolean) - _preferred_language = Column('preferred_language', Unicode) + _preferred_language = Column('preferred_language', SAUnicode) receive_list_copy = Column(Boolean) receive_own_postings = Column(Boolean) delivery_mode = Column(Enum(DeliveryMode)) diff --git a/src/mailman/model/requests.py b/src/mailman/model/requests.py index df5f1062e..0f0f96dfe 100644 --- a/src/mailman/model/requests.py +++ b/src/mailman/model/requests.py @@ -21,12 +21,12 @@ from datetime import timedelta from mailman import public from mailman.database.model import Model from mailman.database.transaction import dbconnection -from mailman.database.types import Enum +from mailman.database.types import Enum, SAUnicode from mailman.interfaces.pending import IPendable, IPendings from mailman.interfaces.requests import IListRequests, RequestType from mailman.utilities.queries import QuerySequence from pickle import dumps, loads -from sqlalchemy import Column, ForeignKey, Integer, Unicode +from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.orm import relationship from zope.component import getUtility from zope.interface import implementer @@ -42,7 +42,7 @@ class DataPendable(dict): def update(self, mapping): # Keys and values must be strings (unicodes, but bytes values are # accepted for now). Any other types for keys are a programming - # error. If we find a non-Unicode value, pickle it and encode it in + # error. If we find a non-SAUnicode value, pickle it and encode it in # such a way that it will be properly reconstituted when unpended. clean_mapping = {} for key, value in mapping.items(): @@ -120,7 +120,7 @@ class ListRequests: if pendable is None: return None data = dict() - # Unpickle any non-Unicode values. + # Unpickle any non-SAUnicode values. for key, value in pendable.items(): if key.startswith('_pck_'): data[key[5:]] = loads(value.encode('raw-unicode-escape')) @@ -146,9 +146,9 @@ class _Request(Model): __tablename__ = '_request' id = Column(Integer, primary_key=True) - key = Column(Unicode) + key = Column(SAUnicode) request_type = Column(Enum(RequestType)) - data_hash = Column(Unicode) + data_hash = Column(SAUnicode) mailing_list_id = Column(Integer, ForeignKey('mailinglist.id'), index=True) mailing_list = relationship('MailingList') diff --git a/src/mailman/model/template.py b/src/mailman/model/template.py index 96ce43e1f..b09c644bb 100644 --- a/src/mailman/model/template.py +++ b/src/mailman/model/template.py @@ -23,6 +23,7 @@ from mailman import public from mailman.config import config from mailman.database.model import Model from mailman.database.transaction import dbconnection +from mailman.database.types import SAUnicode from mailman.interfaces.cache import ICacheManager from mailman.interfaces.domain import IDomain from mailman.interfaces.mailinglist import IMailingList @@ -32,7 +33,7 @@ from mailman.utilities import protocols from mailman.utilities.i18n import find from mailman.utilities.string import expand from requests import HTTPError -from sqlalchemy import Column, Integer, Unicode +from sqlalchemy import Column, Integer from urllib.error import URLError from urllib.parse import urlparse from zope.component import getUtility @@ -47,11 +48,11 @@ class Template(Model): __tablename__ = 'template' id = Column(Integer, primary_key=True) - name = Column(Unicode) - context = Column(Unicode) - uri = Column(Unicode) - username = Column(Unicode, nullable=True) - password = Column(Unicode, nullable=True) + name = Column(SAUnicode) + context = Column(SAUnicode) + uri = Column(SAUnicode) + username = Column(SAUnicode, nullable=True) + password = Column(SAUnicode, nullable=True) def __init__(self, name, context, uri, username, password): self.name = name diff --git a/src/mailman/model/tests/test_listmanager.py b/src/mailman/model/tests/test_listmanager.py index a8ac68c92..ff1bd3026 100644 --- a/src/mailman/model/tests/test_listmanager.py +++ b/src/mailman/model/tests/test_listmanager.py @@ -174,7 +174,9 @@ class TestListCreation(unittest.TestCase): def test_create_list_case_folding(self): # LP: #1117176 describes a problem where list names created in upper - # case are not actually usable by the LMTP server. + # case are not actually usable by the LMTP server. MySQL + # automatically changes the case of the arguments so this test will + # always fail in case of MySQL. self._manager.create('my-LIST@example.com') self.assertIsNone(self._manager.get('my-LIST@example.com')) mlist = self._manager.get('my-list@example.com') diff --git a/src/mailman/model/user.py b/src/mailman/model/user.py index a534ef2a5..edda2a9e8 100644 --- a/src/mailman/model/user.py +++ b/src/mailman/model/user.py @@ -20,7 +20,7 @@ from mailman import public from mailman.database.model import Model from mailman.database.transaction import dbconnection -from mailman.database.types import UUID +from mailman.database.types import SAUnicode, UUID from mailman.interfaces.address import ( AddressAlreadyLinkedError, AddressNotLinkedError) from mailman.interfaces.user import ( @@ -32,7 +32,7 @@ from mailman.model.roster import Memberships from mailman.utilities.datetime import factory as date_factory from mailman.utilities.uid import UIDFactory from sqlalchemy import ( - Boolean, Column, DateTime, ForeignKey, Integer, Unicode) + Boolean, Column, DateTime, ForeignKey, Integer) from sqlalchemy.orm import backref, relationship from zope.event import notify from zope.interface import implementer @@ -49,8 +49,8 @@ class User(Model): __tablename__ = 'user' id = Column(Integer, primary_key=True) - display_name = Column(Unicode) - _password = Column('password', Unicode) + display_name = Column(SAUnicode) + _password = Column('password', SAUnicode) _user_id = Column(UUID, index=True) _created_on = Column(DateTime) is_server_owner = Column(Boolean, default=False) diff --git a/src/mailman/model/workflow.py b/src/mailman/model/workflow.py index 7a0056e5f..1072fa548 100644 --- a/src/mailman/model/workflow.py +++ b/src/mailman/model/workflow.py @@ -20,8 +20,9 @@ from mailman import public from mailman.database.model import Model from mailman.database.transaction import dbconnection +from mailman.database.types import SAUnicode from mailman.interfaces.workflow import IWorkflowState, IWorkflowStateManager -from sqlalchemy import Column, Unicode +from sqlalchemy import Column from zope.interface import implementer @@ -32,10 +33,10 @@ class WorkflowState(Model): __tablename__ = 'workflowstate' - name = Column(Unicode, primary_key=True) - token = Column(Unicode, primary_key=True) - step = Column(Unicode) - data = Column(Unicode) + name = Column(SAUnicode, primary_key=True) + token = Column(SAUnicode, primary_key=True) + step = Column(SAUnicode) + data = Column(SAUnicode) @public |
