summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBarry Warsaw2016-07-29 19:28:46 -0400
committerBarry Warsaw2016-07-29 19:28:46 -0400
commitfebb5289e82c4424cdbcc2297e967bd894cbc8cf (patch)
treebd71f4cbf6988049ac4d5dd65ceb7d5cc51902e7
parent90e84bee5f47cbcdb9e9c367c60a877e325ef3e7 (diff)
downloadmailman-febb5289e82c4424cdbcc2297e967bd894cbc8cf.tar.gz
mailman-febb5289e82c4424cdbcc2297e967bd894cbc8cf.tar.zst
mailman-febb5289e82c4424cdbcc2297e967bd894cbc8cf.zip
-rw-r--r--.gitlab-ci.yml8
-rw-r--r--src/mailman/database/alembic/versions/2bb9b382198_workflow_state_table.py9
-rw-r--r--src/mailman/database/alembic/versions/33bc0099223_add_member_indexes.py31
-rw-r--r--src/mailman/database/alembic/versions/33e1f5f6fa8_.py3
-rw-r--r--src/mailman/database/alembic/versions/42756496720_header_matches.py12
-rw-r--r--src/mailman/database/alembic/versions/46e92facee7_add_serverowner_domainowner.py5
-rw-r--r--src/mailman/database/alembic/versions/47294d3a604_pendable_indexes.py5
-rw-r--r--src/mailman/database/alembic/versions/70af5a4e5790_digests.py9
-rw-r--r--src/mailman/database/alembic/versions/7b254d88f122_members_and_list_moderation_action.py6
-rw-r--r--src/mailman/database/alembic/versions/d4fbb4fd34ca_header_match_order.py32
-rw-r--r--src/mailman/database/alembic/versions/fa0d96e28631_template_manager.py55
-rw-r--r--src/mailman/database/base.py2
-rw-r--r--src/mailman/database/helpers.py5
-rw-r--r--src/mailman/database/mysql.py34
-rw-r--r--src/mailman/database/tests/test_factory.py33
-rw-r--r--src/mailman/database/tests/test_migrations.py14
-rw-r--r--src/mailman/database/types.py46
-rw-r--r--src/mailman/docs/DATABASE.rst47
-rw-r--r--src/mailman/docs/NEWS.rst5
-rw-r--r--src/mailman/model/address.py9
-rw-r--r--src/mailman/model/bans.py7
-rw-r--r--src/mailman/model/bounce.py10
-rw-r--r--src/mailman/model/cache.py7
-rw-r--r--src/mailman/model/domain.py7
-rw-r--r--src/mailman/model/language.py5
-rw-r--r--src/mailman/model/mailinglist.py54
-rw-r--r--src/mailman/model/member.py6
-rw-r--r--src/mailman/model/message.py9
-rw-r--r--src/mailman/model/mime.py6
-rw-r--r--src/mailman/model/pending.py11
-rw-r--r--src/mailman/model/preferences.py6
-rw-r--r--src/mailman/model/requests.py12
-rw-r--r--src/mailman/model/template.py13
-rw-r--r--src/mailman/model/tests/test_listmanager.py4
-rw-r--r--src/mailman/model/user.py8
-rw-r--r--src/mailman/model/workflow.py11
-rw-r--r--tox.ini3
37 files changed, 367 insertions, 182 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 10eeb7725..5b5c28226 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -30,3 +30,11 @@ pgsql:
- MAILMAN_EXTRA_TESTING_CFG=/home/runner/configs/postgres.cfg tox -e py34-pg,py35-pg
tags:
- postgres
+
+mysql:
+ services:
+ - maxking/mysql:latest
+ script:
+ - MAILMAN_EXTRA_TESTING_CFG=/home/runner/configs/mysql.cfg tox -e py34-mysql,py35-mysql
+ tags:
+ - mysql
diff --git a/src/mailman/database/alembic/versions/2bb9b382198_workflow_state_table.py b/src/mailman/database/alembic/versions/2bb9b382198_workflow_state_table.py
index 64def0253..df7a10e00 100644
--- a/src/mailman/database/alembic/versions/2bb9b382198_workflow_state_table.py
+++ b/src/mailman/database/alembic/versions/2bb9b382198_workflow_state_table.py
@@ -9,6 +9,7 @@ Create Date: 2015-03-25 18:09:18.338790
import sqlalchemy as sa
from alembic import op
+from mailman.database.types import SAUnicode
# Revision identifiers, used by Alembic.
@@ -19,10 +20,10 @@ down_revision = '16c2b25c7b'
def upgrade():
op.create_table(
'workflowstate',
- sa.Column('name', sa.Unicode(), nullable=False),
- sa.Column('token', sa.Unicode(), nullable=False),
- sa.Column('step', sa.Unicode(), nullable=True),
- sa.Column('data', sa.Unicode(), nullable=True),
+ sa.Column('name', SAUnicode(), nullable=False),
+ sa.Column('token', SAUnicode(), nullable=False),
+ sa.Column('step', SAUnicode(), nullable=True),
+ sa.Column('data', SAUnicode(), nullable=True),
sa.PrimaryKeyConstraint('name', 'token')
)
diff --git a/src/mailman/database/alembic/versions/33bc0099223_add_member_indexes.py b/src/mailman/database/alembic/versions/33bc0099223_add_member_indexes.py
index ce8f2f3f1..5f01f4a66 100644
--- a/src/mailman/database/alembic/versions/33bc0099223_add_member_indexes.py
+++ b/src/mailman/database/alembic/versions/33bc0099223_add_member_indexes.py
@@ -7,6 +7,7 @@ Create Date: 2015-11-19 23:04:42.449553
"""
from alembic import op
+from mailman.database.helpers import is_mysql
# Revision identifiers, used by Alembic.
@@ -15,22 +16,28 @@ down_revision = '42756496720'
def upgrade():
- op.create_index(op.f('ix_member_address_id'),
- 'member', ['address_id'],
- unique=False)
- op.create_index(op.f('ix_member_preferences_id'),
- 'member', ['preferences_id'],
- unique=False)
- op.create_index(op.f('ix_member_user_id'),
- 'member', ['user_id'],
- unique=False)
op.create_index(op.f('ix_address_email'),
'address', ['email'],
unique=False)
+ # MySQL automatically creates the indexes for primary keys so don't need
+ # to do it explicitly again.
+ if not is_mysql(op.get_bind()):
+ op.create_index(op.f('ix_member_address_id'),
+ 'member', ['address_id'],
+ unique=False)
+ op.create_index(op.f('ix_member_preferences_id'),
+ 'member', ['preferences_id'],
+ unique=False)
+ op.create_index(op.f('ix_member_user_id'),
+ 'member', ['user_id'],
+ unique=False)
def downgrade():
op.drop_index(op.f('ix_address_email'), table_name='address')
- op.drop_index(op.f('ix_member_user_id'), table_name='member')
- op.drop_index(op.f('ix_member_preferences_id'), table_name='member')
- op.drop_index(op.f('ix_member_address_id'), table_name='member')
+ # MySQL automatically creates and removes the indexes for primary keys.
+ # So, you cannot drop it without removing the foreign key constraint.
+ if not is_mysql(op.get_bind()):
+ op.drop_index(op.f('ix_member_user_id'), table_name='member')
+ op.drop_index(op.f('ix_member_preferences_id'), table_name='member')
+ op.drop_index(op.f('ix_member_address_id'), table_name='member')
diff --git a/src/mailman/database/alembic/versions/33e1f5f6fa8_.py b/src/mailman/database/alembic/versions/33e1f5f6fa8_.py
index a8b572518..72754d3c5 100644
--- a/src/mailman/database/alembic/versions/33e1f5f6fa8_.py
+++ b/src/mailman/database/alembic/versions/33e1f5f6fa8_.py
@@ -29,6 +29,7 @@ import sqlalchemy as sa
from alembic import op
from mailman.database.helpers import is_sqlite
+from mailman.database.types import SAUnicode
# Revision identifiers, used by Alembic.
@@ -50,7 +51,7 @@ def upgrade():
# SQLite does not support altering columns.
return
for table, column in COLUMNS_TO_CHANGE:
- op.alter_column(table, column, type_=sa.Unicode)
+ op.alter_column(table, column, type_=SAUnicode)
def downgrade():
diff --git a/src/mailman/database/alembic/versions/42756496720_header_matches.py b/src/mailman/database/alembic/versions/42756496720_header_matches.py
index 4514fb398..d1308a134 100644
--- a/src/mailman/database/alembic/versions/42756496720_header_matches.py
+++ b/src/mailman/database/alembic/versions/42756496720_header_matches.py
@@ -10,7 +10,7 @@ import sqlalchemy as sa
from alembic import op
from mailman.database.helpers import exists_in_db, is_sqlite
-
+from mailman.database.types import SAUnicode
# Revision identifiers, used by Alembic.
revision = '42756496720'
@@ -23,9 +23,9 @@ def upgrade():
'headermatch',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('mailing_list_id', sa.Integer(), nullable=True),
- sa.Column('header', sa.Unicode(), nullable=False),
- sa.Column('pattern', sa.Unicode(), nullable=False),
- sa.Column('chain', sa.Unicode(), nullable=True),
+ sa.Column('header', SAUnicode(), nullable=False),
+ sa.Column('pattern', SAUnicode(), nullable=False),
+ sa.Column('chain', SAUnicode(), nullable=True),
sa.ForeignKeyConstraint(['mailing_list_id'], ['mailinglist.id'], ),
sa.PrimaryKeyConstraint('id')
)
@@ -73,8 +73,8 @@ def downgrade():
header_match_table = sa.sql.table(
'headermatch',
sa.sql.column('mailing_list_id', sa.Integer),
- sa.sql.column('header', sa.Unicode),
- sa.sql.column('pattern', sa.Unicode),
+ sa.sql.column('header', SAUnicode),
+ sa.sql.column('pattern', SAUnicode),
)
for mlist_id, header, pattern in connection.execute(
header_match_table.select()).fetchall():
diff --git a/src/mailman/database/alembic/versions/46e92facee7_add_serverowner_domainowner.py b/src/mailman/database/alembic/versions/46e92facee7_add_serverowner_domainowner.py
index 1c689670a..bf734934b 100644
--- a/src/mailman/database/alembic/versions/46e92facee7_add_serverowner_domainowner.py
+++ b/src/mailman/database/alembic/versions/46e92facee7_add_serverowner_domainowner.py
@@ -56,8 +56,9 @@ def downgrade():
if not is_sqlite(op.get_bind()):
op.drop_column('user', 'is_server_owner')
if not exists_in_db(op.get_bind(), 'domain', 'contact_address'):
- # SQLite may not have removed it.
+ # SQLite may not have removed it. Add a fixed length VARCHAR for
+ # MySQL.
op.add_column(
'domain',
- sa.Column('contact_address', sa.VARCHAR(), nullable=True))
+ sa.Column('contact_address', sa.VARCHAR(255), nullable=True))
op.drop_table('domain_owner')
diff --git a/src/mailman/database/alembic/versions/47294d3a604_pendable_indexes.py b/src/mailman/database/alembic/versions/47294d3a604_pendable_indexes.py
index 5e465af24..4aafb4613 100644
--- a/src/mailman/database/alembic/versions/47294d3a604_pendable_indexes.py
+++ b/src/mailman/database/alembic/versions/47294d3a604_pendable_indexes.py
@@ -13,6 +13,7 @@ import json
import sqlalchemy as sa
from alembic import op
+from mailman.database.types import SAUnicode
# revision identifiers, used by Alembic.
@@ -34,8 +35,8 @@ pended_table = sa.sql.table(
keyvalue_table = sa.sql.table(
'pendedkeyvalue',
sa.sql.column('id', sa.Integer),
- sa.sql.column('key', sa.Unicode),
- sa.sql.column('value', sa.Unicode),
+ sa.sql.column('key', SAUnicode),
+ sa.sql.column('value', SAUnicode),
sa.sql.column('pended_id', sa.Integer),
)
diff --git a/src/mailman/database/alembic/versions/70af5a4e5790_digests.py b/src/mailman/database/alembic/versions/70af5a4e5790_digests.py
index 50c87ccf3..1f9a93bcd 100644
--- a/src/mailman/database/alembic/versions/70af5a4e5790_digests.py
+++ b/src/mailman/database/alembic/versions/70af5a4e5790_digests.py
@@ -20,7 +20,10 @@ down_revision = '47294d3a604'
def upgrade():
with op.batch_alter_table('mailinglist') as batch_op:
- batch_op.alter_column('digestable', new_column_name='digests_enabled')
+ batch_op.alter_column('digestable',
+ new_column_name='digests_enabled',
+ existing_type=sa.Boolean)
+ # All column modifications require existing types for Mysql.
batch_op.drop_column('nondigestable')
# Non-database migration: rename the list's data-path.
for dirname in os.listdir(config.LIST_DATA_DIR):
@@ -34,7 +37,9 @@ def upgrade():
def downgrade():
with op.batch_alter_table('mailinglist') as batch_op:
- batch_op.alter_column('digests_enabled', new_column_name='digestable')
+ batch_op.alter_column('digests_enabled',
+ new_column_name='digestable',
+ existing_type=sa.Boolean)
# The data for this column is lost, it's not used anyway.
batch_op.add_column(sa.Column('nondigestable', sa.Boolean))
for dirname in os.listdir(config.LIST_DATA_DIR):
diff --git a/src/mailman/database/alembic/versions/7b254d88f122_members_and_list_moderation_action.py b/src/mailman/database/alembic/versions/7b254d88f122_members_and_list_moderation_action.py
index 6b4d7bfd1..9f19e3278 100644
--- a/src/mailman/database/alembic/versions/7b254d88f122_members_and_list_moderation_action.py
+++ b/src/mailman/database/alembic/versions/7b254d88f122_members_and_list_moderation_action.py
@@ -13,7 +13,7 @@ fallback to the list's default.
import sqlalchemy as sa
from alembic import op
-from mailman.database.types import Enum
+from mailman.database.types import Enum, SAUnicode
from mailman.interfaces.action import Action
from mailman.interfaces.member import MemberRole
@@ -26,7 +26,7 @@ down_revision = 'd4fbb4fd34ca'
mailinglist_table = sa.sql.table(
'mailinglist',
sa.sql.column('id', sa.Integer),
- sa.sql.column('list_id', sa.Unicode),
+ sa.sql.column('list_id', SAUnicode),
sa.sql.column('default_member_action', Enum(Action)),
sa.sql.column('default_nonmember_action', Enum(Action)),
)
@@ -35,7 +35,7 @@ mailinglist_table = sa.sql.table(
member_table = sa.sql.table(
'member',
sa.sql.column('id', sa.Integer),
- sa.sql.column('list_id', sa.Unicode),
+ sa.sql.column('list_id', SAUnicode),
sa.sql.column('role', Enum(MemberRole)),
sa.sql.column('moderation_action', Enum(Action)),
)
diff --git a/src/mailman/database/alembic/versions/d4fbb4fd34ca_header_match_order.py b/src/mailman/database/alembic/versions/d4fbb4fd34ca_header_match_order.py
index 2608bb812..313963b86 100644
--- a/src/mailman/database/alembic/versions/d4fbb4fd34ca_header_match_order.py
+++ b/src/mailman/database/alembic/versions/d4fbb4fd34ca_header_match_order.py
@@ -9,6 +9,7 @@ Create Date: 2016-02-01 15:57:09.807678
import sqlalchemy as sa
from alembic import op
+from mailman.database.helpers import is_mysql
# Revision identifiers, used by Alembic.
@@ -20,19 +21,34 @@ def upgrade():
with op.batch_alter_table('headermatch') as batch_op:
batch_op.add_column(
sa.Column('position', sa.Integer(), nullable=True))
- batch_op.alter_column(
- 'mailing_list_id', existing_type=sa.INTEGER(), nullable=False)
batch_op.create_index(
op.f('ix_headermatch_position'), ['position'], unique=False)
- batch_op.create_index(
- op.f('ix_headermatch_mailing_list_id'), ['mailing_list_id'],
- unique=False)
+ if not is_mysql(op.get_bind()):
+ # MySQL automatically creates indexes for primary keys.
+ batch_op.create_index(
+ op.f('ix_headermatch_mailing_list_id'), ['mailing_list_id'],
+ unique=False)
+ # MySQL doesn't allow changing columns used in a foreign key
+ # constrains since MySQL version 5.6. We need to drop the
+ # constraint before changing the column. But, since the
+ # constraint name is auto-generated, we can't really hardcode the
+ # name here to use batch_op.drop_constraint(). Until we have a
+ # better fix for this, it should be safe to skip this.
+ batch_op.alter_column(
+ 'mailing_list_id', existing_type=sa.INTEGER(), nullable=False)
def downgrade():
with op.batch_alter_table('headermatch') as batch_op:
- batch_op.drop_index(op.f('ix_headermatch_mailing_list_id'))
batch_op.drop_index(op.f('ix_headermatch_position'))
- batch_op.alter_column(
- 'mailing_list_id', existing_type=sa.INTEGER(), nullable=True)
batch_op.drop_column('position')
+
+ if not is_mysql(op.get_bind()):
+ # MySQL automatically creates and removes the indexes for primary
+ # keys. So, you cannot drop it without removing the foreign key
+ # constraint.
+ batch_op.drop_index(op.f('ix_headermatch_mailing_list_id'))
+ # MySQL doesn't allow changing columns used in foreign_key
+ # constraints.
+ batch_op.alter_column(
+ 'mailing_list_id', existing_type=sa.INTEGER(), nullable=True)
diff --git a/src/mailman/database/alembic/versions/fa0d96e28631_template_manager.py b/src/mailman/database/alembic/versions/fa0d96e28631_template_manager.py
index 5920dd4e4..58d688453 100644
--- a/src/mailman/database/alembic/versions/fa0d96e28631_template_manager.py
+++ b/src/mailman/database/alembic/versions/fa0d96e28631_template_manager.py
@@ -12,6 +12,7 @@ import sqlalchemy as sa
from alembic import op
from mailman.config import config
from mailman.database.helpers import exists_in_db
+from mailman.database.types import SAUnicode
# revision identifiers, used by Alembic.
@@ -35,8 +36,8 @@ def upgrade():
op.create_table(
'file_cache',
sa.Column('id', sa.Integer(), nullable=False),
- sa.Column('key', sa.Unicode(), nullable=False),
- sa.Column('file_id', sa.Unicode(), nullable=True),
+ sa.Column('key', SAUnicode(), nullable=False),
+ sa.Column('file_id', SAUnicode(), nullable=True),
sa.Column('is_bytes', sa.Boolean(), nullable=False),
sa.Column('created_on', sa.DateTime(), nullable=False),
sa.Column('expires_on', sa.DateTime(), nullable=False),
@@ -45,10 +46,10 @@ def upgrade():
template_table = op.create_table(
'template',
sa.Column('id', sa.Integer(), nullable=False),
- sa.Column('name', sa.Unicode(), nullable=False),
- sa.Column('context', sa.Unicode(), nullable=True),
- sa.Column('uri', sa.Unicode(), nullable=False),
- sa.Column('username', sa.Unicode(), nullable=True),
+ sa.Column('name', SAUnicode(), nullable=False),
+ sa.Column('context', SAUnicode(), nullable=True),
+ sa.Column('uri', SAUnicode(), nullable=False),
+ sa.Column('username', SAUnicode(), nullable=True),
sa.Column('password', sa.DateTime(), nullable=True),
sa.PrimaryKeyConstraint('id')
)
@@ -60,13 +61,13 @@ def upgrade():
mlist_table = sa.sql.table(
'mailinglist',
sa.sql.column('id', sa.Integer),
- sa.sql.column('list_id', sa.Unicode),
- sa.sql.column('digest_footer_uri', sa.Unicode),
- sa.sql.column('digest_header_uri', sa.Unicode),
- sa.sql.column('footer_uri', sa.Unicode),
- sa.sql.column('header_uri', sa.Unicode),
- sa.sql.column('goodbye_message_uri', sa.Unicode),
- sa.sql.column('welcome_message_uri', sa.Unicode),
+ sa.sql.column('list_id', SAUnicode),
+ sa.sql.column('digest_footer_uri', SAUnicode),
+ sa.sql.column('digest_header_uri', SAUnicode),
+ sa.sql.column('footer_uri', SAUnicode),
+ sa.sql.column('header_uri', SAUnicode),
+ sa.sql.column('goodbye_message_uri', SAUnicode),
+ sa.sql.column('welcome_message_uri', SAUnicode),
)
for (mlist_id, list_id,
digest_footer_uri, digest_header_uri,
@@ -131,30 +132,30 @@ def downgrade():
if not exists_in_db(op.get_bind(), 'mailinglist', column):
op.add_column(
'mailinglist',
- sa.Column(column, sa.Unicode, nullable=True))
- op.add_column('domain', sa.Column('base_url', sa.Unicode))
+ sa.Column(column, SAUnicode, nullable=True))
+ op.add_column('domain', sa.Column('base_url', SAUnicode))
# Put all the templates with a context mapping the list-id back into the
# mailinglist table. No other contexts are supported, so just throw those
# away.
template_table = sa.sql.table(
'template',
sa.sql.column('id', sa.Integer),
- sa.sql.column('name', sa.Unicode),
- sa.sql.column('context', sa.Unicode),
- sa.sql.column('uri', sa.Unicode),
- sa.sql.column('username', sa.Unicode),
- sa.sql.column('password', sa.Unicode),
+ sa.sql.column('name', SAUnicode),
+ sa.sql.column('context', SAUnicode),
+ sa.sql.column('uri', SAUnicode),
+ sa.sql.column('username', SAUnicode),
+ sa.sql.column('password', SAUnicode),
)
mlist_table = sa.sql.table(
'mailinglist',
sa.sql.column('id', sa.Integer),
- sa.sql.column('list_id', sa.Unicode),
- sa.sql.column('digest_footer_uri', sa.Unicode),
- sa.sql.column('digest_header_uri', sa.Unicode),
- sa.sql.column('footer_uri', sa.Unicode),
- sa.sql.column('header_uri', sa.Unicode),
- sa.sql.column('goodbye_message_uri', sa.Unicode),
- sa.sql.column('welcome_message_uri', sa.Unicode),
+ sa.sql.column('list_id', SAUnicode),
+ sa.sql.column('digest_footer_uri', SAUnicode),
+ sa.sql.column('digest_header_uri', SAUnicode),
+ sa.sql.column('footer_uri', SAUnicode),
+ sa.sql.column('header_uri', SAUnicode),
+ sa.sql.column('goodbye_message_uri', SAUnicode),
+ sa.sql.column('welcome_message_uri', SAUnicode),
)
connection = op.get_bind()
for (table_id, name, context, uri, username, password
diff --git a/src/mailman/database/base.py b/src/mailman/database/base.py
index dda3665af..f57228029 100644
--- a/src/mailman/database/base.py
+++ b/src/mailman/database/base.py
@@ -103,7 +103,7 @@ class SABaseDatabase:
# engines, and yes, we could have chmod'd the file after the fact, but
# half dozen and all...
self.url = url
- self.engine = create_engine(url)
+ self.engine = create_engine(url, isolation_level='READ UNCOMMITTED')
session = sessionmaker(bind=self.engine)
self.store = session()
self.store.commit()
diff --git a/src/mailman/database/helpers.py b/src/mailman/database/helpers.py
index f58d559c5..b1dc381df 100644
--- a/src/mailman/database/helpers.py
+++ b/src/mailman/database/helpers.py
@@ -28,6 +28,11 @@ def is_sqlite(bind):
@public
+def is_mysql(bind):
+ return bind.dialect.name == 'mysql'
+
+
+@public
def exists_in_db(bind, tablename, columnname=None):
md = sa.MetaData()
md.reflect(bind=bind)
diff --git a/src/mailman/database/mysql.py b/src/mailman/database/mysql.py
new file mode 100644
index 000000000..0c0de54a9
--- /dev/null
+++ b/src/mailman/database/mysql.py
@@ -0,0 +1,34 @@
+# Copyright (C) 2016 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
+
+"""MySQL database support"""
+
+from mailman import public
+from mailman.database.base import SABaseDatabase
+from mailman.database.model import Model
+
+
+@public
+class MySQLDatabase(SABaseDatabase):
+ """Database class for MySQL."""
+
+ def _post_reset(self, store):
+ """Reset AUTO_INCREMENT counters for all the tables."""
+ super()._post_reset(store)
+ tables = reversed(Model.metadata.sorted_tables)
+ for table in tables:
+ store.execute('ALTER TABLE {} AUTO_INCREMENT = 1;'.format(table))
diff --git a/src/mailman/database/tests/test_factory.py b/src/mailman/database/tests/test_factory.py
index 74b45e43a..b0aa89fc4 100644
--- a/src/mailman/database/tests/test_factory.py
+++ b/src/mailman/database/tests/test_factory.py
@@ -24,10 +24,12 @@ from contextlib import suppress
from mailman.config import config
from mailman.database.alembic import alembic_cfg
from mailman.database.factory import LAST_STORM_SCHEMA_VERSION, SchemaManager
+from mailman.database.helpers import is_mysql
from mailman.database.model import Model
+from mailman.database.types import SAUnicode
from mailman.interfaces.database import DatabaseError
from mailman.testing.layers import ConfigLayer
-from sqlalchemy import Column, Integer, MetaData, Table, Unicode
+from sqlalchemy import Column, Integer, MetaData, Table
from sqlalchemy.exc import OperationalError, ProgrammingError
from sqlalchemy.schema import Index
from unittest.mock import patch
@@ -60,8 +62,8 @@ class TestSchemaManager(unittest.TestCase):
version_table = Table(
'version', Model.metadata,
Column('id', Integer, primary_key=True),
- Column('component', Unicode),
- Column('version', Unicode),
+ Column('component', SAUnicode),
+ Column('version', SAUnicode),
)
version_table.create(config.db.engine)
config.db.store.execute(version_table.insert().values(
@@ -71,12 +73,15 @@ class TestSchemaManager(unittest.TestCase):
# all DB engines...
config.db.engine.execute(
'ALTER TABLE mailinglist ADD COLUMN acceptable_aliases_id INT')
- Index('ix_user__user_id').drop(bind=config.db.engine)
- # Don't pollute our main metadata object, create a new one.
- md = MetaData()
- user_table = Model.metadata.tables['user'].tometadata(md)
- Index('ix_user_user_id', user_table.c._user_id).create(
- bind=config.db.engine)
+ # In case of MySQL, you cannot create/drop indexes on primary keys
+ # manually as it is handled automatically by MySQL.
+ if not is_mysql(config.db.engine):
+ Index('ix_user__user_id').drop(bind=config.db.engine)
+ # Don't pollute our main metadata object, create a new one.
+ md = MetaData()
+ user_table = Model.metadata.tables['user'].tometadata(md)
+ Index('ix_user_user_id', user_table.c._user_id).create(
+ bind=config.db.engine)
config.db.commit()
def _drop_storm_database(self):
@@ -88,10 +93,12 @@ class TestSchemaManager(unittest.TestCase):
version = Model.metadata.tables['version']
version.drop(config.db.engine, checkfirst=True)
Model.metadata.remove(version)
- # If it's nonexistent, PostgreSQL raises a ProgrammingError, while
- # SQLite raises an OperationalError.
- with suppress(ProgrammingError, OperationalError):
- Index('ix_user_user_id').drop(bind=config.db.engine)
+ # If it's nonexistent, PostgreSQL raises a ProgrammingError while
+ # SQLite raises an OperationalError. Since MySQL automatically handles
+ # indexes for primary keys, don't try doing it with that backend.
+ if not is_mysql(config.db.engine):
+ with suppress(ProgrammingError, OperationalError):
+ Index('ix_user_user_id').drop(bind=config.db.engine)
config.db.commit()
def test_current_database(self):
diff --git a/src/mailman/database/tests/test_migrations.py b/src/mailman/database/tests/test_migrations.py
index 88a2a30ab..2ad41ae53 100644
--- a/src/mailman/database/tests/test_migrations.py
+++ b/src/mailman/database/tests/test_migrations.py
@@ -28,7 +28,7 @@ from mailman.database.alembic import alembic_cfg
from mailman.database.helpers import exists_in_db
from mailman.database.model import Model
from mailman.database.transaction import transaction
-from mailman.database.types import Enum
+from mailman.database.types import Enum, SAUnicode
from mailman.interfaces.action import Action
from mailman.interfaces.cache import ICacheManager
from mailman.interfaces.member import MemberRole
@@ -83,8 +83,8 @@ class TestMigrations(unittest.TestCase):
header_match_table = sa.sql.table(
'headermatch',
sa.sql.column('mailing_list_id', sa.Integer),
- sa.sql.column('header', sa.Unicode),
- sa.sql.column('pattern', sa.Unicode),
+ sa.sql.column('header', SAUnicode),
+ sa.sql.column('pattern', SAUnicode),
)
# Bring the DB to the revision that is being tested.
alembic.command.downgrade(alembic_cfg, '42756496720')
@@ -126,8 +126,8 @@ class TestMigrations(unittest.TestCase):
keyvalue_table = sa.sql.table(
'pendedkeyvalue',
sa.sql.column('id', sa.Integer),
- sa.sql.column('key', sa.Unicode),
- sa.sql.column('value', sa.Unicode),
+ sa.sql.column('key', SAUnicode),
+ sa.sql.column('value', SAUnicode),
sa.sql.column('pended_id', sa.Integer),
)
def get_from_db(): # noqa: E301
@@ -230,14 +230,14 @@ class TestMigrations(unittest.TestCase):
sa.sql.table(
'mailinglist',
sa.sql.column('id', sa.Integer),
- sa.sql.column('list_id', sa.Unicode),
+ sa.sql.column('list_id', SAUnicode),
sa.sql.column('default_member_action', Enum(Action)),
sa.sql.column('default_nonmember_action', Enum(Action)),
)
member_table = sa.sql.table(
'member',
sa.sql.column('id', sa.Integer),
- sa.sql.column('list_id', sa.Unicode),
+ sa.sql.column('list_id', SAUnicode),
sa.sql.column('address_id', sa.Integer),
sa.sql.column('role', Enum(MemberRole)),
sa.sql.column('moderation_action', Enum(Action)),
diff --git a/src/mailman/database/types.py b/src/mailman/database/types.py
index 28b261514..bbd040d96 100644
--- a/src/mailman/database/types.py
+++ b/src/mailman/database/types.py
@@ -22,7 +22,8 @@ import uuid
from mailman import public
from sqlalchemy import Integer
from sqlalchemy.dialects import postgresql
-from sqlalchemy.types import CHAR, TypeDecorator
+from sqlalchemy.ext.compiler import compiles
+from sqlalchemy.types import CHAR, TypeDecorator, Unicode
@public
@@ -80,3 +81,46 @@ class UUID(TypeDecorator):
return value
else:
return uuid.UUID(value)
+
+
+@public
+class SAUnicode(TypeDecorator):
+ """Unicode datatype to support fixed length VARCHAR in MySQL.
+
+ This type compiles to VARCHAR(255) in case of MySQL, and in case of
+ other dailects defaults to the Unicode type. This was created so
+ that we don't have to alter the output of the default Unicode data
+ type and it can still be used if needed in the codebase.
+ """
+ impl = Unicode
+
+
+@compiles(SAUnicode)
+def default_sa_unicode(element, compiler, **kw):
+ return compiler.visit_Unicode(element, **kw)
+
+
+@compiles(SAUnicode, 'mysql')
+def compile_sa_unicode(element, compiler, **kw):
+ # We hardcode the collate here to make string comparison case sensitive.
+ return 'VARCHAR(255) COLLATE utf8_bin'
+
+
+@public
+class SAUnicodeLarge(TypeDecorator):
+ """Similar to SAUnicode type, but compiles to VARCHAR(510).
+
+ This is double size of SAUnicode defined above.
+ """
+ impl = Unicode
+
+
+@compiles(SAUnicodeLarge, 'mysql')
+def compile_sa_unicode_large(element, compiler, **kw):
+ # We hardcode the collate here to make string comparison case sensitive.
+ return 'VARCHAR(510) COLLATE utf8_bin'
+
+
+@compiles(SAUnicode)
+def defalt_sa_unicode_large(element, compiler, **kw):
+ return compiler.visit_unicode(element, **kw)
diff --git a/src/mailman/docs/DATABASE.rst b/src/mailman/docs/DATABASE.rst
index b27b1f788..cc7d09eb9 100644
--- a/src/mailman/docs/DATABASE.rst
+++ b/src/mailman/docs/DATABASE.rst
@@ -7,11 +7,11 @@ relational database. By default, Mailman uses Python's built-in SQLite3_
database, however, SQLAlchemy is compatible with PostgreSQL_ and MySQL, among
possibly others.
-Currently, Mailman is known to work with either the default SQLite3 database,
-or PostgreSQL. (Volunteers to port it to other databases are welcome!). If
-you want to use SQLite3, you generally don't need to change anything, but if
-you want Mailman to use PostgreSQL, you'll need to set that up first, and then
-change a configuration variable in your ``/etc/mailman.cfg`` file.
+Currently, Mailman is known to work with the SQLite3, PostgreSQL, and MySQL
+databases. (Volunteers to port it to other databases are welcome!). If you
+want to use SQLite3, you generally don't need to change anything, but if you
+want Mailman to use PostgreSQL or MySQL, you'll need to set those up first,
+and then change a configuration variable in your ``/etc/mailman.cfg`` file.
Two configuration variables control which database Mailman uses. The first
names the class implementing the database interface. The second names the URL
@@ -63,6 +63,42 @@ it::
My thanks to Stephen A. Goss for his contribution of PostgreSQL support.
+MySQL
+=====
+
+First, you need to configure MySQL itself. Lets say you create the `mailman`
+database in MySQL via::
+
+ mysql> CREATE DATABASE mailman;
+
+In some cases, our test suite requires the default collation of the database
+to be set to `utf8_unicode_ci`. You can change it via the MySQL command line
+like this::
+
+ mysql> ALTER DATABASE mailman DEFAULT COLLATE utf8_unicode_ci;
+
+You would also need the Python driver `pymysql` for MySQL.::
+
+ $ pip install pymysql
+
+You would then need to set both the `class` and `url` variables in
+`mailman.cfg` like so::
+
+ [database]
+ class: mailman.database.mysql.MySQLDatabase
+ url: mysql+pymysql://myuser:mypassword@mymysqlhost/mailman?charset=utf8&use_unicode=1
+
+The last part of the url specifies the charset that client expects from the
+server and to use Unicode via the flag `use_unicode`. You can find more about
+these options on the `SQLAlchemy's MySQL page`_.
+
+If you have any problems, you may need to delete the database and re-create
+it::
+
+ mysql> DROP DATABASE mailman;
+ mysql> CREATE DATABASE mailman;
+
+
Database Migrations
===================
@@ -101,3 +137,4 @@ integer in the database. A more complex migration would be needed for
.. _MySQL: http://dev.mysql.com/
.. _`Ubuntu article`: https://help.ubuntu.com/community/PostgreSQL
.. _`Alembic`: https://alembic.readthedocs.org/en/latest/
+.. _`SQLAlchemy's MySQL page`: http://docs.sqlalchemy.org/en/latest/dialects/mysql.html#unicode
diff --git a/src/mailman/docs/NEWS.rst b/src/mailman/docs/NEWS.rst
index 93bac61ad..f2c63a8f3 100644
--- a/src/mailman/docs/NEWS.rst
+++ b/src/mailman/docs/NEWS.rst
@@ -111,6 +111,11 @@ Command line
``[shell]history_file`` variable in mailman.cfg. Also, many useful names
are pre-populated in the namespace of the shell. (Closes: #228)
+Database
+--------
+
+ * MySQL is now an officially supported database. Given by Abhilash Raj.
+
Interfaces
----------
* Implement reasons for why a message is being held for moderator approval.
diff --git a/src/mailman/model/address.py b/src/mailman/model/address.py
index 7a3485378..a31e9a429 100644
--- a/src/mailman/model/address.py
+++ b/src/mailman/model/address.py
@@ -20,10 +20,11 @@
from email.utils import formataddr
from mailman import public
from mailman.database.model import Model
+from mailman.database.types import SAUnicode
from mailman.interfaces.address import (
AddressVerificationEvent, IAddress, IEmailValidator)
from mailman.utilities.datetime import now
-from sqlalchemy import Column, DateTime, ForeignKey, Integer, Unicode
+from sqlalchemy import Column, DateTime, ForeignKey, Integer
from sqlalchemy.orm import backref, relationship
from zope.component import getUtility
from zope.event import notify
@@ -38,9 +39,9 @@ class Address(Model):
__tablename__ = 'address'
id = Column(Integer, primary_key=True)
- email = Column(Unicode, index=True)
- _original = Column(Unicode)
- display_name = Column(Unicode)
+ email = Column(SAUnicode, index=True)
+ _original = Column(SAUnicode)
+ display_name = Column(SAUnicode)
_verified_on = Column('verified_on', DateTime)
registered_on = Column(DateTime)
diff --git a/src/mailman/model/bans.py b/src/mailman/model/bans.py
index e690145a4..e2b6490eb 100644
--- a/src/mailman/model/bans.py
+++ b/src/mailman/model/bans.py
@@ -22,8 +22,9 @@ import re
from mailman import public
from mailman.database.model import Model
from mailman.database.transaction import dbconnection
+from mailman.database.types import SAUnicode
from mailman.interfaces.bans import IBan, IBanManager
-from sqlalchemy import Column, Integer, Unicode
+from sqlalchemy import Column, Integer
from zope.interface import implementer
@@ -35,8 +36,8 @@ class Ban(Model):
__tablename__ = 'ban'
id = Column(Integer, primary_key=True)
- email = Column(Unicode, index=True)
- list_id = Column(Unicode, index=True)
+ email = Column(SAUnicode, index=True)
+ list_id = Column(SAUnicode, index=True)
def __init__(self, email, list_id):
super().__init__()
diff --git a/src/mailman/model/bounce.py b/src/mailman/model/bounce.py
index b7f217048..856d91f20 100644
--- a/src/mailman/model/bounce.py
+++ b/src/mailman/model/bounce.py
@@ -20,11 +20,11 @@
from mailman import public
from mailman.database.model import Model
from mailman.database.transaction import dbconnection
-from mailman.database.types import Enum
+from mailman.database.types import Enum, SAUnicode
from mailman.interfaces.bounce import (
BounceContext, IBounceEvent, IBounceProcessor)
from mailman.utilities.datetime import now
-from sqlalchemy import Boolean, Column, DateTime, Integer, Unicode
+from sqlalchemy import Boolean, Column, DateTime, Integer
from zope.interface import implementer
@@ -36,10 +36,10 @@ class BounceEvent(Model):
__tablename__ = 'bounceevent'
id = Column(Integer, primary_key=True)
- list_id = Column(Unicode)
- email = Column(Unicode)
+ list_id = Column(SAUnicode)
+ email = Column(SAUnicode)
timestamp = Column(DateTime)
- message_id = Column(Unicode)
+ message_id = Column(SAUnicode)
context = Column(Enum(BounceContext))
processed = Column(Boolean)
diff --git a/src/mailman/model/cache.py b/src/mailman/model/cache.py
index d9c0d0445..6be5fbc9d 100644
--- a/src/mailman/model/cache.py
+++ b/src/mailman/model/cache.py
@@ -26,9 +26,10 @@ from mailman import public
from mailman.config import config
from mailman.database.model import Model
from mailman.database.transaction import dbconnection
+from mailman.database.types import SAUnicode
from mailman.interfaces.cache import ICacheManager
from mailman.utilities.datetime import now
-from sqlalchemy import Boolean, Column, DateTime, Integer, Unicode
+from sqlalchemy import Boolean, Column, DateTime, Integer
from zope.interface import implementer
@@ -36,8 +37,8 @@ class CacheEntry(Model):
__tablename__ = 'file_cache'
id = Column(Integer, primary_key=True)
- key = Column(Unicode)
- file_id = Column(Unicode)
+ key = Column(SAUnicode)
+ file_id = Column(SAUnicode)
is_bytes = Column(Boolean)
created_on = Column(DateTime)
expires_on = Column(DateTime)
diff --git a/src/mailman/model/domain.py b/src/mailman/model/domain.py
index 575d157cf..5bb569585 100644
--- a/src/mailman/model/domain.py
+++ b/src/mailman/model/domain.py
@@ -20,13 +20,14 @@
from mailman import public
from mailman.database.model import Model
from mailman.database.transaction import dbconnection
+from mailman.database.types import SAUnicode
from mailman.interfaces.domain import (
BadDomainSpecificationError, DomainCreatedEvent, DomainCreatingEvent,
DomainDeletedEvent, DomainDeletingEvent, IDomain, IDomainManager)
from mailman.interfaces.user import IUser
from mailman.interfaces.usermanager import IUserManager
from mailman.model.mailinglist import MailingList
-from sqlalchemy import Column, Integer, Unicode
+from sqlalchemy import Column, Integer
from sqlalchemy.orm import relationship
from zope.component import getUtility
from zope.event import notify
@@ -42,8 +43,8 @@ class Domain(Model):
id = Column(Integer, primary_key=True)
- mail_host = Column(Unicode)
- description = Column(Unicode)
+ mail_host = Column(SAUnicode)
+ description = Column(SAUnicode)
owners = relationship('User',
secondary='domain_owner',
backref='domains')
diff --git a/src/mailman/model/language.py b/src/mailman/model/language.py
index f0a3326a8..596e580e1 100644
--- a/src/mailman/model/language.py
+++ b/src/mailman/model/language.py
@@ -19,8 +19,9 @@
from mailman import public
from mailman.database.model import Model
+from mailman.database.types import SAUnicode
from mailman.interfaces.languages import ILanguage
-from sqlalchemy import Column, Integer, Unicode
+from sqlalchemy import Column, Integer
from zope.interface import implementer
@@ -32,4 +33,4 @@ class Language(Model):
__tablename__ = 'language'
id = Column(Integer, primary_key=True)
- code = Column(Unicode)
+ code = Column(SAUnicode)
diff --git a/src/mailman/model/mailinglist.py b/src/mailman/model/mailinglist.py
index 9d7d63d9b..933384797 100644
--- a/src/mailman/model/mailinglist.py
+++ b/src/mailman/model/mailinglist.py
@@ -23,7 +23,7 @@ from mailman import public
from mailman.config import config
from mailman.database.model import Model
from mailman.database.transaction import dbconnection
-from mailman.database.types import Enum
+from mailman.database.types import Enum, SAUnicode
from mailman.interfaces.action import Action, FilterAction
from mailman.interfaces.address import IAddress
from mailman.interfaces.archiver import ArchivePolicy
@@ -51,7 +51,7 @@ from mailman.utilities.filesystem import makedirs
from mailman.utilities.string import expand
from sqlalchemy import (
Boolean, Column, DateTime, Float, ForeignKey, Integer, Interval,
- LargeBinary, PickleType, Unicode)
+ LargeBinary, PickleType)
from sqlalchemy.event import listen
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.orm import relationship
@@ -78,9 +78,9 @@ class MailingList(Model):
# are currently missing.
# List identity
- list_name = Column(Unicode, index=True)
- mail_host = Column(Unicode, index=True)
- _list_id = Column('list_id', Unicode, index=True, unique=True)
+ list_name = Column(SAUnicode, index=True)
+ mail_host = Column(SAUnicode, index=True)
+ _list_id = Column('list_id', SAUnicode, index=True, unique=True)
allow_list_posts = Column(Boolean)
include_rfc2369_headers = Column(Boolean)
advertised = Column(Boolean)
@@ -106,11 +106,11 @@ class MailingList(Model):
# Automatic responses.
autoresponse_grace_period = Column(Interval)
autorespond_owner = Column(Enum(ResponseAction))
- autoresponse_owner_text = Column(Unicode)
+ autoresponse_owner_text = Column(SAUnicode)
autorespond_postings = Column(Enum(ResponseAction))
- autoresponse_postings_text = Column(Unicode)
+ autoresponse_postings_text = Column(SAUnicode)
autorespond_requests = Column(Enum(ResponseAction))
- autoresponse_request_text = Column(Unicode)
+ autoresponse_request_text = Column(SAUnicode)
# Content filters.
filter_action = Column(Enum(FilterAction))
filter_content = Column(Boolean)
@@ -118,7 +118,7 @@ class MailingList(Model):
convert_html_to_plaintext = Column(Boolean)
# Bounces.
bounce_info_stale_after = Column(Interval) # XXX
- bounce_matching_headers = Column(Unicode) # XXX
+ bounce_matching_headers = Column(SAUnicode) # XXX
bounce_notify_owner_on_disable = Column(Boolean) # XXX
bounce_notify_owner_on_removal = Column(Boolean) # XXX
bounce_score_threshold = Column(Integer) # XXX
@@ -130,7 +130,7 @@ class MailingList(Model):
# Miscellaneous
default_member_action = Column(Enum(Action))
default_nonmember_action = Column(Enum(Action))
- description = Column(Unicode)
+ description = Column(SAUnicode)
digests_enabled = Column(Boolean)
digest_is_default = Column(Boolean)
digest_send_periodic = Column(Boolean)
@@ -144,36 +144,36 @@ class MailingList(Model):
gateway_to_mail = Column(Boolean)
gateway_to_news = Column(Boolean)
hold_these_nonmembers = Column(PickleType)
- info = Column(Unicode)
- linked_newsgroup = Column(Unicode)
+ info = Column(SAUnicode)
+ linked_newsgroup = Column(SAUnicode)
max_days_to_hold = Column(Integer)
max_message_size = Column(Integer)
max_num_recipients = Column(Integer)
- member_moderation_notice = Column(Unicode)
+ member_moderation_notice = Column(SAUnicode)
mime_is_default_digest = Column(Boolean)
# FIXME: There should be no moderator_password
moderator_password = Column(LargeBinary) # TODO : was RawStr()
newsgroup_moderation = Column(Enum(NewsgroupModeration))
nntp_prefix_subject_too = Column(Boolean)
- nonmember_rejection_notice = Column(Unicode)
+ nonmember_rejection_notice = Column(SAUnicode)
obscure_addresses = Column(Boolean)
- owner_chain = Column(Unicode)
- owner_pipeline = Column(Unicode)
+ owner_chain = Column(SAUnicode)
+ owner_pipeline = Column(SAUnicode)
personalize = Column(Enum(Personalization))
post_id = Column(Integer)
- posting_chain = Column(Unicode)
- posting_pipeline = Column(Unicode)
- _preferred_language = Column('preferred_language', Unicode)
- display_name = Column(Unicode)
+ posting_chain = Column(SAUnicode)
+ posting_pipeline = Column(SAUnicode)
+ _preferred_language = Column('preferred_language', SAUnicode)
+ display_name = Column(SAUnicode)
reject_these_nonmembers = Column(PickleType)
reply_goes_to_list = Column(Enum(ReplyToMunging))
- reply_to_address = Column(Unicode)
+ reply_to_address = Column(SAUnicode)
require_explicit_destination = Column(Boolean)
respond_to_post_requests = Column(Boolean)
scrub_nondigest = Column(Boolean)
send_goodbye_message = Column(Boolean)
send_welcome_message = Column(Boolean)
- subject_prefix = Column(Unicode)
+ subject_prefix = Column(SAUnicode)
subscription_policy = Column(Enum(SubscriptionPolicy))
topics = Column(PickleType)
topics_bodylines_limit = Column(Integer)
@@ -487,7 +487,7 @@ class AcceptableAlias(Model):
Integer, ForeignKey('mailinglist.id'),
index=True, nullable=False)
mailing_list = relationship('MailingList', backref='acceptablealias')
- alias = Column(Unicode, index=True, nullable=False)
+ alias = Column(SAUnicode, index=True, nullable=False)
def __init__(self, mailing_list, alias):
super().__init__()
@@ -545,7 +545,7 @@ class ListArchiver(Model):
index=True, nullable=False)
mailing_list = relationship('MailingList')
- name = Column(Unicode, nullable=False)
+ name = Column(SAUnicode, nullable=False)
_is_enabled = Column(Boolean)
def __init__(self, mailing_list, archiver_name, system_archiver):
@@ -617,9 +617,9 @@ class HeaderMatch(Model):
index=True, nullable=False)
_position = Column('position', Integer, index=True, default=0)
- header = Column(Unicode)
- pattern = Column(Unicode)
- chain = Column(Unicode, nullable=True)
+ header = Column(SAUnicode)
+ pattern = Column(SAUnicode)
+ chain = Column(SAUnicode, nullable=True)
def __init__(self, **kw):
position = kw.pop('position', None)
diff --git a/src/mailman/model/member.py b/src/mailman/model/member.py
index 3aaa5e392..aacaa73b6 100644
--- a/src/mailman/model/member.py
+++ b/src/mailman/model/member.py
@@ -21,7 +21,7 @@ from mailman import public
from mailman.core.constants import system_preferences
from mailman.database.model import Model
from mailman.database.transaction import dbconnection
-from mailman.database.types import Enum, UUID
+from mailman.database.types import Enum, SAUnicode, UUID
from mailman.interfaces.action import Action
from mailman.interfaces.address import IAddress
from mailman.interfaces.listmanager import IListManager
@@ -30,7 +30,7 @@ from mailman.interfaces.member import (
from mailman.interfaces.user import IUser, UnverifiedAddressError
from mailman.interfaces.usermanager import IUserManager
from mailman.utilities.uid import UIDFactory
-from sqlalchemy import Column, ForeignKey, Integer, Unicode
+from sqlalchemy import Column, ForeignKey, Integer
from sqlalchemy.orm import relationship
from zope.component import getUtility
from zope.event import notify
@@ -50,7 +50,7 @@ class Member(Model):
id = Column(Integer, primary_key=True)
_member_id = Column(UUID)
role = Column(Enum(MemberRole), index=True)
- list_id = Column(Unicode, index=True)
+ list_id = Column(SAUnicode, index=True)
moderation_action = Column(Enum(Action))
address_id = Column(Integer, ForeignKey('address.id'), index=True)
diff --git a/src/mailman/model/message.py b/src/mailman/model/message.py
index c4fe63f1c..576baab5c 100644
--- a/src/mailman/model/message.py
+++ b/src/mailman/model/message.py
@@ -20,8 +20,9 @@
from mailman import public
from mailman.database.model import Model
from mailman.database.transaction import dbconnection
+from mailman.database.types import SAUnicode
from mailman.interfaces.messages import IMessage
-from sqlalchemy import Column, Integer, Unicode
+from sqlalchemy import Column, Integer
from zope.interface import implementer
@@ -34,9 +35,9 @@ class Message(Model):
id = Column(Integer, primary_key=True)
# This is a Messge-ID field representation, not a database row id.
- message_id = Column(Unicode)
- message_id_hash = Column(Unicode)
- path = Column(Unicode)
+ message_id = Column(SAUnicode)
+ message_id_hash = Column(SAUnicode)
+ path = Column(SAUnicode)
@dbconnection
def __init__(self, store, message_id, message_id_hash, path):
diff --git a/src/mailman/model/mime.py b/src/mailman/model/mime.py
index 849eac385..da5f11f05 100644
--- a/src/mailman/model/mime.py
+++ b/src/mailman/model/mime.py
@@ -19,9 +19,9 @@
from mailman import public
from mailman.database.model import Model
-from mailman.database.types import Enum
+from mailman.database.types import Enum, SAUnicode
from mailman.interfaces.mime import FilterType, IContentFilter
-from sqlalchemy import Column, ForeignKey, Integer, Unicode
+from sqlalchemy import Column, ForeignKey, Integer
from sqlalchemy.orm import relationship
from zope.interface import implementer
@@ -39,7 +39,7 @@ class ContentFilter(Model):
mailing_list = relationship('MailingList')
filter_type = Column(Enum(FilterType))
- filter_pattern = Column(Unicode)
+ filter_pattern = Column(SAUnicode)
def __init__(self, mailing_list, filter_pattern, filter_type):
self.mailing_list = mailing_list
diff --git a/src/mailman/model/pending.py b/src/mailman/model/pending.py
index 0e19f0cbf..9d8315605 100644
--- a/src/mailman/model/pending.py
+++ b/src/mailman/model/pending.py
@@ -24,11 +24,12 @@ from mailman import public
from mailman.config import config
from mailman.database.model import Model
from mailman.database.transaction import dbconnection
+from mailman.database.types import SAUnicode
from mailman.interfaces.pending import (
IPendable, IPended, IPendedKeyValue, IPendings)
from mailman.utilities.datetime import now
from mailman.utilities.uid import TokenFactory
-from sqlalchemy import Column, DateTime, ForeignKey, Integer, Unicode, and_
+from sqlalchemy import Column, DateTime, ForeignKey, Integer, and_
from sqlalchemy.orm import aliased, relationship
from zope.interface import implementer
from zope.interface.verify import verifyObject
@@ -45,8 +46,8 @@ class PendedKeyValue(Model):
__tablename__ = 'pendedkeyvalue'
id = Column(Integer, primary_key=True)
- key = Column(Unicode, index=True)
- value = Column(Unicode, index=True)
+ key = Column(SAUnicode, index=True)
+ value = Column(SAUnicode, index=True)
pended_id = Column(Integer, ForeignKey('pended.id'), index=True)
def __init__(self, key, value):
@@ -62,9 +63,9 @@ class Pended(Model):
__tablename__ = 'pended'
id = Column(Integer, primary_key=True)
- token = Column(Unicode, index=True)
+ token = Column(SAUnicode, index=True)
expiration_date = Column(DateTime, index=True)
- key_values = relationship('PendedKeyValue', cascade="all, delete-orphan")
+ key_values = relationship('PendedKeyValue', cascade='all, delete-orphan')
@public
diff --git a/src/mailman/model/preferences.py b/src/mailman/model/preferences.py
index 71775117f..366ed97c2 100644
--- a/src/mailman/model/preferences.py
+++ b/src/mailman/model/preferences.py
@@ -20,11 +20,11 @@
from mailman import public
from mailman.database.model import Model
from mailman.database.transaction import dbconnection
-from mailman.database.types import Enum
+from mailman.database.types import Enum, SAUnicode
from mailman.interfaces.languages import ILanguageManager
from mailman.interfaces.member import DeliveryMode, DeliveryStatus
from mailman.interfaces.preferences import IPreferences
-from sqlalchemy import Boolean, Column, Integer, Unicode
+from sqlalchemy import Boolean, Column, Integer
from zope.component import getUtility
from zope.interface import implementer
@@ -39,7 +39,7 @@ class Preferences(Model):
id = Column(Integer, primary_key=True)
acknowledge_posts = Column(Boolean)
hide_address = Column(Boolean)
- _preferred_language = Column('preferred_language', Unicode)
+ _preferred_language = Column('preferred_language', SAUnicode)
receive_list_copy = Column(Boolean)
receive_own_postings = Column(Boolean)
delivery_mode = Column(Enum(DeliveryMode))
diff --git a/src/mailman/model/requests.py b/src/mailman/model/requests.py
index df5f1062e..0f0f96dfe 100644
--- a/src/mailman/model/requests.py
+++ b/src/mailman/model/requests.py
@@ -21,12 +21,12 @@ from datetime import timedelta
from mailman import public
from mailman.database.model import Model
from mailman.database.transaction import dbconnection
-from mailman.database.types import Enum
+from mailman.database.types import Enum, SAUnicode
from mailman.interfaces.pending import IPendable, IPendings
from mailman.interfaces.requests import IListRequests, RequestType
from mailman.utilities.queries import QuerySequence
from pickle import dumps, loads
-from sqlalchemy import Column, ForeignKey, Integer, Unicode
+from sqlalchemy import Column, ForeignKey, Integer
from sqlalchemy.orm import relationship
from zope.component import getUtility
from zope.interface import implementer
@@ -42,7 +42,7 @@ class DataPendable(dict):
def update(self, mapping):
# Keys and values must be strings (unicodes, but bytes values are
# accepted for now). Any other types for keys are a programming
- # error. If we find a non-Unicode value, pickle it and encode it in
+ # error. If we find a non-SAUnicode value, pickle it and encode it in
# such a way that it will be properly reconstituted when unpended.
clean_mapping = {}
for key, value in mapping.items():
@@ -120,7 +120,7 @@ class ListRequests:
if pendable is None:
return None
data = dict()
- # Unpickle any non-Unicode values.
+ # Unpickle any non-SAUnicode values.
for key, value in pendable.items():
if key.startswith('_pck_'):
data[key[5:]] = loads(value.encode('raw-unicode-escape'))
@@ -146,9 +146,9 @@ class _Request(Model):
__tablename__ = '_request'
id = Column(Integer, primary_key=True)
- key = Column(Unicode)
+ key = Column(SAUnicode)
request_type = Column(Enum(RequestType))
- data_hash = Column(Unicode)
+ data_hash = Column(SAUnicode)
mailing_list_id = Column(Integer, ForeignKey('mailinglist.id'), index=True)
mailing_list = relationship('MailingList')
diff --git a/src/mailman/model/template.py b/src/mailman/model/template.py
index 96ce43e1f..b09c644bb 100644
--- a/src/mailman/model/template.py
+++ b/src/mailman/model/template.py
@@ -23,6 +23,7 @@ from mailman import public
from mailman.config import config
from mailman.database.model import Model
from mailman.database.transaction import dbconnection
+from mailman.database.types import SAUnicode
from mailman.interfaces.cache import ICacheManager
from mailman.interfaces.domain import IDomain
from mailman.interfaces.mailinglist import IMailingList
@@ -32,7 +33,7 @@ from mailman.utilities import protocols
from mailman.utilities.i18n import find
from mailman.utilities.string import expand
from requests import HTTPError
-from sqlalchemy import Column, Integer, Unicode
+from sqlalchemy import Column, Integer
from urllib.error import URLError
from urllib.parse import urlparse
from zope.component import getUtility
@@ -47,11 +48,11 @@ class Template(Model):
__tablename__ = 'template'
id = Column(Integer, primary_key=True)
- name = Column(Unicode)
- context = Column(Unicode)
- uri = Column(Unicode)
- username = Column(Unicode, nullable=True)
- password = Column(Unicode, nullable=True)
+ name = Column(SAUnicode)
+ context = Column(SAUnicode)
+ uri = Column(SAUnicode)
+ username = Column(SAUnicode, nullable=True)
+ password = Column(SAUnicode, nullable=True)
def __init__(self, name, context, uri, username, password):
self.name = name
diff --git a/src/mailman/model/tests/test_listmanager.py b/src/mailman/model/tests/test_listmanager.py
index a8ac68c92..ff1bd3026 100644
--- a/src/mailman/model/tests/test_listmanager.py
+++ b/src/mailman/model/tests/test_listmanager.py
@@ -174,7 +174,9 @@ class TestListCreation(unittest.TestCase):
def test_create_list_case_folding(self):
# LP: #1117176 describes a problem where list names created in upper
- # case are not actually usable by the LMTP server.
+ # case are not actually usable by the LMTP server. MySQL
+ # automatically changes the case of the arguments so this test will
+ # always fail in case of MySQL.
self._manager.create('my-LIST@example.com')
self.assertIsNone(self._manager.get('my-LIST@example.com'))
mlist = self._manager.get('my-list@example.com')
diff --git a/src/mailman/model/user.py b/src/mailman/model/user.py
index a534ef2a5..edda2a9e8 100644
--- a/src/mailman/model/user.py
+++ b/src/mailman/model/user.py
@@ -20,7 +20,7 @@
from mailman import public
from mailman.database.model import Model
from mailman.database.transaction import dbconnection
-from mailman.database.types import UUID
+from mailman.database.types import SAUnicode, UUID
from mailman.interfaces.address import (
AddressAlreadyLinkedError, AddressNotLinkedError)
from mailman.interfaces.user import (
@@ -32,7 +32,7 @@ from mailman.model.roster import Memberships
from mailman.utilities.datetime import factory as date_factory
from mailman.utilities.uid import UIDFactory
from sqlalchemy import (
- Boolean, Column, DateTime, ForeignKey, Integer, Unicode)
+ Boolean, Column, DateTime, ForeignKey, Integer)
from sqlalchemy.orm import backref, relationship
from zope.event import notify
from zope.interface import implementer
@@ -49,8 +49,8 @@ class User(Model):
__tablename__ = 'user'
id = Column(Integer, primary_key=True)
- display_name = Column(Unicode)
- _password = Column('password', Unicode)
+ display_name = Column(SAUnicode)
+ _password = Column('password', SAUnicode)
_user_id = Column(UUID, index=True)
_created_on = Column(DateTime)
is_server_owner = Column(Boolean, default=False)
diff --git a/src/mailman/model/workflow.py b/src/mailman/model/workflow.py
index 7a0056e5f..1072fa548 100644
--- a/src/mailman/model/workflow.py
+++ b/src/mailman/model/workflow.py
@@ -20,8 +20,9 @@
from mailman import public
from mailman.database.model import Model
from mailman.database.transaction import dbconnection
+from mailman.database.types import SAUnicode
from mailman.interfaces.workflow import IWorkflowState, IWorkflowStateManager
-from sqlalchemy import Column, Unicode
+from sqlalchemy import Column
from zope.interface import implementer
@@ -32,10 +33,10 @@ class WorkflowState(Model):
__tablename__ = 'workflowstate'
- name = Column(Unicode, primary_key=True)
- token = Column(Unicode, primary_key=True)
- step = Column(Unicode)
- data = Column(Unicode)
+ name = Column(SAUnicode, primary_key=True)
+ token = Column(SAUnicode, primary_key=True)
+ step = Column(SAUnicode)
+ data = Column(SAUnicode)
@public
diff --git a/tox.ini b/tox.ini
index bce179a0c..90054a2fb 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
[tox]
-envlist = {py34,py35}{,-coverage,-diffcov}{,-pg},qa
+envlist = {py34,py35}{,-coverage,-diffcov}{,-pg}{,-mysql},qa
recreate = True
skip_missing_interpreters = True
@@ -18,6 +18,7 @@ usedevelop = True
deps =
{coverage,diffcov}: coverage
pg: psycopg2
+ mysql: pymysql
diffcov: diff_cover
passenv =
MAILMAN_*