summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/mailman/commands/cli_migrate.py26
-rw-r--r--src/mailman/commands/docs/conf.rst2
-rw-r--r--src/mailman/config/alembic.cfg57
-rw-r--r--src/mailman/config/config.py11
-rw-r--r--src/mailman/config/schema.cfg9
-rw-r--r--src/mailman/database/alembic/__init__.py32
-rw-r--r--src/mailman/database/alembic/env.py68
-rw-r--r--src/mailman/database/base.py1
-rw-r--r--src/mailman/database/factory.py13
-rw-r--r--src/mailman/database/postgresql.py3
-rw-r--r--src/mailman/database/tests/test_factory.py5
-rw-r--r--src/mailman/database/types.py2
-rw-r--r--src/mailman/model/docs/messagestore.rst5
-rw-r--r--src/mailman/model/message.py2
-rw-r--r--src/mailman/model/messagestore.py5
-rw-r--r--src/mailman/styles/base.py8
-rw-r--r--src/mailman/testing/layers.py6
-rw-r--r--src/mailman/utilities/importer.py12
-rw-r--r--src/mailman/utilities/modules.py15
19 files changed, 181 insertions, 101 deletions
diff --git a/src/mailman/commands/cli_migrate.py b/src/mailman/commands/cli_migrate.py
index 85fd07bd4..82bf4a708 100644
--- a/src/mailman/commands/cli_migrate.py
+++ b/src/mailman/commands/cli_migrate.py
@@ -22,20 +22,23 @@ from __future__ import absolute_import, print_function, unicode_literals
__metaclass__ = type
__all__ = [
'Migrate',
-]
+ ]
+
from alembic import command
-from alembic.config import Config
from zope.interface import implementer
from mailman.config import config
from mailman.core.i18n import _
+from mailman.database.alembic import alembic_cfg
from mailman.interfaces.command import ICLISubCommand
+from mailman.utilities.modules import expand_path
+
@implementer(ICLISubCommand)
class Migrate:
- """Migrate the mailman database to the schema."""
+ """Migrate the Mailman database to the latest schema."""
name = 'migrate'
@@ -43,16 +46,17 @@ class Migrate:
"""See `ICLISubCommand`."""
command_parser.add_argument(
'-a', '--autogenerate',
- action="store_true", help=_("""\
- Autogenerate the migration script using alembic"""))
- pass
+ action='store_true', help=_("""\
+ Autogenerate the migration script using Alembic."""))
+ command_parser.add_argument(
+ '-q', '--quiet',
+ action='store_true', default=False,
+ help=('Produce less output.'))
def process(self, args):
- alembic_cfg= Config()
- alembic_cfg.set_main_option(
- "script_location", config.alembic['script_location'])
if args.autogenerate:
command.revision(alembic_cfg, autogenerate=True)
else:
- command.upgrade(alembic_cfg, "head")
- print("Updated the database schema.")
+ command.upgrade(alembic_cfg, 'head')
+ if not args.quiet:
+ print('Updated the database schema.')
diff --git a/src/mailman/commands/docs/conf.rst b/src/mailman/commands/docs/conf.rst
index 99ca6b054..7b8529ac3 100644
--- a/src/mailman/commands/docs/conf.rst
+++ b/src/mailman/commands/docs/conf.rst
@@ -22,7 +22,7 @@ To get a list of all key-value pairs of any section, you need to call the
command without any options.
>>> command.process(FakeArgs)
- [alembic] script_location: mailman:database/alembic
+ [logging.archiver] path: mailman.log
...
[passwords] password_length: 8
...
diff --git a/src/mailman/config/alembic.cfg b/src/mailman/config/alembic.cfg
new file mode 100644
index 000000000..b1f53a3d2
--- /dev/null
+++ b/src/mailman/config/alembic.cfg
@@ -0,0 +1,57 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = mailman.database:alembic
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers = console
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = WARN
+handlers = console
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = WARN
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S
diff --git a/src/mailman/config/config.py b/src/mailman/config/config.py
index 287de9798..0ba88e089 100644
--- a/src/mailman/config/config.py
+++ b/src/mailman/config/config.py
@@ -33,7 +33,7 @@ import sys
from ConfigParser import SafeConfigParser
from flufl.lock import Lock
from lazr.config import ConfigSchema, as_boolean
-from pkg_resources import resource_filename, resource_stream, resource_string
+from pkg_resources import resource_stream, resource_string
from string import Template
from zope.component import getUtility
from zope.event import notify
@@ -46,7 +46,7 @@ from mailman.interfaces.configuration import (
ConfigurationUpdatedEvent, IConfiguration, MissingConfigurationFileError)
from mailman.interfaces.languages import ILanguageManager
from mailman.utilities.filesystem import makedirs
-from mailman.utilities.modules import call_name
+from mailman.utilities.modules import call_name, expand_path
SPACE = ' '
@@ -306,12 +306,7 @@ def external_configuration(path):
:return: A `ConfigParser` instance.
"""
# Is the context coming from a file system or Python path?
- if path.startswith('python:'):
- resource_path = path[7:]
- package, dot, resource = resource_path.rpartition('.')
- cfg_path = resource_filename(package, resource + '.cfg')
- else:
- cfg_path = path
+ cfg_path = expand_path(path)
parser = SafeConfigParser()
files = parser.read(cfg_path)
if files != [cfg_path]:
diff --git a/src/mailman/config/schema.cfg b/src/mailman/config/schema.cfg
index f42ba9b66..e8f89ab41 100644
--- a/src/mailman/config/schema.cfg
+++ b/src/mailman/config/schema.cfg
@@ -204,8 +204,10 @@ class: mailman.database.sqlite.SQLiteDatabase
url: sqlite:///$DATA_DIR/mailman.db
debug: no
-# The module path to the migrations modules.
-migrations_path: mailman.database.schema
+# Where can we find the Alembic migration scripts? The `python:` schema means
+# that this is a module path relative to the Mailman 3 source installation.
+alembic_scripts: mailman.database:alembic
+
[logging.template]
# This defines various log settings. The options available are:
@@ -640,6 +642,3 @@ rewrite_duplicate_headers:
CC X-Original-CC
Content-Transfer-Encoding X-Original-Content-Transfer-Encoding
MIME-Version X-MIME-Version
-
-[alembic]
-script_location: mailman:database/alembic
diff --git a/src/mailman/database/alembic/__init__.py b/src/mailman/database/alembic/__init__.py
index e69de29bb..73f30832e 100644
--- a/src/mailman/database/alembic/__init__.py
+++ b/src/mailman/database/alembic/__init__.py
@@ -0,0 +1,32 @@
+# Copyright (C) 2014 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
+
+"Alembic config init."
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+ 'alembic_cfg'
+]
+
+
+from alembic.config import Config
+from mailman.utilities.modules import expand_path
+
+
+alembic_cfg=Config(expand_path("python:mailman.config.alembic"))
diff --git a/src/mailman/database/alembic/env.py b/src/mailman/database/alembic/env.py
index 402c4f7da..d1caec58d 100644
--- a/src/mailman/database/alembic/env.py
+++ b/src/mailman/database/alembic/env.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2006-2014 by the Free Software Foundation, Inc.
+# Copyright (C) 2014 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
@@ -15,40 +15,45 @@
# You should have received a copy of the GNU General Public License along with
# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
-from __future__ import with_statement
+"""Alembic migration environment."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+ 'run_migrations_offline',
+ 'run_migrations_online',
+ ]
+
from alembic import context
-from alembic.config import Config
-from sqlalchemy import create_engine, pool
+from contextlib import closing
+from logging.config import fileConfig
+from sqlalchemy import create_engine
from mailman.core import initialize
from mailman.config import config
-from mailman.utilities.string import expand
+from mailman.database.alembic import alembic_cfg
from mailman.database.model import Model
+from mailman.utilities.modules import expand_path
+from mailman.utilities.string import expand
-target_metadata = Model.metadata
+fileConfig(alembic_cfg.config_file_name)
+
def run_migrations_offline():
"""Run migrations in 'offline' mode.
- This configures the context with just a URL
- and not an Engine, though an Engine is acceptable
- here as well. By skipping the Engine creation
- we don't even need a DBAPI to be available.
-
- Calls to context.execute() here emit the given string to the
- script output.
+ This configures the context with just a URL and not an Engine,
+ though an Engine is acceptable here as well. By skipping the Engine
+ creation we don't even need a DBAPI to be available.
+ Calls to context.execute() here emit the given string to the script
+ output.
"""
- if not config.initialized:
- initialize.initialize_1(context.config.config_file_name)
- alembic_cfg= Config()
- alembic_cfg.set_main_option(
- "script_location", config.alembic['script_location'])
url = expand(config.database.url, config.paths)
- context.configure(url=url, target_metadata=target_metadata)
-
+ context.configure(url=url, target_metadata=Model.metadata)
with context.begin_transaction():
context.run_migrations()
@@ -56,30 +61,19 @@ def run_migrations_offline():
def run_migrations_online():
"""Run migrations in 'online' mode.
- In this scenario we need to create an Engine
- and associate a connection with the context.
-
+ In this scenario we need to create an Engine and associate a
+ connection with the context.
"""
- if not config.initialized:
- initialize.initialize_1(context.config.config_file_name)
- alembic_cfg= Config()
- alembic_cfg.set_main_option(
- "script_location", config.alembic['script_location'])
- alembic_cfg.set_section_option('logger_alembic' ,'level' , 'ERROR')
url = expand(config.database.url, config.paths)
engine = create_engine(url)
connection = engine.connect()
- context.configure(
- connection=connection,
- target_metadata=target_metadata
- )
-
- try:
+ with closing(connection):
+ context.configure(
+ connection=connection, target_metadata=Model.metadata)
with context.begin_transaction():
context.run_migrations()
- finally:
- connection.close()
+
if context.is_offline_mode():
run_migrations_offline()
diff --git a/src/mailman/database/base.py b/src/mailman/database/base.py
index 784efcd68..6e86faf04 100644
--- a/src/mailman/database/base.py
+++ b/src/mailman/database/base.py
@@ -26,7 +26,6 @@ __all__ = [
import logging
from alembic import command
-from alembic.config import Config
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from zope.interface import implementer
diff --git a/src/mailman/database/factory.py b/src/mailman/database/factory.py
index fedfb816f..7d5c2cc73 100644
--- a/src/mailman/database/factory.py
+++ b/src/mailman/database/factory.py
@@ -30,7 +30,6 @@ import os
import types
from alembic import command
-from alembic.config import Config as AlembicConfig
from alembic.migration import MigrationContext
from alembic.script import ScriptDirectory
from flufl.lock import Lock
@@ -40,8 +39,9 @@ from zope.interface.verify import verifyObject
from mailman.config import config
from mailman.database.model import Model
+from mailman.database.alembic import alembic_cfg
from mailman.interfaces.database import IDatabase, IDatabaseFactory
-from mailman.utilities.modules import call_name
+from mailman.utilities.modules import call_name, expand_path
@@ -70,10 +70,7 @@ class SchemaManager:
def __init__(self, database):
self.database = database
- self.alembic_cfg = AlembicConfig()
- self.alembic_cfg.set_main_option(
- "script_location", config.alembic['script_location'])
- self.script = ScriptDirectory.from_config(self.alembic_cfg)
+ self.script = ScriptDirectory.from_config(alembic_cfg)
def get_storm_schema_version(self):
md = MetaData()
@@ -89,10 +86,10 @@ class SchemaManager:
def _create(self):
# initial DB creation
Model.metadata.create_all(self.database.engine)
- command.stamp(self.alembic_cfg, "head")
+ command.stamp(alembic_cfg, "head")
def _upgrade(self):
- command.upgrade(self.alembic_cfg, "head")
+ command.upgrade(alembic_cfg, "head")
def setup_db(self):
context = MigrationContext.configure(self.database.store.connection())
diff --git a/src/mailman/database/postgresql.py b/src/mailman/database/postgresql.py
index ca1068302..717b69dd1 100644
--- a/src/mailman/database/postgresql.py
+++ b/src/mailman/database/postgresql.py
@@ -26,7 +26,7 @@ __all__ = [
from mailman.database.base import SABaseDatabase
-from operator import attrgetter
+from mailman.database.model import Model
@@ -40,7 +40,6 @@ class PostgreSQLDatabase(SABaseDatabase):
restart from zero for new tests.
"""
super(PostgreSQLDatabase, self)._post_reset(store)
- from mailman.database.model import Model
tables = reversed(Model.metadata.sorted_tables)
# Recipe adapted from
# http://stackoverflow.com/questions/544791/
diff --git a/src/mailman/database/tests/test_factory.py b/src/mailman/database/tests/test_factory.py
index 42a05fe1b..81febbde5 100644
--- a/src/mailman/database/tests/test_factory.py
+++ b/src/mailman/database/tests/test_factory.py
@@ -35,6 +35,7 @@ from mailman.config import config
from mailman.testing.layers import DatabaseLayer
from mailman.database.factory import SchemaManager, _reset
from mailman.database.sqlite import SQLiteDatabase
+from mailman.database.alembic import alembic_cfg
from mailman.database.model import Model
@@ -75,7 +76,7 @@ class TestSchemaManager(unittest.TestCase):
def test_current_db(self):
"""The database is already at the latest version"""
- alembic.command.stamp(self.schema_mgr.alembic_cfg, "head")
+ alembic.command.stamp(alembic_cfg, "head")
self.schema_mgr._create = Mock()
self.schema_mgr._upgrade = Mock()
self.schema_mgr.setup_db()
@@ -114,7 +115,7 @@ class TestSchemaManager(unittest.TestCase):
def test_old_db(self):
"""The database is in an old revision, must upgrade"""
- alembic.command.stamp(self.schema_mgr.alembic_cfg, "head")
+ alembic.command.stamp(alembic_cfg, "head")
md = MetaData()
md.reflect(bind=config.db.engine)
config.db.store.execute(md.tables["alembic_version"].delete())
diff --git a/src/mailman/database/types.py b/src/mailman/database/types.py
index eec6df6d5..1984b08b5 100644
--- a/src/mailman/database/types.py
+++ b/src/mailman/database/types.py
@@ -29,8 +29,8 @@ __all__ = [
import uuid
from sqlalchemy import Integer
-from sqlalchemy.types import TypeDecorator, CHAR
from sqlalchemy.dialects import postgresql
+from sqlalchemy.types import TypeDecorator, CHAR
diff --git a/src/mailman/model/docs/messagestore.rst b/src/mailman/model/docs/messagestore.rst
index bb853e575..f2f2ca9d2 100644
--- a/src/mailman/model/docs/messagestore.rst
+++ b/src/mailman/model/docs/messagestore.rst
@@ -28,8 +28,9 @@ header, you will get an exception.
However, if the message has a ``Message-ID`` header, it can be stored.
>>> msg['Message-ID'] = '<87myycy5eh.fsf@uwakimon.sk.tsukuba.ac.jp>'
- >>> message_store.add(msg)
- 'AGDWSNXXKCWEILKKNYTBOHRDQGOX3Y35'
+ >>> x_message_id_hash = message_store.add(msg)
+ >>> print(x_message_id_hash)
+ AGDWSNXXKCWEILKKNYTBOHRDQGOX3Y35
>>> print(msg.as_string())
Subject: An important message
Message-ID: <87myycy5eh.fsf@uwakimon.sk.tsukuba.ac.jp>
diff --git a/src/mailman/model/message.py b/src/mailman/model/message.py
index c03d4bbd6..691861d46 100644
--- a/src/mailman/model/message.py
+++ b/src/mailman/model/message.py
@@ -40,10 +40,10 @@ class Message(Model):
__tablename__ = 'message'
id = Column(Integer, primary_key=True)
+ # This is a Messge-ID field representation, not a database row id.
message_id = Column(Unicode)
message_id_hash = Column(LargeBinary)
path = Column(LargeBinary)
- # This is a Messge-ID field representation, not a database row id.
@dbconnection
def __init__(self, store, message_id, message_id_hash, path):
diff --git a/src/mailman/model/messagestore.py b/src/mailman/model/messagestore.py
index 0b8a0ac78..19fa8133f 100644
--- a/src/mailman/model/messagestore.py
+++ b/src/mailman/model/messagestore.py
@@ -117,9 +117,8 @@ class MessageStore:
def get_message_by_hash(self, store, message_id_hash):
# It's possible the hash came from a message header, in which case it
# will be a Unicode. However when coming from source code, it may be
- # an 8-string. Coerce to the latter if necessary; it must be
- # US-ASCII.
- if isinstance(message_id_hash, unicode):
+ # bytes object. Coerce to the latter if necessary; it must be ASCII.
+ if not isinstance(message_id_hash, bytes):
message_id_hash = message_id_hash.encode('ascii')
row = store.query(Message).filter_by(
message_id_hash=message_id_hash).first()
diff --git a/src/mailman/styles/base.py b/src/mailman/styles/base.py
index 4f051c13c..0d65bbebb 100644
--- a/src/mailman/styles/base.py
+++ b/src/mailman/styles/base.py
@@ -64,12 +64,8 @@ class Identity:
mlist.info = ''
mlist.preferred_language = 'en'
mlist.subject_prefix = _('[$mlist.display_name] ')
- # Set this to Never if the list's preferred language uses us-ascii,
- # otherwise set it to As Needed.
- if mlist.preferred_language.charset == 'us-ascii':
- mlist.encode_ascii_prefixes = False
- else:
- mlist.encode_ascii_prefixes = True
+ mlist.encode_ascii_prefixes = (
+ mlist.preferred_language.charset != 'us-ascii')
diff --git a/src/mailman/testing/layers.py b/src/mailman/testing/layers.py
index 68cb07f8c..c1346b29f 100644
--- a/src/mailman/testing/layers.py
+++ b/src/mailman/testing/layers.py
@@ -195,9 +195,9 @@ class ConfigLayer(MockAndMonkeyLayer):
@classmethod
def tearDown(cls):
assert cls.var_dir is not None, 'Layer not set up'
- # Reset the test database after the tests are done so that there
- # is no data incase the tests are rerun with a database layer like
- # mysql or postgresql which are not deleted in teardown.
+ # Reset the test database after the tests are done so that there is no
+ # data in case the tests are rerun with a database layer like mysql or
+ # postgresql which are not deleted in teardown.
reset_the_world()
config.pop('test config')
shutil.rmtree(cls.var_dir)
diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py
index 99531194f..5b2be9742 100644
--- a/src/mailman/utilities/importer.py
+++ b/src/mailman/utilities/importer.py
@@ -128,12 +128,6 @@ def nonmember_action_mapping(value):
}[value]
-def int_to_bool(value):
- if value:
- return True
- else:
- return False
-
def check_language_code(code):
if code is None:
@@ -178,9 +172,9 @@ TYPES = dict(
personalize=Personalization,
preferred_language=check_language_code,
reply_goes_to_list=ReplyToMunging,
- allow_list_posts=int_to_bool,
- include_rfc2369_headers=int_to_bool,
- nntp_prefix_subject_too=int_to_bool,
+ allow_list_posts=bool,
+ include_rfc2369_headers=bool,
+ nntp_prefix_subject_too=bool,
)
diff --git a/src/mailman/utilities/modules.py b/src/mailman/utilities/modules.py
index 5dfec95db..9ff0e50cd 100644
--- a/src/mailman/utilities/modules.py
+++ b/src/mailman/utilities/modules.py
@@ -22,6 +22,7 @@ from __future__ import absolute_import, print_function, unicode_literals
__metaclass__ = type
__all__ = [
'call_name',
+ 'expand_path',
'find_components',
'find_name',
'scan_module',
@@ -31,7 +32,7 @@ __all__ = [
import os
import sys
-from pkg_resources import resource_listdir
+from pkg_resources import resource_filename, resource_listdir
@@ -110,3 +111,15 @@ def find_components(package, interface):
continue
for component in scan_module(module, interface):
yield component
+
+
+
+def expand_path(url):
+ """Expand a python: path, returning the absolute file system path."""
+ # Is the context coming from a file system or Python path?
+ if url.startswith('python:'):
+ resource_path = url[7:]
+ package, dot, resource = resource_path.rpartition('.')
+ return resource_filename(package, resource + '.cfg')
+ else:
+ return url
generated by cgit v1.2.3-70-g09d2 (git 2.51.2) at 2025-11-08 17:26:07 +0000