Database

--------
 * The ORM layer, previously implemented with Storm, has been replaced by
   SQLAlchemy, thanks to the fantastic work by Abhilash Raj and Aurélien
   Bompard.  Alembic is now used for all database schema migrations.
 * The new logger `mailman.database` logs any errors at the database layer.

API
---
 * Several changes to the internal API:
   - `IListManager.mailing_lists` is guaranteed to be sorted in List-ID order.
   - `IDomains.mailing_lists` is guaranteed to be sorted in List-ID order.
   - Iteration over domains via the `IDomainManager` is guaranteed to be sorted
     by `IDomain.mail_host` order.
   - `ITemporaryDatabase` interface and all implementations are removed.

3 files changed, 48 insertions, 2 deletions

diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py
index cf22af24f..baaa0e020 100644
--- a/src/mailman/utilities/importer.py
+++ b/src/mailman/utilities/importer.py
@@ -152,6 +152,7 @@ enabled: yes
 
 # Attributes in Mailman 2 which have a different type in Mailman 3.
 TYPES = dict(
+    allow_list_posts=bool,
     autorespond_owner=ResponseAction,
     autorespond_postings=ResponseAction,
     autorespond_requests=ResponseAction,
@@ -161,12 +162,15 @@ TYPES = dict(
     default_member_action=member_action_mapping,
     default_nonmember_action=nonmember_action_mapping,
     digest_volume_frequency=DigestFrequency,
+    encode_ascii_prefixes=bool,
     filter_action=filter_action_mapping,
     filter_extensions=list_members_to_unicode,
     filter_types=list_members_to_unicode,
     forward_unrecognized_bounces_to=UnrecognizedBounceDisposition,
+    include_rfc2369_headers=bool,
     moderator_password=unicode_to_string,
     newsgroup_moderation=NewsgroupModeration,
+    nntp_prefix_subject_too=bool,
     pass_extensions=list_members_to_unicode,
     pass_types=list_members_to_unicode,
     personalize=Personalization,
@@ -186,7 +190,6 @@ NAME_MAPPINGS = dict(
     filter_mime_types='filter_types',
     generic_nonmember_action='default_nonmember_action',
     include_list_post_header='allow_list_posts',
-    last_post_time='last_post_at',
     member_moderation_action='default_member_action',
     mod_password='moderator_password',
     news_moderation='newsgroup_moderation',
@@ -198,6 +201,14 @@ NAME_MAPPINGS = dict(
     send_welcome_msg='send_welcome_message',
     )
 
+# These DateTime fields of the mailinglist table need a type conversion to
+# Python datetime object for SQLite databases.
+DATETIME_COLUMNS = [
+    'created_at',
+    'digest_last_sent_at',
+    'last_post_time',
+    ]
+
 EXCLUDES = set((
     'digest_members',
     'members',
@@ -217,6 +228,9 @@ def import_config_pck(mlist, config_dict):
         # Some attributes must not be directly imported.
         if key in EXCLUDES:
             continue
+        # These objects need explicit type conversions.
+        if key in DATETIME_COLUMNS:
+            continue
         # Some attributes from Mailman 2 were renamed in Mailman 3.
         key = NAME_MAPPINGS.get(key, key)
         # Handle the simple case where the key is an attribute of the
@@ -238,6 +252,15 @@ def import_config_pck(mlist, config_dict):
             except (TypeError, KeyError):
                 print('Type conversion error for key "{}": {}'.format(
                     key, value), file=sys.stderr)
+    for key in DATETIME_COLUMNS:
+        try:
+            value = datetime.datetime.utcfromtimestamp(config_dict[key])
+        except KeyError:
+            continue
+        if key == 'last_post_time':
+            setattr(mlist, 'last_post_at', value)
+            continue
+        setattr(mlist, key, value)
     # Handle the archiving policy.  In MM2.1 there were two boolean options
     # but only three of the four possible states were valid.  Now there's just
     # an enum.
diff --git a/src/mailman/utilities/modules.py b/src/mailman/utilities/modules.py
index 5dfec95db..9ff0e50cd 100644
--- a/src/mailman/utilities/modules.py
+++ b/src/mailman/utilities/modules.py
@@ -22,6 +22,7 @@ from __future__ import absolute_import, print_function, unicode_literals
 __metaclass__ = type
 __all__ = [
     'call_name',
+    'expand_path',
     'find_components',
     'find_name',
     'scan_module',
@@ -31,7 +32,7 @@ __all__ = [
 import os
 import sys
 
-from pkg_resources import resource_listdir
+from pkg_resources import resource_filename, resource_listdir
 
 
 
@@ -110,3 +111,15 @@ def find_components(package, interface):
             continue
         for component in scan_module(module, interface):
             yield component
+
+
+
+def expand_path(url):
+    """Expand a python: path, returning the absolute file system path."""
+    # Is the context coming from a file system or Python path?
+    if url.startswith('python:'):
+        resource_path = url[7:]
+        package, dot, resource = resource_path.rpartition('.')
+        return resource_filename(package, resource + '.cfg')
+    else:
+        return url
diff --git a/src/mailman/utilities/tests/test_import.py b/src/mailman/utilities/tests/test_import.py
index 308fa8af2..e6eb9344c 100644
--- a/src/mailman/utilities/tests/test_import.py
+++ b/src/mailman/utilities/tests/test_import.py
@@ -34,6 +34,7 @@ import unittest
 from datetime import timedelta, datetime
 from enum import Enum
 from pkg_resources import resource_filename
+from sqlalchemy.exc import IntegrityError
 from zope.component import getUtility
 
 from mailman.app.lifecycle import create_list
@@ -291,6 +292,15 @@ class TestBasicImport(unittest.TestCase):
         else:
             self.fail('Import21Error was not raised')
 
+    def test_encode_ascii_prefixes(self):
+        self._pckdict['encode_ascii_prefixes'] = 2
+        self.assertEqual(self._mlist.encode_ascii_prefixes, False)
+        try:
+            self._import()
+        except IntegrityError as e:
+            self.fail(e)
+        self.assertEqual(self._mlist.encode_ascii_prefixes, True)
+
 
 
 class TestArchiveImport(unittest.TestCase):