Convert to the Storm Python ORM <storm.canonical.com>. There were several

reasons for this, but most importantly, the changes from SQLAlchemy/Elixir 0.3
to 0.4 were substantial and caused a lot of work.  This work unfortunately did
not result in a working branch due to very strange and inconsistent behavior
with Unicode columns.  Sometimes such columns would return Unicode, sometimes
8-bit strings, with no rhyme or reason.  I gave up debugging this after many
hours of head scratching.

Oh yeah, no more flush!

Storm enforces Unicode columns, which is very nice, though requires us to add
lots of 'u's in places we didn't have them before.  Ultimately, this is a good
thing so that the core of Mailman will be Unicode consistent.

One thing I still want to clean up after this, is the function-scoped imports
in the model code.  Part of the reason for the separate model classes was to
avoid this, but for now, we'll live with it.  Storm's architecture requires us
to maintain a database-table-class registry for simple clearing after tests
in Database._reset().  This is made fairly simple by Storm allowing us to use
our own metaclass for model classes.

Storm does require that we write our own SQL files, which is a downside, but I
think our schema will be easy enough that this won't be a huge burden.  Plus
we have a head-start <wink>.

Another cool thing about Storm is the explicit use of stores for objects.
This should eventually allow me to flesh out my idea of storage pillars for 1)
lists, 2) users, 3) messages.

Some other changes:

- pylint and pyflakes cleanups
- SQLALCHEMY_ENGINE_URL -> DEFAULT_DATABASE_URL
- Don't import-* from Version in Defaults.py
- Add interface method to Mailman.Message.Message so that __getitem__() and
  get_all() always return Unicode headers, even when the underlying objects
  are strings.  This should generally be safe as headers are required by RFC
  to be within the ASCII range.
- Fix bin/arch.py to use proper initialization.

1 files changed, 57 insertions, 28 deletions

diff --git a/Mailman/database/model/pending.py b/Mailman/database/model/pending.py
index 75bb59d3c..3f3e2caa0 100644
--- a/Mailman/database/model/pending.py
+++ b/Mailman/database/model/pending.py
@@ -17,40 +17,51 @@
 
 """Implementations of the IPendable and IPending interfaces."""
 
+import sys
 import time
 import random
 import hashlib
 import datetime
 
-from elixir import *
+from storm.locals import *
 from zope.interface import implements
 from zope.interface.verify import verifyObject
 
 from Mailman.configuration import config
+from Mailman.database import Model
 from Mailman.interfaces import (
-    IPendings, IPendable, IPendedKeyValue, IPended)
-
-PEND_KIND = 'Mailman.database.model.pending.Pended'
+    IPendable, IPended, IPendedKeyValue, IPendings)
 
 
 
-class PendedKeyValue(Entity):
+class PendedKeyValue(Model):
     """A pended key/value pair, tied to a token."""
 
     implements(IPendedKeyValue)
 
-    key = Field(Unicode)
-    value = Field(Unicode)
-    pended = ManyToOne(PEND_KIND)
+    def __init__(self, key, value):
+        self.key = key
+        self.value = value
+
+    id = Int(primary=True)
+    key = Unicode()
+    value = Unicode()
+    pended_id = Int()
 
 
-class Pended(Entity):
+class Pended(Model):
     """A pended event, tied to a token."""
 
     implements(IPended)
 
-    token = Field(Unicode)
-    expiration_date = Field(DateTime)
+    def __init__(self, token, expiration_date):
+        self.token = token
+        self.expiration_date = expiration_date
+
+    id = Int(primary=True)
+    token = RawStr()
+    expiration_date = DateTime()
+    key_values = ReferenceSet(id, PendedKeyValue.pended_id)
 
 
 
@@ -82,7 +93,7 @@ class Pendings(object):
             token = hashlib.sha1(repr(x)).hexdigest()
             # In practice, we'll never get a duplicate, but we'll be anal
             # about checking anyway.
-            if Pended.query.filter_by(token=token).count() == 0:
+            if config.db.store.find(Pended, token=token).count() == 0:
                 break
         else:
             raise AssertionError('Could not find a valid pendings token')
@@ -91,11 +102,24 @@ class Pendings(object):
             token=token,
             expiration_date=datetime.datetime.now() + lifetime)
         for key, value in pendable.items():
-            PendedKeyValue(key=key, value=value, pended=pending)
+            if isinstance(key, str):
+                key = unicode(key, 'utf-8')
+            if isinstance(value, str):
+                value = unicode(value, 'utf-8')
+            elif type(value) is int:
+                value = u'__builtin__.int\1%s' % value
+            elif type(value) is float:
+                value = u'__builtin__.float\1%s' % value
+            elif type(value) is bool:
+                value = u'__builtin__.bool\1%s' % value
+            keyval = PendedKeyValue(key=key, value=value)
+            pending.key_values.add(keyval)
+        config.db.store.add(pending)
         return token
 
     def confirm(self, token, expunge=True):
-        pendings = Pended.query.filter_by(token=token)
+        store = config.db.store
+        pendings = store.find(Pended, token=token)
         if pendings.count() == 0:
             return None
         assert pendings.count() == 1, (
@@ -103,27 +127,32 @@ class Pendings(object):
         pending = pendings[0]
         pendable = UnpendedPendable()
         # Find all PendedKeyValue entries that are associated with the pending
-        # object's ID.
-        q = PendedKeyValue.query.filter(
-            PendedKeyValue.c.pended_id == Pended.c.id).filter(
-            Pended.c.id == pending.id)
-        for keyvalue in q.all():
-            pendable[keyvalue.key] = keyvalue.value
+        # object's ID.  Watch out for type conversions.
+        for keyvalue in store.find(PendedKeyValue,
+                                   PendedKeyValue.pended_id == pending.id):
+            if keyvalue.value is not None and '\1' in keyvalue.value:
+                typename, value = keyvalue.value.split('\1', 1)
+                package, classname = typename.rsplit('.', 1)
+                __import__(package)
+                module = sys.modules[package]
+                pendable[keyvalue.key] = getattr(module, classname)(value)
+            else:
+                pendable[keyvalue.key] = keyvalue.value
             if expunge:
-                keyvalue.delete()
+                store.remove(keyvalue)
         if expunge:
-            pending.delete()
+            store.remove(pending)
         return pendable
 
     def evict(self):
+        store = config.db.store
         now = datetime.datetime.now()
-        for pending in Pended.query.filter_by().all():
+        for pending in store.find(Pended):
             if pending.expiration_date < now:
                 # Find all PendedKeyValue entries that are associated with the
                 # pending object's ID.
-                q = PendedKeyValue.query.filter(
-                    PendedKeyValue.c.pended_id == Pended.c.id).filter(
-                    Pended.c.id == pending.id)
+                q = store.find(PendedKeyValue,
+                               PendedKeyValue.pended_id == pending.id)
                 for keyvalue in q:
-                    keyvalue.delete()
-                pending.delete()
+                    store.remove(keyvalue)
+                store.remove(pending)