merging thread

5 files changed, 74 insertions, 18 deletions

diff --git a/mailman/bin/testall.py b/mailman/bin/testall.py
index 1037b28ef..9e9dd543d 100644
--- a/mailman/bin/testall.py
+++ b/mailman/bin/testall.py
@@ -24,6 +24,7 @@ import re
 import grp
 import pwd
 import sys
+import random
 import shutil
 import optparse
 import tempfile
@@ -74,6 +75,12 @@ Reduce verbosity by 1 (but not below 0)."""))
     parser.add_option('-c', '--coverage',
                       default=False, action='store_true',
                       help=_('Enable code coverage.'))
+    parser.add_option('-r', '--randomize',
+                      default=False, action='store_true',
+                      help=_("""\
+Randomize the tests; good for finding subtle dependency errors.  Note that
+this isn't completely random though because the doctests are not mixed with
+the Python tests.  Each type of test is randomized within its group."""))
     options, arguments = parser.parse_args()
     if len(arguments) == 0:
         arguments = ['.']
@@ -135,14 +142,19 @@ def filter_tests(suite, patterns):
     return new
 
 
-def suite(patterns=None):
+def suite(patterns, randomize):
     if patterns is None:
         patterns = '.'
     loader = unittest.TestLoader()
     # Search for all tests that match the given patterns
     testnames = search()
     suite = loader.loadTestsFromNames(testnames)
-    return filter_tests(suite, patterns)
+    tests = filter_tests(suite, patterns)
+    if randomize:
+        random.shuffle(tests._tests)
+    else:
+        tests._tests.sort()
+    return tests
 
 
 
@@ -153,7 +165,10 @@ def main():
 
     # Set verbosity level for test_documentation.py.  XXX There should be a
     # better way to do this.
-    config.verbosity = parser.options.verbosity
+    class Bag: pass
+    config.tests = Bag()
+    config.tests.verbosity = parser.options.verbosity
+    config.tests.randomize = parser.options.randomize
 
     # Turn on code coverage if selected.
     if parser.options.coverage:
@@ -245,7 +260,7 @@ def main():
         import mailman
         basedir = os.path.dirname(mailman.__file__)
         runner = unittest.TextTestRunner(verbosity=parser.options.verbosity)
-        results = runner.run(suite(parser.arguments))
+        results = runner.run(suite(parser.arguments, parser.options.randomize))
     finally:
         os.remove(cfg_out)
         os.remove(logging_cfg)
diff --git a/mailman/database/__init__.py b/mailman/database/__init__.py
index 1615c291f..1f73071c0 100644
--- a/mailman/database/__init__.py
+++ b/mailman/database/__init__.py
@@ -45,6 +45,8 @@ from mailman.interfaces import IDatabase, SchemaVersionMismatchError
 
 
 class StockDatabase:
+    """The standard database, using Storm on top of SQLite."""
+
     implements(IDatabase)
 
     def __init__(self):
@@ -56,6 +58,7 @@ class StockDatabase:
         self._store = None
 
     def initialize(self, debug=None):
+        """See `IDatabase`."""
         # Serialize this so we don't get multiple processes trying to create
         # the database at the same time.
         with Lock(os.path.join(config.LOCK_DIR, 'dbcreate.lck')):
@@ -66,6 +69,19 @@ class StockDatabase:
         self.pendings = Pendings()
         self.requests = Requests()
 
+    def begin(self):
+        """See `IDatabase`."""
+        # Storm takes care of this for us.
+        pass
+
+    def commit(self):
+        """See `IDatabase`."""
+        self.store.commit()
+
+    def abort(self):
+        """See `IDatabase`."""
+        self.store.rollback()
+
     def _create(self, debug):
         # Calculate the engine url.
         url = Template(config.DEFAULT_DATABASE_URL).safe_substitute(
@@ -105,7 +121,6 @@ class StockDatabase:
                 sql = fp.read()
             for statement in sql.split(';'):
                 store.execute(statement + ';')
-            store.commit()
         # Validate schema version.
         v = store.find(Version, component=u'schema').one()
         if not v:
@@ -117,8 +132,10 @@ class StockDatabase:
             # XXX Update schema
             raise SchemaVersionMismatchError(v.version)
         self.store = store
+        store.commit()
 
     def _reset(self):
+        """See `IDatabase`."""
         from mailman.database.model import ModelMeta
         self.store.rollback()
         ModelMeta._reset(self.store)
diff --git a/mailman/interfaces/database.py b/mailman/interfaces/database.py
index 0bacdaa3a..706613ba0 100644
--- a/mailman/interfaces/database.py
+++ b/mailman/interfaces/database.py
@@ -65,10 +65,14 @@ class IDatabase(Interface):
         This is only used by the test framework.
         """
 
-    # XXX Eventually we probably need to support a transaction manager
-    # interface, e.g. begin(), commit(), abort().  We will probably also need
-    # to support a shutdown() method for cleanly disconnecting from the
-    # database.sy
+    def begin():
+        """Begin the current transaction."""
+
+    def commit():
+        """Commit the current transaction."""
+
+    def abort():
+        """Abort the current transaction."""
 
     list_manager = Attribute(
         """The IListManager instance provided by the database layer.""")
diff --git a/mailman/queue/__init__.py b/mailman/queue/__init__.py
index fb6b07479..e30608138 100644
--- a/mailman/queue/__init__.py
+++ b/mailman/queue/__init__.py
@@ -274,6 +274,7 @@ class Runner:
                 log.error('Skipping and preserving unparseable message: %s',
                           filebase)
                 self._switchboard.finish(filebase, preserve=True)
+                config.db.abort()
                 continue
             try:
                 self._onefile(msg, msgdata)
@@ -303,11 +304,13 @@ class Runner:
                     log.error('SHUNTING FAILED, preserving original entry: %s',
                               filebase)
                     self._switchboard.finish(filebase, preserve=True)
-            # Other work we want to do each time through the loop
+                    config.db.abort()
+            # Other work we want to do each time through the loop.
             Utils.reap(self._kids, once=True)
             self._doperiodic()
             if self._shortcircuit():
                 break
+            config.db.commit()
         return len(files)
 
     def _onefile(self, msg, msgdata):
diff --git a/mailman/tests/test_documentation.py b/mailman/tests/test_documentation.py
index d11d4bd70..e805b10fa 100644
--- a/mailman/tests/test_documentation.py
+++ b/mailman/tests/test_documentation.py
@@ -18,6 +18,7 @@
 """Harness for testing Mailman's documentation."""
 
 import os
+import random
 import doctest
 import unittest
 
@@ -53,7 +54,12 @@ def specialized_message_from_string(text):
 
 def setup(testobj):
     """Test setup."""
+    # In general, I don't like adding convenience functions, since I think
+    # doctests should do the imports themselves.  It makes for better
+    # documentation that way.  However, a few are really useful, or help to
+    # hide some icky test implementation details.
     testobj.globs['message_from_string'] = specialized_message_from_string
+    testobj.globs['commit'] = config.db.commit
 
 
 
@@ -72,6 +78,7 @@ def cleaning_teardown(testobj):
     # Clear out messages in the message store.
     for message in config.db.message_store.messages:
         config.db.message_store.delete_message(message['message-id'])
+    config.db.commit()
 
 
 
@@ -88,17 +95,27 @@ def test_suite():
     flags = (doctest.ELLIPSIS |
              doctest.NORMALIZE_WHITESPACE |
              doctest.REPORT_NDIFF)
-    if config.verbosity <= 2:
+    if config.tests.verbosity <= 2:
         flags |= doctest.REPORT_ONLY_FIRST_FAILURE
     # Add all the doctests in all subpackages.
+    doctest_files = {}
     for docsdir in packages:
         for filename in os.listdir(os.path.join('mailman', docsdir)):
             if os.path.splitext(filename)[1] == '.txt':
-                test = doctest.DocFileSuite(
-                    os.path.join(docsdir, filename),
-                    package='mailman',
-                    optionflags=flags,
-                    setUp=setup,
-                    tearDown=cleaning_teardown)
-                suite.addTest(test)
+                doctest_files[filename] = os.path.join(docsdir, filename)
+    # Sort or randomize the tests.
+    if config.tests.randomize:
+        files = doctest_files.keys()
+        random.shuffle(files)
+    else:
+        files = sorted(doctest_files)
+    for filename in files:
+        path = doctest_files[filename]
+        test = doctest.DocFileSuite(
+            path,
+            package='mailman',
+            optionflags=flags,
+            setUp=setup,
+            tearDown=cleaning_teardown)
+        suite.addTest(test)
     return suite