From 5598b9eea196e4085aa91aaf8a0cacaffa200355 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Mon, 23 Jul 2012 10:40:53 -0400 Subject: Checkpointing Postgres port of test suite. - Refactor load_schema() into a separate load_sql() method. - Add API for test suite to make a temporary database. - Add code to migrate the PostgreSQL database. - Comment out `moderation_callback` from the PostgreSQL SQL; this must have snuck in accidentally via the contributed port. - Refactor test_migrations.py --- src/mailman/database/postgresql.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) (limited to 'src/mailman/database/postgresql.py') diff --git a/src/mailman/database/postgresql.py b/src/mailman/database/postgresql.py index 988f7a1af..14855c3f3 100644 --- a/src/mailman/database/postgresql.py +++ b/src/mailman/database/postgresql.py @@ -26,10 +26,23 @@ __all__ = [ from operator import attrgetter +from urlparse import urlsplit, urlunsplit +from mailman.config import config from mailman.database.base import StormBaseDatabase + + +class _TemporaryDB: + def __init__(self, database): + self.database = database + + def cleanup(self): + self.database.execute('ROLLBACK TO SAVEPOINT testing;') + self.database.execute('DROP DATABASE mmtest;') + + class PostgreSQLDatabase(StormBaseDatabase): """Database class for PostgreSQL.""" @@ -40,8 +53,8 @@ class PostgreSQLDatabase(StormBaseDatabase): """See `BaseDatabase`.""" table_query = ('SELECT table_name FROM information_schema.tables ' "WHERE table_schema = 'public'") - table_names = set(item[0] for item in - store.execute(table_query)) + results = store.execute(table_query) + table_names = set(item[0] for item in results) return 'version' in table_names def _post_reset(self, store): @@ -63,3 +76,18 @@ class PostgreSQLDatabase(StormBaseDatabase): max("id") IS NOT null) FROM "{0}"; """.format(model_class.__storm_table__)) + + @staticmethod + def _make_temporary(): + from mailman.testing.helpers import configuration + parts = urlsplit(config.database.url) + assert parts.scheme == 'postgres' + new_parts = list(parts) + new_parts[2] = '/mmtest' + url = urlunsplit(new_parts) + database = PostgreSQLDatabase() + database.store.execute('SAVEPOINT testing;') + database.store.execute('CREATE DATABASE mmtest;') + with configuration('database', url=url): + database.initialize() + return _TemporaryDB(database) -- cgit v1.2.3-70-g09d2 From 9ef8a1268f1b2902ad46852937dd7bc977c5b2b1 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Wed, 25 Jul 2012 12:23:29 -0400 Subject: Very nearly there with PostgreSQL support for testing the beta2 migration. - Improve migration logging - Disable pre_reset() and post_reset() on migrations, which might need to be re-enabled for SQLite support. - Be sure to record the migration version in PostgreSQL. - Parameterize the Error that will occur. - Better sample data for PostgreSQL and SQLite, which have different formats. --- src/mailman/database/base.py | 10 +- src/mailman/database/postgresql.py | 16 ++- .../database/schema/mm_00000000000000_base.py | 12 +- src/mailman/database/schema/mm_20120407000000.py | 14 ++- src/mailman/database/sqlite.py | 2 + .../database/tests/data/migration_postgres_1.sql | 133 ++++++++++++++++++++ .../database/tests/data/migration_sqlite_1.sql | 133 ++++++++++++++++++++ .../database/tests/data/migration_test_1.sql | 135 --------------------- src/mailman/database/tests/test_migrations.py | 74 ++++++----- 9 files changed, 336 insertions(+), 193 deletions(-) create mode 100644 src/mailman/database/tests/data/migration_postgres_1.sql create mode 100644 src/mailman/database/tests/data/migration_sqlite_1.sql delete mode 100644 src/mailman/database/tests/data/migration_test_1.sql (limited to 'src/mailman/database/postgresql.py') diff --git a/src/mailman/database/base.py b/src/mailman/database/base.py index f674c9a16..af035914b 100644 --- a/src/mailman/database/base.py +++ b/src/mailman/database/base.py @@ -172,9 +172,12 @@ class StormBaseDatabase: parts = module_fn.split('_') if len(parts) < 2: continue - version = parts[1] - if len(version.strip()) == 0 or version in versions: - # This one is already loaded. + version = parts[1].strip() + if len(version) == 0: + # Not a schema migration file. + continue + if version in versions: + log.debug('already migrated to %s', version) continue if until is not None and version > until: # We're done. @@ -184,6 +187,7 @@ class StormBaseDatabase: upgrade = getattr(sys.modules[module_path], 'upgrade', None) if upgrade is None: continue + log.debug('migrating db to %s: %s', version, module_path) upgrade(self, self.store, version, module_path) def load_sql(self, store, sql): diff --git a/src/mailman/database/postgresql.py b/src/mailman/database/postgresql.py index 14855c3f3..879594459 100644 --- a/src/mailman/database/postgresql.py +++ b/src/mailman/database/postgresql.py @@ -25,6 +25,8 @@ __all__ = [ ] +import psycopg2 + from operator import attrgetter from urlparse import urlsplit, urlunsplit @@ -39,8 +41,9 @@ class _TemporaryDB: self.database = database def cleanup(self): - self.database.execute('ROLLBACK TO SAVEPOINT testing;') - self.database.execute('DROP DATABASE mmtest;') + self.database.store.execute('ABORT;') + self.database.store.close() + config.db.store.execute('DROP DATABASE mmtest;') @@ -48,6 +51,7 @@ class PostgreSQLDatabase(StormBaseDatabase): """Database class for PostgreSQL.""" TAG = 'postgres' + Error = psycopg2.ProgrammingError def _database_exists(self, store): """See `BaseDatabase`.""" @@ -85,9 +89,13 @@ class PostgreSQLDatabase(StormBaseDatabase): new_parts = list(parts) new_parts[2] = '/mmtest' url = urlunsplit(new_parts) + # Use the existing database connection to create a new testing + # database. Create a savepoint, which will make it easy to reset + # after the test. + config.db.store.execute('ABORT;') + config.db.store.execute('CREATE DATABASE mmtest;') + # Now create a new, temporary database. database = PostgreSQLDatabase() - database.store.execute('SAVEPOINT testing;') - database.store.execute('CREATE DATABASE mmtest;') with configuration('database', url=url): database.initialize() return _TemporaryDB(database) diff --git a/src/mailman/database/schema/mm_00000000000000_base.py b/src/mailman/database/schema/mm_00000000000000_base.py index ef448d620..f98b30a8d 100644 --- a/src/mailman/database/schema/mm_00000000000000_base.py +++ b/src/mailman/database/schema/mm_00000000000000_base.py @@ -38,11 +38,11 @@ def upgrade(database, store, version, module_path): -def pre_reset(store): - global _helper - from mailman.testing.database import ResetHelper - _helper = ResetHelper(VERSION, store) +## def pre_reset(store): +## global _helper +## from mailman.testing.database import ResetHelper +## _helper = ResetHelper(VERSION, store) -def post_reset(store): - _helper.restore(store) +## def post_reset(store): +## _helper.restore(store) diff --git a/src/mailman/database/schema/mm_20120407000000.py b/src/mailman/database/schema/mm_20120407000000.py index ad0515a5d..017141683 100644 --- a/src/mailman/database/schema/mm_20120407000000.py +++ b/src/mailman/database/schema/mm_20120407000000.py @@ -125,14 +125,16 @@ def upgrade_postgres(database, store, version, module_path): # Now drop the old columns. store.execute('ALTER TABLE mailinglist DROP COLUMN archive;') store.execute('ALTER TABLE mailinglist DROP COLUMN archive_private;') + # Record the migration in the version table. + database.load_schema(store, version, None, module_path) -def pre_reset(store): - global _helper - from mailman.testing.database import ResetHelper - _helper = ResetHelper(VERSION, store) +## def pre_reset(store): +## global _helper +## from mailman.testing.database import ResetHelper +## _helper = ResetHelper(VERSION, store) -def post_reset(store): - _helper.restore(store) +## def post_reset(store): +## _helper.restore(store) diff --git a/src/mailman/database/sqlite.py b/src/mailman/database/sqlite.py index 4e5df55a5..39e7e8113 100644 --- a/src/mailman/database/sqlite.py +++ b/src/mailman/database/sqlite.py @@ -27,6 +27,7 @@ __all__ = [ import os import shutil +import sqlite3 import tempfile from urlparse import urlparse @@ -49,6 +50,7 @@ class SQLiteDatabase(StormBaseDatabase): """Database class for SQLite.""" TAG = 'sqlite' + Error = sqlite3.OperationalError def _database_exists(self, store): """See `BaseDatabase`.""" diff --git a/src/mailman/database/tests/data/migration_postgres_1.sql b/src/mailman/database/tests/data/migration_postgres_1.sql new file mode 100644 index 000000000..f144367d0 --- /dev/null +++ b/src/mailman/database/tests/data/migration_postgres_1.sql @@ -0,0 +1,133 @@ +INSERT INTO "acceptablealias" VALUES(1,'foo@example.com',1); +INSERT INTO "acceptablealias" VALUES(2,'bar@example.com',1); + +INSERT INTO "address" VALUES( + 1,'anne@example.com',NULL,'Anne Person', + '2012-04-19 00:52:24.826432','2012-04-19 00:49:42.373769',1,2); +INSERT INTO "address" VALUES( + 2,'bart@example.com',NULL,'Bart Person', + '2012-04-19 00:53:25.878800','2012-04-19 00:49:52.882050',2,4); + +INSERT INTO "domain" VALUES( + 1,'example.com','http://example.com',NULL,'postmaster@example.com'); + +INSERT INTO "mailinglist" VALUES( + -- id,list_name,mail_host,include_list_post_header,include_rfc2369_headers + 1,'test','example.com',True,True, + -- created_at,admin_member_chunksize,next_request_id,next_digest_number + '2012-04-19 00:46:13.173844',30,1,1, + -- digest_last_sent_at,volume,last_post_at,accept_these_nonmembers + NULL,1,NULL,E'\\x80025D71012E', + -- acceptable_aliases_id,admin_immed_notify,admin_notify_mchanges + NULL,True,False, + -- administrivia,advertised,anonymous_list,archive,archive_private + True,True,False,True,False, + -- archive_volume_frequency + 1, + --autorespond_owner,autoresponse_owner_text + 0,'', + -- autorespond_postings,autoresponse_postings_text + 0,'', + -- autorespond_requests,authoresponse_requests_text + 0,'', + -- autoresponse_grace_period + '90 days, 0:00:00', + -- forward_unrecognized_bounces_to,process_bounces + 1,True, + -- bounce_info_stale_after,bounce_matching_headers + '7 days, 0:00:00',' +# Lines that *start* with a ''#'' are comments. +to: friend@public.com +message-id: relay.comanche.denmark.eu +from: list@listme.com +from: .*@uplinkpro.com +', + -- bounce_notify_owner_on_disable,bounce_notify_owner_on_removal + True,True, + -- bounce_score_threshold,bounce_you_are_disabled_warnings + 5,3, + -- bounce_you_are_disabled_warnings_interval + '7 days, 0:00:00', + -- filter_action,filter_content,collapse_alternatives + 2,False,True, + -- convert_html_to_plaintext,default_member_action,default_nonmember_action + False,4,0, + -- description + '', + -- digest_footer_uri + 'mailman:///$listname/$language/footer-generic.txt', + -- digest_header_uri + NULL, + -- digest_is_default,digest_send_periodic,digest_size_threshold + False,True,30.0, + -- digest_volume_frequency,digestable,discard_these_nonmembers + 1,True,E'\\x80025D71012E', + -- emergency,encode_ascii_prefixes,first_strip_reply_to + False,False,False, + -- footer_uri + 'mailman:///$listname/$language/footer-generic.txt', + -- forward_auto_discards,gateway_to_mail,gateway_to_news + True,False,FAlse, + -- generic_nonmember_action,goodby_message_uri + 1,'', + -- header_matches,header_uri,hold_these_nonmembers,info,linked_newsgroup + E'\\x80025D71012E',NULL,E'\\x80025D71012E','','', + -- max_days_to_hold,max_message_size,max_num_recipients + 0,40,10, + -- member_moderation_notice,mime_is_default_digest,moderator_password + '',False,NULL, + -- new_member_options,news_moderation,news_prefix_subject_too + 256,0,True, + -- nntp_host,nondigestable,nonmember_rejection_notice,obscure_addresses + '',True,'',True, + -- owner_chain,owner_pipeline,personalize,post_id + 'default-owner-chain','default-owner-pipeline',0,1, + -- posting_chain,posting_pipeline,preferred_language,private_roster + 'default-posting-chain','default-posting-pipeline','en',True, + -- display_name,reject_these_nonmembers + 'Test',E'\\x80025D71012E', + -- reply_goes_to_list,reply_to_address + 0,'', + -- require_explicit_destination,respond_to_post_requests + True,True, + -- scrub_nondigest,send_goodbye_message,send_reminders,send_welcome_message + False,True,True,True, + -- subject_prefix,subscribe_auto_approval + '[Test] ',E'\\x80025D71012E', + -- subscribe_policy,topics,topics_bodylines_limit,topics_enabled + 1,E'\\x80025D71012E',5,False, + -- unsubscribe_policy,welcome_message_uri + 0,'mailman:///welcome.txt'); + +INSERT INTO "member" VALUES( + 1,'d1243f4d-e604-4f6b-af52-98d0a7bce0f1',1,'test@example.com',4,NULL,5,1); +INSERT INTO "member" VALUES( + 2,'dccc3851-fdfb-4afa-90cf-bdcbf80ad0fd',2,'test@example.com',3,NULL,6,1); +INSERT INTO "member" VALUES( + 3,'479be431-45f2-473d-bc3c-7eac614030ac',3,'test@example.com',3,NULL,7,2); +INSERT INTO "member" VALUES( + 4,'e2dc604c-d93a-4b91-b5a8-749e3caade36',1,'test@example.com',4,NULL,8,2); + +INSERT INTO "preferences" VALUES(1,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(2,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(3,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(4,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(5,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(6,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(7,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(8,NULL,NULL,NULL,NULL,NULL,NULL,NULL); + +INSERT INTO "user" VALUES( + 1,'Anne Person',NULL,'0adf3caa-6f26-46f8-a11d-5256c8148592', + '2012-04-19 00:49:42.370493',1,1); +INSERT INTO "user" VALUES( + 2,'Bart Person',NULL,'63f5d1a2-e533-4055-afe4-475dec3b1163', + '2012-04-19 00:49:52.868746',2,3); + +INSERT INTO "uid" VALUES(1,'8bf9a615-f23e-4980-b7d1-90ac0203c66f'); +INSERT INTO "uid" VALUES(2,'0adf3caa-6f26-46f8-a11d-5256c8148592'); +INSERT INTO "uid" VALUES(3,'63f5d1a2-e533-4055-afe4-475dec3b1163'); +INSERT INTO "uid" VALUES(4,'d1243f4d-e604-4f6b-af52-98d0a7bce0f1'); +INSERT INTO "uid" VALUES(5,'dccc3851-fdfb-4afa-90cf-bdcbf80ad0fd'); +INSERT INTO "uid" VALUES(6,'479be431-45f2-473d-bc3c-7eac614030ac'); +INSERT INTO "uid" VALUES(7,'e2dc604c-d93a-4b91-b5a8-749e3caade36'); diff --git a/src/mailman/database/tests/data/migration_sqlite_1.sql b/src/mailman/database/tests/data/migration_sqlite_1.sql new file mode 100644 index 000000000..c8810b130 --- /dev/null +++ b/src/mailman/database/tests/data/migration_sqlite_1.sql @@ -0,0 +1,133 @@ +INSERT INTO "acceptablealias" VALUES(1,'foo@example.com',1); +INSERT INTO "acceptablealias" VALUES(2,'bar@example.com',1); + +INSERT INTO "address" VALUES( + 1,'anne@example.com',NULL,'Anne Person', + '2012-04-19 00:52:24.826432','2012-04-19 00:49:42.373769',1,2); +INSERT INTO "address" VALUES( + 2,'bart@example.com',NULL,'Bart Person', + '2012-04-19 00:53:25.878800','2012-04-19 00:49:52.882050',2,4); + +INSERT INTO "domain" VALUES( + 1,'example.com','http://example.com',NULL,'postmaster@example.com'); + +INSERT INTO "mailinglist" VALUES( + -- id,list_name,mail_host,include_list_post_header,include_rfc2369_headers + 1,'test','example.com',1,1, + -- created_at,admin_member_chunksize,next_request_id,next_digest_number + '2012-04-19 00:46:13.173844',30,1,1, + -- digest_last_sent_at,volume,last_post_at,accept_these_nonmembers + NULL,1,NULL,X'80025D71012E', + -- acceptable_aliases_id,admin_immed_notify,admin_notify_mchanges + NULL,1,0, + -- administrivia,advertised,anonymous_list,archive,archive_private + 1,1,0,1,0, + -- archive_volume_frequency + 1, + --autorespond_owner,autoresponse_owner_text + 0,'', + -- autorespond_postings,autoresponse_postings_text + 0,'', + -- autorespond_requests,authoresponse_requests_text + 0,'', + -- autoresponse_grace_period + '90 days, 0:00:00', + -- forward_unrecognized_bounces_to,process_bounces + 1,1, + -- bounce_info_stale_after,bounce_matching_headers + '7 days, 0:00:00',' +# Lines that *start* with a ''#'' are comments. +to: friend@public.com +message-id: relay.comanche.denmark.eu +from: list@listme.com +from: .*@uplinkpro.com +', + -- bounce_notify_owner_on_disable,bounce_notify_owner_on_removal + 1,1, + -- bounce_score_threshold,bounce_you_are_disabled_warnings + 5,3, + -- bounce_you_are_disabled_warnings_interval + '7 days, 0:00:00', + -- filter_action,filter_content,collapse_alternatives + 2,0,1, + -- convert_html_to_plaintext,default_member_action,default_nonmember_action + 0,4,0, + -- description + '', + -- digest_footer_uri + 'mailman:///$listname/$language/footer-generic.txt', + -- digest_header_uri + NULL, + -- digest_is_default,digest_send_periodic,digest_size_threshold + 0,1,30.0, + -- digest_volume_frequency,digestable,discard_these_nonmembers + 1,1,X'80025D71012E', + -- emergency,encode_ascii_prefixes,first_strip_reply_to + 0,0,0, + -- footer_uri + 'mailman:///$listname/$language/footer-generic.txt', + -- forward_auto_discards,gateway_to_mail,gateway_to_news + 1,0,0, + -- generic_nonmember_action,goodby_message_uri + 1,'', + -- header_matches,header_uri,hold_these_nonmembers,info,linked_newsgroup + X'80025D71012E',NULL,X'80025D71012E','','', + -- max_days_to_hold,max_message_size,max_num_recipients + 0,40,10, + -- member_moderation_notice,mime_is_default_digest,moderator_password + '',0,NULL, + -- new_member_options,news_moderation,news_prefix_subject_too + 256,0,1, + -- nntp_host,nondigestable,nonmember_rejection_notice,obscure_addresses + '',1,'',1, + -- owner_chain,owner_pipeline,personalize,post_id + 'default-owner-chain','default-owner-pipeline',0,1, + -- posting_chain,posting_pipeline,preferred_language,private_roster + 'default-posting-chain','default-posting-pipeline','en',1, + -- display_name,reject_these_nonmembers + 'Test',X'80025D71012E', + -- reply_goes_to_list,reply_to_address + 0,'', + -- require_explicit_destination,respond_to_post_requests + 1,1, + -- scrub_nondigest,send_goodbye_message,send_reminders,send_welcome_message + 0,1,1,1, + -- subject_prefix,subscribe_auto_approval + '[Test] ',X'80025D71012E', + -- subscribe_policy,topics,topics_bodylines_limit,topics_enabled + 1,X'80025D71012E',5,0, + -- unsubscribe_policy,welcome_message_uri + 0,'mailman:///welcome.txt'); + +INSERT INTO "member" VALUES( + 1,'d1243f4d-e604-4f6b-af52-98d0a7bce0f1',1,'test@example.com',4,NULL,5,1); +INSERT INTO "member" VALUES( + 2,'dccc3851-fdfb-4afa-90cf-bdcbf80ad0fd',2,'test@example.com',3,NULL,6,1); +INSERT INTO "member" VALUES( + 3,'479be431-45f2-473d-bc3c-7eac614030ac',3,'test@example.com',3,NULL,7,2); +INSERT INTO "member" VALUES( + 4,'e2dc604c-d93a-4b91-b5a8-749e3caade36',1,'test@example.com',4,NULL,8,2); + +INSERT INTO "preferences" VALUES(1,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(2,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(3,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(4,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(5,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(6,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(7,NULL,NULL,NULL,NULL,NULL,NULL,NULL); +INSERT INTO "preferences" VALUES(8,NULL,NULL,NULL,NULL,NULL,NULL,NULL); + +INSERT INTO "user" VALUES( + 1,'Anne Person',NULL,'0adf3caa-6f26-46f8-a11d-5256c8148592', + '2012-04-19 00:49:42.370493',1,1); +INSERT INTO "user" VALUES( + 2,'Bart Person',NULL,'63f5d1a2-e533-4055-afe4-475dec3b1163', + '2012-04-19 00:49:52.868746',2,3); + +INSERT INTO "uid" VALUES(1,'8bf9a615-f23e-4980-b7d1-90ac0203c66f'); +INSERT INTO "uid" VALUES(2,'0adf3caa-6f26-46f8-a11d-5256c8148592'); +INSERT INTO "uid" VALUES(3,'63f5d1a2-e533-4055-afe4-475dec3b1163'); +INSERT INTO "uid" VALUES(4,'d1243f4d-e604-4f6b-af52-98d0a7bce0f1'); +INSERT INTO "uid" VALUES(5,'dccc3851-fdfb-4afa-90cf-bdcbf80ad0fd'); +INSERT INTO "uid" VALUES(6,'479be431-45f2-473d-bc3c-7eac614030ac'); +INSERT INTO "uid" VALUES(7,'e2dc604c-d93a-4b91-b5a8-749e3caade36'); diff --git a/src/mailman/database/tests/data/migration_test_1.sql b/src/mailman/database/tests/data/migration_test_1.sql deleted file mode 100644 index 101feea77..000000000 --- a/src/mailman/database/tests/data/migration_test_1.sql +++ /dev/null @@ -1,135 +0,0 @@ -INSERT INTO "acceptablealias" VALUES(1,'foo@example.com',1); -INSERT INTO "acceptablealias" VALUES(2,'bar@example.com',1); - -INSERT INTO "address" VALUES( - 1,'anne@example.com',NULL,'Anne Person', - '2012-04-19 00:52:24.826432','2012-04-19 00:49:42.373769',1,2); -INSERT INTO "address" VALUES( - 2,'bart@example.com',NULL,'Bart Person', - '2012-04-19 00:53:25.878800','2012-04-19 00:49:52.882050',2,4); - --- ConfigLayer.testSetUp() will already initialize the domain. --- --- INSERT INTO "domain" VALUES( --- 1,'example.com','http://example.com',NULL,'postmaster@example.com'); - -INSERT INTO "mailinglist" VALUES( - -- id,list_name,mail_host,include_list_post_header,include_rfc2369_headers - 1,'test','example.com',1,1, - -- created_at,admin_member_chunksize,next_request_id,next_digest_number - '2012-04-19 00:46:13.173844',30,1,1, - -- digest_last_sent_at,volume,last_post_at,accept_these_nonmembers - NULL,1,NULL,X'80025D71012E', - -- acceptable_aliases_id,admin_immed_notify,admin_notify_mchanges - NULL,1,0, - -- administrivia,advertised,anonymous_list,archive,archive_private - 1,1,0,1,0, - -- archive_volume_frequency - 1, - --autorespond_owner,autoresponse_owner_text - 0,'', - -- autorespond_postings,autoresponse_postings_text - 0,'', - -- autorespond_requests,authoresponse_requests_text - 0,'', - -- autoresponse_grace_period - '90 days, 0:00:00', - -- forward_unrecognized_bounces_to,process_bounces - 1,1, - -- bounce_info_stale_after,bounce_matching_headers - '7 days, 0:00:00',' -# Lines that *start* with a ''#'' are comments. -to: friend@public.com -message-id: relay.comanche.denmark.eu -from: list@listme.com -from: .*@uplinkpro.com -', - -- bounce_notify_owner_on_disable,bounce_notify_owner_on_removal - 1,1, - -- bounce_score_threshold,bounce_you_are_disabled_warnings - 5,3, - -- bounce_you_are_disabled_warnings_interval - '7 days, 0:00:00', - -- filter_action,filter_content,collapse_alternatives - 2,0,1, - -- convert_html_to_plaintext,default_member_action,default_nonmember_action - 0,4,0, - -- description - '', - -- digest_footer_uri - 'mailman:///$listname/$language/footer-generic.txt', - -- digest_header_uri - NULL, - -- digest_is_default,digest_send_periodic,digest_size_threshold - 0,1,30.0, - -- digest_volume_frequency,digestable,discard_these_nonmembers - 1,1,X'80025D71012E', - -- emergency,encode_ascii_prefixes,first_strip_reply_to - 0,0,0, - -- footer_uri - 'mailman:///$listname/$language/footer-generic.txt', - -- forward_auto_discards,gateway_to_mail,gateway_to_news - 1,0,0, - -- generic_nonmember_action,goodby_message_uri - 1,'', - -- header_matches,header_uri,hold_these_nonmembers,info,linked_newsgroup - X'80025D71012E',NULL,X'80025D71012E','','', - -- max_days_to_hold,max_message_size,max_num_recipients - 0,40,10, - -- member_moderation_notice,mime_is_default_digest,moderator_password - '',0,NULL, - -- new_member_options,news_moderation,news_prefix_subject_too - 256,0,1, - -- nntp_host,nondigestable,nonmember_rejection_notice,obscure_addresses - '',1,'',1, - -- owner_chain,owner_pipeline,personalize,post_id - 'default-owner-chain','default-owner-pipeline',0,1, - -- posting_chain,posting_pipeline,preferred_language,private_roster - 'default-posting-chain','default-posting-pipeline','en',1, - -- display_name,reject_these_nonmembers - 'Test',X'80025D71012E', - -- reply_goes_to_list,reply_to_address - 0,'', - -- require_explicit_destination,respond_to_post_requests - 1,1, - -- scrub_nondigest,send_goodbye_message,send_reminders,send_welcome_message - 0,1,1,1, - -- subject_prefix,subscribe_auto_approval - '[Test] ',X'80025D71012E', - -- subscribe_policy,topics,topics_bodylines_limit,topics_enabled - 1,X'80025D71012E',5,0, - -- unsubscribe_policy,welcome_message_uri - 0,'mailman:///welcome.txt'); - -INSERT INTO "member" VALUES( - 1,'d1243f4d-e604-4f6b-af52-98d0a7bce0f1',1,'test@example.com',4,NULL,5,1); -INSERT INTO "member" VALUES( - 2,'dccc3851-fdfb-4afa-90cf-bdcbf80ad0fd',2,'test@example.com',3,NULL,6,1); -INSERT INTO "member" VALUES( - 3,'479be431-45f2-473d-bc3c-7eac614030ac',3,'test@example.com',3,NULL,7,2); -INSERT INTO "member" VALUES( - 4,'e2dc604c-d93a-4b91-b5a8-749e3caade36',1,'test@example.com',4,NULL,8,2); - -INSERT INTO "preferences" VALUES(1,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO "preferences" VALUES(2,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO "preferences" VALUES(3,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO "preferences" VALUES(4,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO "preferences" VALUES(5,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO "preferences" VALUES(6,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO "preferences" VALUES(7,NULL,NULL,NULL,NULL,NULL,NULL,NULL); -INSERT INTO "preferences" VALUES(8,NULL,NULL,NULL,NULL,NULL,NULL,NULL); - -INSERT INTO "user" VALUES( - 1,'Anne Person',NULL,'0adf3caa-6f26-46f8-a11d-5256c8148592', - '2012-04-19 00:49:42.370493',1,1); -INSERT INTO "user" VALUES( - 2,'Bart Person',NULL,'63f5d1a2-e533-4055-afe4-475dec3b1163', - '2012-04-19 00:49:52.868746',2,3); - -INSERT INTO "uid" VALUES(1,'8bf9a615-f23e-4980-b7d1-90ac0203c66f'); -INSERT INTO "uid" VALUES(2,'0adf3caa-6f26-46f8-a11d-5256c8148592'); -INSERT INTO "uid" VALUES(3,'63f5d1a2-e533-4055-afe4-475dec3b1163'); -INSERT INTO "uid" VALUES(4,'d1243f4d-e604-4f6b-af52-98d0a7bce0f1'); -INSERT INTO "uid" VALUES(5,'dccc3851-fdfb-4afa-90cf-bdcbf80ad0fd'); -INSERT INTO "uid" VALUES(6,'479be431-45f2-473d-bc3c-7eac614030ac'); -INSERT INTO "uid" VALUES(7,'e2dc604c-d93a-4b91-b5a8-749e3caade36'); diff --git a/src/mailman/database/tests/test_migrations.py b/src/mailman/database/tests/test_migrations.py index 0e6edae47..90f2a9dfe 100644 --- a/src/mailman/database/tests/test_migrations.py +++ b/src/mailman/database/tests/test_migrations.py @@ -27,10 +27,6 @@ __all__ = [ ] -import os -import shutil -import sqlite3 -import tempfile import unittest from pkg_resources import resource_string @@ -41,7 +37,7 @@ from mailman.interfaces.domain import IDomainManager from mailman.interfaces.archiver import ArchivePolicy from mailman.interfaces.listmanager import IListManager from mailman.interfaces.mailinglist import IAcceptableAliasSet -from mailman.testing.helpers import configuration, temporary_db +from mailman.testing.helpers import temporary_db from mailman.testing.layers import ConfigLayer from mailman.utilities.modules import call_name @@ -73,7 +69,7 @@ class TestMigration20120407Schema(unittest.TestCase): def tearDown(self): self._temporary.cleanup() - def test_sqlite_base(self): + def test_pre_upgrade_columns_base(self): # Test that before the migration, the old table columns are present # and the new database columns are not. # @@ -82,9 +78,12 @@ class TestMigration20120407Schema(unittest.TestCase): # Verify that the database has not yet been migrated. for missing in ('archive_policy', 'nntp_prefix_subject_too'): - self.assertRaises(sqlite3.OperationalError, + self.assertRaises(self._database.Error, self._database.store.execute, 'select {0} from mailinglist;'.format(missing)) + # Avoid PostgreSQL complaint: InternalError: current transaction + # is aborted, commands ignored until end of transaction block. + self._database.store.execute('ABORT;') for present in ('archive', 'archive_private', 'archive_volume_frequency', @@ -96,7 +95,7 @@ class TestMigration20120407Schema(unittest.TestCase): self._database.store.execute( 'select {0} from mailinglist;'.format(present)) - def test_sqlite_migration(self): + def test_post_upgrade_columns_migration(self): # Test that after the migration, the old table columns are missing # and the new database columns are present. # @@ -116,9 +115,12 @@ class TestMigration20120407Schema(unittest.TestCase): 'news_moderation', 'news_prefix_subject_too', 'nntp_host'): - self.assertRaises(sqlite3.OperationalError, + self.assertRaises(self._database.Error, self._database.store.execute, 'select {0} from mailinglist;'.format(missing)) + # Avoid PostgreSQL complaint: InternalError: current transaction + # is aborted, commands ignored until end of transaction block. + self._database.store.execute('ABORT;') @@ -148,7 +150,8 @@ class TestMigration20120407Data(unittest.TestCase): self._database.load_migrations('20120406999999') # Load the previous schema's sample data. sample_data = resource_string( - 'mailman.database.tests.data', 'migration_test_1.sql') + 'mailman.database.tests.data', + 'migration_{0}_1.sql'.format(database_class.TAG)) self._database.load_sql(self._database.store, sample_data) # Update to the current migration we're testing. self._database.load_migrations('20120407000000') @@ -230,7 +233,8 @@ class TestMigration20120407ArchiveData(unittest.TestCase): self._database.load_migrations('20120406999999') # Load the previous schema's sample data. sample_data = resource_string( - 'mailman.database.tests.data', 'migration_test_1.sql') + 'mailman.database.tests.data', + 'migration_{0}_1.sql'.format(database_class.TAG)) self._database.load_sql(self._database.store, sample_data) def _upgrade(self): @@ -244,13 +248,11 @@ class TestMigration20120407ArchiveData(unittest.TestCase): # Test that the new archive_policy value is updated correctly. In the # case of old column archive=0, the archive_private column is # ignored. This test sets it to 0 to ensure it's ignored. - with configuration('database', url=self._url): - # Set the old archive values. - self._database.store.execute( - 'UPDATE mailinglist SET archive = 0, archive_private = 0 ' - 'WHERE id = 1;') - # Complete the migration - self._upgrade() + self._database.store.execute( + 'UPDATE mailinglist SET archive = False, archive_private = False ' + 'WHERE id = 1;') + # Complete the migration + self._upgrade() with temporary_db(self._database): mlist = getUtility(IListManager).get('test@example.com') self.assertEqual(mlist.archive_policy, ArchivePolicy.never) @@ -259,13 +261,11 @@ class TestMigration20120407ArchiveData(unittest.TestCase): # Test that the new archive_policy value is updated correctly. In the # case of old column archive=0, the archive_private column is # ignored. This test sets it to 1 to ensure it's ignored. - with configuration('database', url=self._url): - # Set the old archive values. - self._database.store.execute( - 'UPDATE mailinglist SET archive = 0, archive_private = 1 ' - 'WHERE id = 1;') - # Complete the migration - self._upgrade() + self._database.store.execute( + 'UPDATE mailinglist SET archive = False, archive_private = True ' + 'WHERE id = 1;') + # Complete the migration + self._upgrade() with temporary_db(self._database): mlist = getUtility(IListManager).get('test@example.com') self.assertEqual(mlist.archive_policy, ArchivePolicy.never) @@ -273,13 +273,11 @@ class TestMigration20120407ArchiveData(unittest.TestCase): def test_archive_policy_private(self): # Test that the new archive_policy value is updated correctly for # private archives. - with configuration('database', url=self._url): - # Set the old archive values. - self._database.store.execute( - 'UPDATE mailinglist SET archive = 1, archive_private = 1 ' - 'WHERE id = 1;') - # Complete the migration - self._upgrade() + self._database.store.execute( + 'UPDATE mailinglist SET archive = True, archive_private = True ' + 'WHERE id = 1;') + # Complete the migration + self._upgrade() with temporary_db(self._database): mlist = getUtility(IListManager).get('test@example.com') self.assertEqual(mlist.archive_policy, ArchivePolicy.private) @@ -287,13 +285,11 @@ class TestMigration20120407ArchiveData(unittest.TestCase): def test_archive_policy_public(self): # Test that the new archive_policy value is updated correctly for # public archives. - with configuration('database', url=self._url): - # Set the old archive values. - self._database.store.execute( - 'UPDATE mailinglist SET archive = 1, archive_private = 0 ' - 'WHERE id = 1;') - # Complete the migration - self._upgrade() + self._database.store.execute( + 'UPDATE mailinglist SET archive = True, archive_private = False ' + 'WHERE id = 1;') + # Complete the migration + self._upgrade() with temporary_db(self._database): mlist = getUtility(IListManager).get('test@example.com') self.assertEqual(mlist.archive_policy, ArchivePolicy.public) -- cgit v1.2.3-70-g09d2 From 762a4cc0838618e549e0562df96cad361f8cde5a Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Wed, 25 Jul 2012 12:32:04 -0400 Subject: No need to parameterize the exceptions, since Storm does this for us. --- src/mailman/database/postgresql.py | 3 --- src/mailman/database/sqlite.py | 2 -- src/mailman/database/tests/test_migrations.py | 5 +++-- 3 files changed, 3 insertions(+), 7 deletions(-) (limited to 'src/mailman/database/postgresql.py') diff --git a/src/mailman/database/postgresql.py b/src/mailman/database/postgresql.py index 879594459..6362f999c 100644 --- a/src/mailman/database/postgresql.py +++ b/src/mailman/database/postgresql.py @@ -25,8 +25,6 @@ __all__ = [ ] -import psycopg2 - from operator import attrgetter from urlparse import urlsplit, urlunsplit @@ -51,7 +49,6 @@ class PostgreSQLDatabase(StormBaseDatabase): """Database class for PostgreSQL.""" TAG = 'postgres' - Error = psycopg2.ProgrammingError def _database_exists(self, store): """See `BaseDatabase`.""" diff --git a/src/mailman/database/sqlite.py b/src/mailman/database/sqlite.py index 39e7e8113..4e5df55a5 100644 --- a/src/mailman/database/sqlite.py +++ b/src/mailman/database/sqlite.py @@ -27,7 +27,6 @@ __all__ = [ import os import shutil -import sqlite3 import tempfile from urlparse import urlparse @@ -50,7 +49,6 @@ class SQLiteDatabase(StormBaseDatabase): """Database class for SQLite.""" TAG = 'sqlite' - Error = sqlite3.OperationalError def _database_exists(self, store): """See `BaseDatabase`.""" diff --git a/src/mailman/database/tests/test_migrations.py b/src/mailman/database/tests/test_migrations.py index 90f2a9dfe..777e84fac 100644 --- a/src/mailman/database/tests/test_migrations.py +++ b/src/mailman/database/tests/test_migrations.py @@ -30,6 +30,7 @@ __all__ = [ import unittest from pkg_resources import resource_string +from storm.exceptions import DatabaseError from zope.component import getUtility from mailman.config import config @@ -78,7 +79,7 @@ class TestMigration20120407Schema(unittest.TestCase): # Verify that the database has not yet been migrated. for missing in ('archive_policy', 'nntp_prefix_subject_too'): - self.assertRaises(self._database.Error, + self.assertRaises(DatabaseError, self._database.store.execute, 'select {0} from mailinglist;'.format(missing)) # Avoid PostgreSQL complaint: InternalError: current transaction @@ -115,7 +116,7 @@ class TestMigration20120407Schema(unittest.TestCase): 'news_moderation', 'news_prefix_subject_too', 'nntp_host'): - self.assertRaises(self._database.Error, + self.assertRaises(DatabaseError, self._database.store.execute, 'select {0} from mailinglist;'.format(missing)) # Avoid PostgreSQL complaint: InternalError: current transaction -- cgit v1.2.3-70-g09d2 From ce410fe79601009220e850ad07b6931d29918376 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Wed, 25 Jul 2012 13:35:53 -0400 Subject: Refactor once again for SQLite/PostgreSQL differences. --- src/mailman/database/postgresql.py | 7 ++ src/mailman/database/sqlite.py | 8 +++ src/mailman/database/tests/test_migrations.py | 93 ++++++++------------------- src/mailman/testing/testing.cfg | 6 +- 4 files changed, 46 insertions(+), 68 deletions(-) (limited to 'src/mailman/database/postgresql.py') diff --git a/src/mailman/database/postgresql.py b/src/mailman/database/postgresql.py index 6362f999c..e7c40407a 100644 --- a/src/mailman/database/postgresql.py +++ b/src/mailman/database/postgresql.py @@ -35,6 +35,10 @@ from mailman.database.base import StormBaseDatabase class _TemporaryDB: + # For the test suite; bool column values. + TRUE = 'True' + FALSE = 'False' + def __init__(self, database): self.database = database @@ -43,6 +47,9 @@ class _TemporaryDB: self.database.store.close() config.db.store.execute('DROP DATABASE mmtest;') + def abort(self): + self.database.store.execute('ABORT;') + class PostgreSQLDatabase(StormBaseDatabase): diff --git a/src/mailman/database/sqlite.py b/src/mailman/database/sqlite.py index 4e5df55a5..54dce0352 100644 --- a/src/mailman/database/sqlite.py +++ b/src/mailman/database/sqlite.py @@ -36,6 +36,10 @@ from mailman.database.base import StormBaseDatabase class _TemporaryDB: + # For the test suite; bool column values. + TRUE = 1 + FALSE = 0 + def __init__(self, database, tempdir): self.database = database self._tempdir = tempdir @@ -43,6 +47,10 @@ class _TemporaryDB: def cleanup(self): shutil.rmtree(self._tempdir) + def abort(self): + # No abort needed. + pass + class SQLiteDatabase(StormBaseDatabase): diff --git a/src/mailman/database/tests/test_migrations.py b/src/mailman/database/tests/test_migrations.py index 777e84fac..9d20e0dae 100644 --- a/src/mailman/database/tests/test_migrations.py +++ b/src/mailman/database/tests/test_migrations.py @@ -44,7 +44,7 @@ from mailman.utilities.modules import call_name -class TestMigration20120407Schema(unittest.TestCase): +class MigrationTestBase(unittest.TestCase): """Test the dated migration (LP: #971013) Circa: 3.0b1 -> 3.0b2 @@ -63,13 +63,18 @@ class TestMigration20120407Schema(unittest.TestCase): def setUp(self): database_class_name = config.database['class'] - database_class = call_name(database_class_name) - self._temporary = database_class._make_temporary() + self._database_class = call_name(database_class_name) + self._temporary = self._database_class._make_temporary() self._database = self._temporary.database def tearDown(self): self._temporary.cleanup() + + +class TestMigration20120407Schema(MigrationTestBase): + """Test column migrations.""" + def test_pre_upgrade_columns_base(self): # Test that before the migration, the old table columns are present # and the new database columns are not. @@ -82,9 +87,7 @@ class TestMigration20120407Schema(unittest.TestCase): self.assertRaises(DatabaseError, self._database.store.execute, 'select {0} from mailinglist;'.format(missing)) - # Avoid PostgreSQL complaint: InternalError: current transaction - # is aborted, commands ignored until end of transaction block. - self._database.store.execute('ABORT;') + self._temporary.abort() for present in ('archive', 'archive_private', 'archive_volume_frequency', @@ -119,47 +122,25 @@ class TestMigration20120407Schema(unittest.TestCase): self.assertRaises(DatabaseError, self._database.store.execute, 'select {0} from mailinglist;'.format(missing)) - # Avoid PostgreSQL complaint: InternalError: current transaction - # is aborted, commands ignored until end of transaction block. - self._database.store.execute('ABORT;') + self._temporary.abort() -class TestMigration20120407Data(unittest.TestCase): - """Test the dated migration (LP: #971013) - - Circa: 3.0b1 -> 3.0b2 - - table mailinglist: - * news_moderation -> newsgroup_moderation - * news_prefix_subject_too -> nntp_prefix_subject_too - * ADD archive_policy - * REMOVE archive - * REMOVE archive_private - * REMOVE archive_volume_frequency - * REMOVE nntp_host - """ - - layer = ConfigLayer +class TestMigration20120407Data(MigrationTestBase): + """Test non-migrated data.""" def setUp(self): - database_class_name = config.database['class'] - database_class = call_name(database_class_name) - self._temporary = database_class._make_temporary() - self._database = self._temporary.database + MigrationTestBase.setUp(self) # Load all the migrations to just before the one we're testing. self._database.load_migrations('20120406999999') # Load the previous schema's sample data. sample_data = resource_string( 'mailman.database.tests.data', - 'migration_{0}_1.sql'.format(database_class.TAG)) + 'migration_{0}_1.sql'.format(self._database_class.TAG)) self._database.load_sql(self._database.store, sample_data) # Update to the current migration we're testing. self._database.load_migrations('20120407000000') - def tearDown(self): - self._temporary.cleanup() - def test_migration_domains(self): # Test that the domains table, which isn't touched, doesn't change. with temporary_db(self._database): @@ -208,50 +189,30 @@ class TestMigration20120407Data(unittest.TestCase): -class TestMigration20120407ArchiveData(unittest.TestCase): - """Test the dated migration (LP: #971013) - - Circa: 3.0b1 -> 3.0b2 - - table mailinglist: - * news_moderation -> newsgroup_moderation - * news_prefix_subject_too -> nntp_prefix_subject_too - * ADD archive_policy - * REMOVE archive - * REMOVE archive_private - * REMOVE archive_volume_frequency - * REMOVE nntp_host - """ - - layer = ConfigLayer +class TestMigration20120407ArchiveData(MigrationTestBase): + """Test affected migration data.""" def setUp(self): - database_class_name = config.database['class'] - database_class = call_name(database_class_name) - self._temporary = database_class._make_temporary() - self._database = self._temporary.database + MigrationTestBase.setUp(self) # Load all the migrations to just before the one we're testing. self._database.load_migrations('20120406999999') # Load the previous schema's sample data. sample_data = resource_string( 'mailman.database.tests.data', - 'migration_{0}_1.sql'.format(database_class.TAG)) + 'migration_{0}_1.sql'.format(self._database_class.TAG)) self._database.load_sql(self._database.store, sample_data) def _upgrade(self): # Update to the current migration we're testing. self._database.load_migrations('20120407000000') - def tearDown(self): - self._temporary.cleanup() - def test_migration_archive_policy_never_0(self): # Test that the new archive_policy value is updated correctly. In the # case of old column archive=0, the archive_private column is # ignored. This test sets it to 0 to ensure it's ignored. self._database.store.execute( - 'UPDATE mailinglist SET archive = False, archive_private = False ' - 'WHERE id = 1;') + 'UPDATE mailinglist SET archive = {0}, archive_private = {0} ' + 'WHERE id = 1;'.format(self._temporary.FALSE)) # Complete the migration self._upgrade() with temporary_db(self._database): @@ -263,8 +224,9 @@ class TestMigration20120407ArchiveData(unittest.TestCase): # case of old column archive=0, the archive_private column is # ignored. This test sets it to 1 to ensure it's ignored. self._database.store.execute( - 'UPDATE mailinglist SET archive = False, archive_private = True ' - 'WHERE id = 1;') + 'UPDATE mailinglist SET archive = {0}, archive_private = {1} ' + 'WHERE id = 1;'.format(self._temporary.FALSE, + self._temporary.TRUE)) # Complete the migration self._upgrade() with temporary_db(self._database): @@ -275,8 +237,8 @@ class TestMigration20120407ArchiveData(unittest.TestCase): # Test that the new archive_policy value is updated correctly for # private archives. self._database.store.execute( - 'UPDATE mailinglist SET archive = True, archive_private = True ' - 'WHERE id = 1;') + 'UPDATE mailinglist SET archive = {0}, archive_private = {0} ' + 'WHERE id = 1;'.format(self._temporary.TRUE)) # Complete the migration self._upgrade() with temporary_db(self._database): @@ -287,8 +249,9 @@ class TestMigration20120407ArchiveData(unittest.TestCase): # Test that the new archive_policy value is updated correctly for # public archives. self._database.store.execute( - 'UPDATE mailinglist SET archive = True, archive_private = False ' - 'WHERE id = 1;') + 'UPDATE mailinglist SET archive = {1}, archive_private = {0} ' + 'WHERE id = 1;'.format(self._temporary.FALSE, + self._temporary.TRUE)) # Complete the migration self._upgrade() with temporary_db(self._database): diff --git a/src/mailman/testing/testing.cfg b/src/mailman/testing/testing.cfg index 0be01298b..141d74a8f 100644 --- a/src/mailman/testing/testing.cfg +++ b/src/mailman/testing/testing.cfg @@ -18,9 +18,9 @@ # A testing configuration. # For testing against PostgreSQL. -[database] -class: mailman.database.postgresql.PostgreSQLDatabase -url: postgres://barry:barry@localhost/mailman +# [database] +# class: mailman.database.postgresql.PostgreSQLDatabase +# url: postgres://barry:barry@localhost/mailman [mailman] site_owner: noreply@example.com -- cgit v1.2.3-70-g09d2 From 8dec52e851c4cd8fcdd29702b95fad039fcedf2a Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Wed, 25 Jul 2012 13:39:50 -0400 Subject: One more refactoring. --- src/mailman/database/postgresql.py | 8 ++++---- src/mailman/database/sqlite.py | 6 +++--- src/mailman/database/tests/test_migrations.py | 22 +++++++++++----------- src/mailman/testing/testing.cfg | 6 +++--- 4 files changed, 21 insertions(+), 21 deletions(-) (limited to 'src/mailman/database/postgresql.py') diff --git a/src/mailman/database/postgresql.py b/src/mailman/database/postgresql.py index e7c40407a..1d1abca9e 100644 --- a/src/mailman/database/postgresql.py +++ b/src/mailman/database/postgresql.py @@ -34,7 +34,7 @@ from mailman.database.base import StormBaseDatabase -class _TemporaryDB: +class _TestDB: # For the test suite; bool column values. TRUE = 'True' FALSE = 'False' @@ -86,7 +86,7 @@ class PostgreSQLDatabase(StormBaseDatabase): """.format(model_class.__storm_table__)) @staticmethod - def _make_temporary(): + def _make_testdb(): from mailman.testing.helpers import configuration parts = urlsplit(config.database.url) assert parts.scheme == 'postgres' @@ -98,8 +98,8 @@ class PostgreSQLDatabase(StormBaseDatabase): # after the test. config.db.store.execute('ABORT;') config.db.store.execute('CREATE DATABASE mmtest;') - # Now create a new, temporary database. + # Now create a new, test database. database = PostgreSQLDatabase() with configuration('database', url=url): database.initialize() - return _TemporaryDB(database) + return _TestDB(database) diff --git a/src/mailman/database/sqlite.py b/src/mailman/database/sqlite.py index 54dce0352..cdee8f525 100644 --- a/src/mailman/database/sqlite.py +++ b/src/mailman/database/sqlite.py @@ -35,7 +35,7 @@ from mailman.database.base import StormBaseDatabase -class _TemporaryDB: +class _TestDB: # For the test suite; bool column values. TRUE = 1 FALSE = 0 @@ -76,11 +76,11 @@ class SQLiteDatabase(StormBaseDatabase): os.close(fd) @staticmethod - def _make_temporary(): + def _make_testdb(): from mailman.testing.helpers import configuration tempdir = tempfile.mkdtemp() url = 'sqlite:///' + os.path.join(tempdir, 'mailman.db') database = SQLiteDatabase() with configuration('database', url=url): database.initialize() - return _TemporaryDB(database, tempdir) + return _TestDB(database, tempdir) diff --git a/src/mailman/database/tests/test_migrations.py b/src/mailman/database/tests/test_migrations.py index 9d20e0dae..87f012404 100644 --- a/src/mailman/database/tests/test_migrations.py +++ b/src/mailman/database/tests/test_migrations.py @@ -64,11 +64,11 @@ class MigrationTestBase(unittest.TestCase): def setUp(self): database_class_name = config.database['class'] self._database_class = call_name(database_class_name) - self._temporary = self._database_class._make_temporary() - self._database = self._temporary.database + self._testdb = self._database_class._make_testdb() + self._database = self._testdb.database def tearDown(self): - self._temporary.cleanup() + self._testdb.cleanup() @@ -87,7 +87,7 @@ class TestMigration20120407Schema(MigrationTestBase): self.assertRaises(DatabaseError, self._database.store.execute, 'select {0} from mailinglist;'.format(missing)) - self._temporary.abort() + self._testdb.abort() for present in ('archive', 'archive_private', 'archive_volume_frequency', @@ -122,7 +122,7 @@ class TestMigration20120407Schema(MigrationTestBase): self.assertRaises(DatabaseError, self._database.store.execute, 'select {0} from mailinglist;'.format(missing)) - self._temporary.abort() + self._testdb.abort() @@ -212,7 +212,7 @@ class TestMigration20120407ArchiveData(MigrationTestBase): # ignored. This test sets it to 0 to ensure it's ignored. self._database.store.execute( 'UPDATE mailinglist SET archive = {0}, archive_private = {0} ' - 'WHERE id = 1;'.format(self._temporary.FALSE)) + 'WHERE id = 1;'.format(self._testdb.FALSE)) # Complete the migration self._upgrade() with temporary_db(self._database): @@ -225,8 +225,8 @@ class TestMigration20120407ArchiveData(MigrationTestBase): # ignored. This test sets it to 1 to ensure it's ignored. self._database.store.execute( 'UPDATE mailinglist SET archive = {0}, archive_private = {1} ' - 'WHERE id = 1;'.format(self._temporary.FALSE, - self._temporary.TRUE)) + 'WHERE id = 1;'.format(self._testdb.FALSE, + self._testdb.TRUE)) # Complete the migration self._upgrade() with temporary_db(self._database): @@ -238,7 +238,7 @@ class TestMigration20120407ArchiveData(MigrationTestBase): # private archives. self._database.store.execute( 'UPDATE mailinglist SET archive = {0}, archive_private = {0} ' - 'WHERE id = 1;'.format(self._temporary.TRUE)) + 'WHERE id = 1;'.format(self._testdb.TRUE)) # Complete the migration self._upgrade() with temporary_db(self._database): @@ -250,8 +250,8 @@ class TestMigration20120407ArchiveData(MigrationTestBase): # public archives. self._database.store.execute( 'UPDATE mailinglist SET archive = {1}, archive_private = {0} ' - 'WHERE id = 1;'.format(self._temporary.FALSE, - self._temporary.TRUE)) + 'WHERE id = 1;'.format(self._testdb.FALSE, + self._testdb.TRUE)) # Complete the migration self._upgrade() with temporary_db(self._database): diff --git a/src/mailman/testing/testing.cfg b/src/mailman/testing/testing.cfg index 141d74a8f..0be01298b 100644 --- a/src/mailman/testing/testing.cfg +++ b/src/mailman/testing/testing.cfg @@ -18,9 +18,9 @@ # A testing configuration. # For testing against PostgreSQL. -# [database] -# class: mailman.database.postgresql.PostgreSQLDatabase -# url: postgres://barry:barry@localhost/mailman +[database] +class: mailman.database.postgresql.PostgreSQLDatabase +url: postgres://barry:barry@localhost/mailman [mailman] site_owner: noreply@example.com -- cgit v1.2.3-70-g09d2 From 3ecb13338d36f7f4bccb609bdb2d54ff11359f8f Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Wed, 25 Jul 2012 14:20:06 -0400 Subject: A few more tweaks to get PostgreSQL working. - store.rollback() is better than store.execute('ABORT;') - We need to do a commit after the migrations are loaded. --- src/mailman/database/postgresql.py | 5 +---- src/mailman/database/sqlite.py | 4 ---- src/mailman/database/tests/test_migrations.py | 7 ++++--- 3 files changed, 5 insertions(+), 11 deletions(-) (limited to 'src/mailman/database/postgresql.py') diff --git a/src/mailman/database/postgresql.py b/src/mailman/database/postgresql.py index 1d1abca9e..4200ae4fe 100644 --- a/src/mailman/database/postgresql.py +++ b/src/mailman/database/postgresql.py @@ -43,13 +43,10 @@ class _TestDB: self.database = database def cleanup(self): - self.database.store.execute('ABORT;') + self.database.store.rollback() self.database.store.close() config.db.store.execute('DROP DATABASE mmtest;') - def abort(self): - self.database.store.execute('ABORT;') - class PostgreSQLDatabase(StormBaseDatabase): diff --git a/src/mailman/database/sqlite.py b/src/mailman/database/sqlite.py index cdee8f525..fc1037301 100644 --- a/src/mailman/database/sqlite.py +++ b/src/mailman/database/sqlite.py @@ -47,10 +47,6 @@ class _TestDB: def cleanup(self): shutil.rmtree(self._tempdir) - def abort(self): - # No abort needed. - pass - class SQLiteDatabase(StormBaseDatabase): diff --git a/src/mailman/database/tests/test_migrations.py b/src/mailman/database/tests/test_migrations.py index 87f012404..595a673fc 100644 --- a/src/mailman/database/tests/test_migrations.py +++ b/src/mailman/database/tests/test_migrations.py @@ -75,19 +75,20 @@ class MigrationTestBase(unittest.TestCase): class TestMigration20120407Schema(MigrationTestBase): """Test column migrations.""" - def test_pre_upgrade_columns_base(self): + def test_pre_upgrade_columns_migration(self): # Test that before the migration, the old table columns are present # and the new database columns are not. # # Load all the migrations to just before the one we're testing. self._database.load_migrations('20120406999999') + self._database.store.commit() # Verify that the database has not yet been migrated. for missing in ('archive_policy', 'nntp_prefix_subject_too'): self.assertRaises(DatabaseError, self._database.store.execute, 'select {0} from mailinglist;'.format(missing)) - self._testdb.abort() + self._database.store.rollback() for present in ('archive', 'archive_private', 'archive_volume_frequency', @@ -122,7 +123,7 @@ class TestMigration20120407Schema(MigrationTestBase): self.assertRaises(DatabaseError, self._database.store.execute, 'select {0} from mailinglist;'.format(missing)) - self._testdb.abort() + self._database.store.rollback() -- cgit v1.2.3-70-g09d2 From fc7e14405afd9a89ae1b6cc8cabd861ec4e72ee8 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Wed, 25 Jul 2012 21:50:42 -0400 Subject: Fix resetting PostgreSQL databases, thus making the full test suite pass with them. --- src/mailman/database/factory.py | 36 +++++++++++++++++++++++++++++++++-- src/mailman/database/postgresql.py | 39 +------------------------------------- src/mailman/database/sqlite.py | 2 -- src/mailman/testing/testing.cfg | 6 +++--- 4 files changed, 38 insertions(+), 45 deletions(-) (limited to 'src/mailman/database/postgresql.py') diff --git a/src/mailman/database/factory.py b/src/mailman/database/factory.py index 80ec09b36..0ea2f4776 100644 --- a/src/mailman/database/factory.py +++ b/src/mailman/database/factory.py @@ -31,8 +31,9 @@ import os import types import shutil import tempfile -import functools +from functools import partial +from urlparse import urlsplit, urlunsplit from zope.interface import implementer from zope.interface.verify import verifyObject @@ -64,7 +65,9 @@ def _reset(self): """See `IDatabase`.""" from mailman.database.model import ModelMeta self.store.rollback() + self._pre_reset(self.store) ModelMeta._reset(self.store) + self._post_reset(self.store) self.store.commit() @@ -91,6 +94,12 @@ def _sqlite_cleanup(self, tempdir): shutil.rmtree(tempdir) +def _postgresql_cleanup(self, database, tempdb_name): + database.store.rollback() + database.store.close() + config.db.store.execute('DROP DATABASE {0}'.format(tempdb_name)) + + @implementer(IDatabaseFactory) class DatabaseTemporaryFactory: """Create a temporary database for some of the migration tests.""" @@ -107,9 +116,32 @@ class DatabaseTemporaryFactory: with configuration('database', url=url): database.initialize() database._cleanup = types.MethodType( - functools.partial(_sqlite_cleanup, tempdir=tempdir), database) + partial(_sqlite_cleanup, tempdir=tempdir), database) # bool column values in SQLite must be integers. database.FALSE = 0 database.TRUE = 1 + elif database.TAG == 'postgres': + parts = urlsplit(config.database.url) + assert parts.scheme == 'postgres' + new_parts = list(parts) + new_parts[2] = '/mmtest' + url = urlunsplit(new_parts) + # Use the existing database connection to create a new testing + # database. + config.db.store.execute('ABORT;') + config.db.store.execute('CREATE DATABASE mmtest;') + with configuration('database', url=url): + database.initialize() + database._cleanup = types.MethodType( + partial(_postgresql_cleanup, + database=database, + tempdb_name='mmtest'), + database) + # bool column values in PostgreSQL. + database.FALSE = 'False' + database.TRUE = 'True' + else: + raise RuntimeError( + 'Unsupported test database: {0}'.format(database.TAG)) # Don't load the migrations. return database diff --git a/src/mailman/database/postgresql.py b/src/mailman/database/postgresql.py index 4200ae4fe..1fb831b3b 100644 --- a/src/mailman/database/postgresql.py +++ b/src/mailman/database/postgresql.py @@ -17,7 +17,7 @@ """PostgreSQL database support.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ @@ -26,28 +26,10 @@ __all__ = [ from operator import attrgetter -from urlparse import urlsplit, urlunsplit -from mailman.config import config from mailman.database.base import StormBaseDatabase - - -class _TestDB: - # For the test suite; bool column values. - TRUE = 'True' - FALSE = 'False' - - def __init__(self, database): - self.database = database - - def cleanup(self): - self.database.store.rollback() - self.database.store.close() - config.db.store.execute('DROP DATABASE mmtest;') - - class PostgreSQLDatabase(StormBaseDatabase): """Database class for PostgreSQL.""" @@ -81,22 +63,3 @@ class PostgreSQLDatabase(StormBaseDatabase): max("id") IS NOT null) FROM "{0}"; """.format(model_class.__storm_table__)) - - @staticmethod - def _make_testdb(): - from mailman.testing.helpers import configuration - parts = urlsplit(config.database.url) - assert parts.scheme == 'postgres' - new_parts = list(parts) - new_parts[2] = '/mmtest' - url = urlunsplit(new_parts) - # Use the existing database connection to create a new testing - # database. Create a savepoint, which will make it easy to reset - # after the test. - config.db.store.execute('ABORT;') - config.db.store.execute('CREATE DATABASE mmtest;') - # Now create a new, test database. - database = PostgreSQLDatabase() - with configuration('database', url=url): - database.initialize() - return _TestDB(database) diff --git a/src/mailman/database/sqlite.py b/src/mailman/database/sqlite.py index 199feed9e..305fa006e 100644 --- a/src/mailman/database/sqlite.py +++ b/src/mailman/database/sqlite.py @@ -26,8 +26,6 @@ __all__ = [ import os -import shutil -import tempfile from urlparse import urlparse diff --git a/src/mailman/testing/testing.cfg b/src/mailman/testing/testing.cfg index 141d74a8f..0be01298b 100644 --- a/src/mailman/testing/testing.cfg +++ b/src/mailman/testing/testing.cfg @@ -18,9 +18,9 @@ # A testing configuration. # For testing against PostgreSQL. -# [database] -# class: mailman.database.postgresql.PostgreSQLDatabase -# url: postgres://barry:barry@localhost/mailman +[database] +class: mailman.database.postgresql.PostgreSQLDatabase +url: postgres://barry:barry@localhost/mailman [mailman] site_owner: noreply@example.com -- cgit v1.2.3-70-g09d2 From b60f54fedab835f214f3c88e990ff3bb098e6cad Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Wed, 25 Jul 2012 23:06:30 -0400 Subject: The final bit of refactoring puts the specifics of making a temporary database into the hands of the database modules, by using ZCA adapters. --- src/mailman/config/configure.zcml | 14 +++++++++ src/mailman/database/factory.py | 58 ++++---------------------------------- src/mailman/database/postgresql.py | 40 ++++++++++++++++++++++++++ src/mailman/database/sqlite.py | 28 ++++++++++++++++++ src/mailman/interfaces/database.py | 19 ++++--------- 5 files changed, 94 insertions(+), 65 deletions(-) (limited to 'src/mailman/database/postgresql.py') diff --git a/src/mailman/config/configure.zcml b/src/mailman/config/configure.zcml index 481ef6d0b..ed85ae1a6 100644 --- a/src/mailman/config/configure.zcml +++ b/src/mailman/config/configure.zcml @@ -22,6 +22,20 @@ factory="mailman.model.requests.ListRequests" /> + + + + 0: os.close(fd) + + + +# Test suite adapter for ITemporaryDatabase. + +def _cleanup(self, tempdir): + shutil.rmtree(tempdir) + + +def make_temporary(database): + """Adapts by monkey patching an existing SQLite IDatabase.""" + tempdir = tempfile.mkdtemp() + url = 'sqlite:///' + os.path.join(tempdir, 'mailman.db') + with configuration('database', url=url): + database.initialize() + database._cleanup = types.MethodType( + partial(_cleanup, tempdir=tempdir), + database) + # bool column values in SQLite must be integers. + database.FALSE = 0 + database.TRUE = 1 + return database diff --git a/src/mailman/interfaces/database.py b/src/mailman/interfaces/database.py index a74ddd71f..1f39daee7 100644 --- a/src/mailman/interfaces/database.py +++ b/src/mailman/interfaces/database.py @@ -24,6 +24,7 @@ __all__ = [ 'DatabaseError', 'IDatabase', 'IDatabaseFactory', + 'ITemporaryDatabase', ] @@ -50,19 +51,6 @@ class IDatabase(Interface): configuration file setting. """ - def _make_temporary(): - """Make a temporary database. - - This is a @staticmethod used in the test framework. - - :return: An object with one attribute and one method. The attribute - `database` is the temporary `IDatabase`. The method is a callable - named `cleanup()`, taking no arguments which should be called when - the temporary database is no longer necessary. The database will - already be initialized, but no migrations will have been loaded - into it. - """ - def begin(): """Begin the current transaction.""" @@ -76,6 +64,11 @@ class IDatabase(Interface): """The underlying Storm store on which you can do queries.""") + +class ITemporaryDatabase(Interface): + """Marker interface for test suite adaptation.""" + + class IDatabaseFactory(Interface): "Interface for creating new databases.""" -- cgit v1.2.3-70-g09d2