From a780bf91620d157602489cb64c4f6170b1fac532 Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Thu, 4 Sep 2014 06:21:51 +0530 Subject: add new database base model for sqlalchemy --- src/mailman/database/base.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) (limited to 'src/mailman/database') diff --git a/src/mailman/database/base.py b/src/mailman/database/base.py index cbf88a4ff..1577c981d 100644 --- a/src/mailman/database/base.py +++ b/src/mailman/database/base.py @@ -29,6 +29,8 @@ import logging from lazr.config import as_boolean from pkg_resources import resource_listdir, resource_string +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker from storm.cache import GenerationalCache from storm.locals import create_database, Store from zope.interface import implementer @@ -43,6 +45,42 @@ log = logging.getLogger('mailman.config') NL = '\n' + +@implementer(IDatabase) +class SABaseDatabase: + """The database base class for use with SQLAlchemy. + + Use this as a base class for your DB_Specific derived classes. + """ + TAG='' + + def __inti__(self): + self.url = None + self.store = None + + def begin(self): + pass + + def commit(self): + self.store.commit() + + def abort(self): + self.store.rollback() + + def _prepare(self, url): + pass + + def initialize(Self, debug=None): + url = expand(config.database.url, config.paths) + log.debug('Database url: %s', url) + self.url = url + self._prepare(url) + engine = create_engine(url) + Session = sessionmaker(bind=engine) + store = Session() + self.store = session() + store.commit() + @implementer(IDatabase) class StormBaseDatabase: -- cgit v1.2.3-70-g09d2 From a61f117d8854193ba84fad7955574207dc075e77 Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Thu, 4 Sep 2014 06:29:59 +0530 Subject: change models to use sqlalchemy models --- src/mailman/database/model.py | 5 +- src/mailman/model/address.py | 22 ++--- src/mailman/model/autorespond.py | 12 +-- src/mailman/model/bans.py | 8 +- src/mailman/model/bounce.py | 11 ++- src/mailman/model/digests.py | 11 ++- src/mailman/model/domain.py | 11 ++- src/mailman/model/language.py | 5 +- src/mailman/model/mailinglist.py | 207 ++++++++++++++++++++------------------- 9 files changed, 151 insertions(+), 141 deletions(-) (limited to 'src/mailman/database') diff --git a/src/mailman/database/model.py b/src/mailman/database/model.py index ba2d39213..5fbf4005d 100644 --- a/src/mailman/database/model.py +++ b/src/mailman/database/model.py @@ -27,9 +27,12 @@ __all__ = [ from operator import attrgetter +from sqlalchemy.ext.declarative import declarative_base from storm.properties import PropertyPublisherMeta +Base = declerative_base() + class ModelMeta(PropertyPublisherMeta): """Do more magic on table classes.""" @@ -65,4 +68,4 @@ class ModelMeta(PropertyPublisherMeta): class Model: """Like Storm's `Storm` subclass, but with a bit extra.""" - __metaclass__ = ModelMeta + __metaclass__ = Base diff --git a/src/mailman/model/address.py b/src/mailman/model/address.py index f69679210..636193547 100644 --- a/src/mailman/model/address.py +++ b/src/mailman/model/address.py @@ -27,6 +27,8 @@ __all__ = [ from email.utils import formataddr from storm.locals import DateTime, Int, Reference, Unicode +from sqlalchemy import (Column, Integer, String, Unicode, + ForeignKey, Datetime) from zope.component import getUtility from zope.event import notify from zope.interface import implementer @@ -42,20 +44,18 @@ from mailman.utilities.datetime import now class Address(Model): """See `IAddress`.""" - id = Int(primary=True) - email = Unicode() - _original = Unicode() - display_name = Unicode() - _verified_on = DateTime(name='verified_on') - registered_on = DateTime() + id = Column(Integer, primary_key=True) + email = Column(Unicode) + _original = Column(Unicode) + display_name = Column(Unicode) + _verified_on = Column('verified_on', Datetime) + registered_on = Column(DateTime) + + user = Column(Integer, ForeignKey('user.id')) + preferences = Column(Integer, ForeignKey('preferences.id')) - user_id = Int() - user = Reference(user_id, 'User.id') - preferences_id = Int() - preferences = Reference(preferences_id, 'Preferences.id') def __init__(self, email, display_name): - super(Address, self).__init__() getUtility(IEmailValidator).validate(email) lower_case = email.lower() self.email = lower_case diff --git a/src/mailman/model/autorespond.py b/src/mailman/model/autorespond.py index c5e736613..cc66a3516 100644 --- a/src/mailman/model/autorespond.py +++ b/src/mailman/model/autorespond.py @@ -26,6 +26,8 @@ __all__ = [ ] +from sqlalchemy import (Column, Integer, String, Unicode, + ForeignKey, Date) from storm.locals import And, Date, Desc, Int, Reference from zope.interface import implementer @@ -42,16 +44,14 @@ from mailman.utilities.datetime import today class AutoResponseRecord(Model): """See `IAutoResponseRecord`.""" - id = Int(primary=True) + id = Column(Integer, primary_key=True) - address_id = Int() - address = Reference(address_id, 'Address.id') + address_id = Column(Integer, ForeignKey('address.id')) - mailing_list_id = Int() - mailing_list = Reference(mailing_list_id, 'MailingList.id') + mailing_list_id = Column(Integer, ForeignKey('mailinglist.id')) response_type = Enum(Response) - date_sent = Date() + date_sent = Column(Date) def __init__(self, mailing_list, address, response_type): self.mailing_list = mailing_list diff --git a/src/mailman/model/bans.py b/src/mailman/model/bans.py index 673e8e0c1..3e0683e8a 100644 --- a/src/mailman/model/bans.py +++ b/src/mailman/model/bans.py @@ -29,7 +29,7 @@ import re from storm.locals import Int, Unicode from zope.interface import implementer - +from sqlalchemy import Column, Integer, Unicode from mailman.database.model import Model from mailman.database.transaction import dbconnection from mailman.interfaces.bans import IBan, IBanManager @@ -40,9 +40,9 @@ from mailman.interfaces.bans import IBan, IBanManager class Ban(Model): """See `IBan`.""" - id = Int(primary=True) - email = Unicode() - list_id = Unicode() + id = Column(Integer, primary_key=True) + email = Column(Unicode) + list_id = Column(Unicode) def __init__(self, email, list_id): super(Ban, self).__init__() diff --git a/src/mailman/model/bounce.py b/src/mailman/model/bounce.py index 134c51263..e852daa1d 100644 --- a/src/mailman/model/bounce.py +++ b/src/mailman/model/bounce.py @@ -28,6 +28,7 @@ __all__ = [ from storm.locals import Bool, Int, DateTime, Unicode from zope.interface import implementer +from sqlalchemy import Column, Integer, Unicode, DateTime, Boolean from mailman.database.model import Model from mailman.database.transaction import dbconnection @@ -42,13 +43,13 @@ from mailman.utilities.datetime import now class BounceEvent(Model): """See `IBounceEvent`.""" - id = Int(primary=True) - list_id = Unicode() - email = Unicode() + id = Unicode(Integer, primary_key=True) + list_id = Column(Unicode) + email = Column(Unicode) timestamp = DateTime() - message_id = Unicode() + message_id = Column(Unicode) context = Enum(BounceContext) - processed = Bool() + processed = COlumn(Boolean) def __init__(self, list_id, email, msg, context=None): self.list_id = list_id diff --git a/src/mailman/model/digests.py b/src/mailman/model/digests.py index 5d9f3ddd1..77e0a0663 100644 --- a/src/mailman/model/digests.py +++ b/src/mailman/model/digests.py @@ -27,6 +27,7 @@ __all__ = [ from storm.locals import Int, Reference from zope.interface import implementer +from sqlalchemy import Column, Integer, ForeignKey from mailman.database.model import Model from mailman.database.types import Enum @@ -39,13 +40,13 @@ from mailman.interfaces.member import DeliveryMode class OneLastDigest(Model): """See `IOneLastDigest`.""" - id = Int(primary=True) + id = Column(Integer, primary_key=True) - mailing_list_id = Int() - mailing_list = Reference(mailing_list_id, 'MailingList.id') + mailing_list_id = Column(Integer, ForeignKey('mailinglist.id')) + #mailing_list = Reference(mailing_list_id, 'MailingList.id') - address_id = Int() - address = Reference(address_id, 'Address.id') + address_id = Columne(Integer, ForeignKey('address.id')) + #address = Reference(address_id, 'Address.id') delivery_mode = Enum(DeliveryMode) diff --git a/src/mailman/model/domain.py b/src/mailman/model/domain.py index 28e346022..cd974438c 100644 --- a/src/mailman/model/domain.py +++ b/src/mailman/model/domain.py @@ -27,6 +27,7 @@ __all__ = [ from urlparse import urljoin, urlparse +from sqlalchemy import Column, Unicode, Integer from storm.locals import Int, Unicode from zope.event import notify from zope.interface import implementer @@ -44,12 +45,12 @@ from mailman.model.mailinglist import MailingList class Domain(Model): """Domains.""" - id = Int(primary=True) + id = Column(Integer, primary_key=True) - mail_host = Unicode() - base_url = Unicode() - description = Unicode() - contact_address = Unicode() + mail_host = Column(Unicode) + base_url = Column(Unicode) + description = Column(Unicode) + contact_address = Column(Unicode) def __init__(self, mail_host, description=None, diff --git a/src/mailman/model/language.py b/src/mailman/model/language.py index 14cf53f07..5d5e538ab 100644 --- a/src/mailman/model/language.py +++ b/src/mailman/model/language.py @@ -27,6 +27,7 @@ __all__ = [ from storm.locals import Int, Unicode from zope.interface import implementer +from sqlalchemy import Column, Unicode, Integer from mailman.database import Model from mailman.interfaces import ILanguage @@ -37,5 +38,5 @@ from mailman.interfaces import ILanguage class Language(Model): """See `ILanguage`.""" - id = Int(primary=True) - code = Unicode() + id = Column(Integer, primary_key=True) + code = Column(Unicode) diff --git a/src/mailman/model/mailinglist.py b/src/mailman/model/mailinglist.py index 955a76968..c6f58b8f2 100644 --- a/src/mailman/model/mailinglist.py +++ b/src/mailman/model/mailinglist.py @@ -29,7 +29,10 @@ import os from storm.locals import ( And, Bool, DateTime, Float, Int, Pickle, RawStr, Reference, Store, - TimeDelta, Unicode) + TimeDelta, Unicode, Enum) +from sqlalchemy import ( Boolean, DateTime, Float, Integer, Unicode + PickleType, Interval) + from urlparse import urljoin from zope.component import getUtility from zope.event import notify @@ -73,121 +76,121 @@ UNDERSCORE = '_' class MailingList(Model): """See `IMailingList`.""" - id = Int(primary=True) + id = Column(Integer, primary_key=True) # XXX denotes attributes that should be part of the public interface but # are currently missing. # List identity - list_name = Unicode() - mail_host = Unicode() - _list_id = Unicode(name='list_id') - allow_list_posts = Bool() - include_rfc2369_headers = Bool() - advertised = Bool() - anonymous_list = Bool() + list_name = Column(Unicode) + mail_host = Column(Unicode) + _list_id = Column('list_id', Unicode) + allow_list_posts = Column(Boolean) + include_rfc2369_headers = Column(Boolean) + advertised = Column(Boolean) + anonymous_list = Column(Boolean) # Attributes not directly modifiable via the web u/i - created_at = DateTime() + created_at = Column(DateTime) # Attributes which are directly modifiable via the web u/i. The more # complicated attributes are currently stored as pickles, though that # will change as the schema and implementation is developed. - next_request_id = Int() - next_digest_number = Int() - digest_last_sent_at = DateTime() - volume = Int() - last_post_at = DateTime() + next_request_id = Column(Integer) + next_digest_number = Column(Integer) + digest_last_sent_at = Column(DateTime) + volume = Column(Integer) + last_post_at = Column(DateTime) # Implicit destination. - acceptable_aliases_id = Int() - acceptable_alias = Reference(acceptable_aliases_id, 'AcceptableAlias.id') + acceptable_aliases_id = Column(Integer, ForeignKey('acceptablealias.id')) + # acceptable_alias = Reference(acceptable_aliases_id, 'AcceptableAlias.id') # Attributes which are directly modifiable via the web u/i. The more # complicated attributes are currently stored as pickles, though that # will change as the schema and implementation is developed. - accept_these_nonmembers = Pickle() # XXX - admin_immed_notify = Bool() - admin_notify_mchanges = Bool() - administrivia = Bool() - archive_policy = Enum(ArchivePolicy) + accept_these_nonmembers = Column(PickleType) # XXX + admin_immed_notify = Column(Boolean) + admin_notify_mchanges = Column(Boolean) + administrivia = Column(Boolean) + archive_policy = Column(Enum(ArchivePolicy) # Automatic responses. - autoresponse_grace_period = TimeDelta() - autorespond_owner = Enum(ResponseAction) - autoresponse_owner_text = Unicode() - autorespond_postings = Enum(ResponseAction) - autoresponse_postings_text = Unicode() - autorespond_requests = Enum(ResponseAction) - autoresponse_request_text = Unicode() + autoresponse_grace_period = Column(Interval) + autorespond_owner = Column(Enum(ResponseAction)) + autoresponse_owner_text = Column(Unicode) + autorespond_postings = Column(Enum(ResponseAction)) + autoresponse_postings_text = Column(Unicode) + autorespond_requests = Column(Enum(ResponseAction)) + autoresponse_request_text = Column(Unicode) # Content filters. - filter_action = Enum(FilterAction) - filter_content = Bool() - collapse_alternatives = Bool() - convert_html_to_plaintext = Bool() + filter_action = Column(Enum(FilterAction)) + filter_content = Column(Boolean) + collapse_alternatives = Column(Boolean) + convert_html_to_plaintext = Column(Boolean) # Bounces. - bounce_info_stale_after = TimeDelta() # XXX - bounce_matching_headers = Unicode() # XXX - bounce_notify_owner_on_disable = Bool() # XXX - bounce_notify_owner_on_removal = Bool() # XXX - bounce_score_threshold = Int() # XXX - bounce_you_are_disabled_warnings = Int() # XXX - bounce_you_are_disabled_warnings_interval = TimeDelta() # XXX - forward_unrecognized_bounces_to = Enum(UnrecognizedBounceDisposition) - process_bounces = Bool() + bounce_info_stale_after = Column(Interval) # XXX + bounce_matching_headers = Column(Unicode) # XXX + bounce_notify_owner_on_disable = Column(Boolean) # XXX + bounce_notify_owner_on_removal = Column(Boolean) # XXX + bounce_score_threshold = Column(Integer) # XXX + bounce_you_are_disabled_warnings = Column(Integer) # XXX + bounce_you_are_disabled_warnings_interval = Column(Interval) # XXX + forward_unrecognized_bounces_to = Column(Enum(UnrecognizedBounceDisposition)) + process_bounces = Column(Boolean) # Miscellaneous - default_member_action = Enum(Action) - default_nonmember_action = Enum(Action) - description = Unicode() - digest_footer_uri = Unicode() - digest_header_uri = Unicode() - digest_is_default = Bool() - digest_send_periodic = Bool() - digest_size_threshold = Float() - digest_volume_frequency = Enum(DigestFrequency) - digestable = Bool() - discard_these_nonmembers = Pickle() - emergency = Bool() - encode_ascii_prefixes = Bool() - first_strip_reply_to = Bool() - footer_uri = Unicode() - forward_auto_discards = Bool() - gateway_to_mail = Bool() - gateway_to_news = Bool() - goodbye_message_uri = Unicode() - header_matches = Pickle() - header_uri = Unicode() - hold_these_nonmembers = Pickle() - info = Unicode() - linked_newsgroup = Unicode() - max_days_to_hold = Int() - max_message_size = Int() - max_num_recipients = Int() - member_moderation_notice = Unicode() - mime_is_default_digest = Bool() + default_member_action = Column(Enum(Action)) + default_nonmember_action = Column(Enum(Action)) + description = Column(Unicode) + digest_footer_uri = Column(Unicode) + digest_header_uri = Column(Unicode) + digest_is_default = Column(Boolean) + digest_send_periodic = Column(Boolean) + digest_size_threshold = Column(Float) + digest_volume_frequency = Column(Enum(DigestFrequency)) + digestable = Column(Boolean) + discard_these_nonmembers = Column(PickleType) + emergency = Column(Boolean) + encode_ascii_prefixes = Column(Boolean) + first_strip_reply_to = Column(Boolean) + footer_uri = Column(Unicode) + forward_auto_discards = Column(Boolean) + gateway_to_mail = Column(Boolean) + gateway_to_news = Column(Boolean) + goodbye_message_uri = Column(Unicode) + header_matches = Column(PickleType) + header_uri = Column(Unicode) + hold_these_nonmembers = Column(PickleType) + info = Column(Unicode) + linked_newsgroup = Column(Unicode) + max_days_to_hold = Column(Integer) + max_message_size = Column(Integer) + max_num_recipients = Column(Integer) + member_moderation_notice = Column(Unicode) + mime_is_default_digest = Column(Boolean) # FIXME: There should be no moderator_password moderator_password = RawStr() - newsgroup_moderation = Enum(NewsgroupModeration) - nntp_prefix_subject_too = Bool() - nondigestable = Bool() - nonmember_rejection_notice = Unicode() - obscure_addresses = Bool() - owner_chain = Unicode() - owner_pipeline = Unicode() - personalize = Enum(Personalization) - post_id = Int() - posting_chain = Unicode() - posting_pipeline = Unicode() - _preferred_language = Unicode(name='preferred_language') - display_name = Unicode() - reject_these_nonmembers = Pickle() - reply_goes_to_list = Enum(ReplyToMunging) - reply_to_address = Unicode() - require_explicit_destination = Bool() - respond_to_post_requests = Bool() - scrub_nondigest = Bool() - send_goodbye_message = Bool() - send_welcome_message = Bool() - subject_prefix = Unicode() - topics = Pickle() - topics_bodylines_limit = Int() - topics_enabled = Bool() - welcome_message_uri = Unicode() + newsgroup_moderation = Column(Enum(NewsgroupModeration)) + nntp_prefix_subject_too = Column(Boolean) + nondigestable = Column(Boolean) + nonmember_rejection_notice = Column(Unicode) + obscure_addresses = Column(Boolean) + owner_chain = Column(Unicode) + owner_pipeline = Column(Unicode) + personalize = Column(Enum(Personalization)) + post_id = Column(Integer) + posting_chain = Column(Unicode) + posting_pipeline = Column(Unicode) + _preferred_language = Column('preferred_language', Unicode) + display_name = Column(Unicode) + reject_these_nonmembers = Column(PickleType) + reply_goes_to_list = Column(Enum(ReplyToMunging)) + reply_to_address = Column(Unicode) + require_explicit_destination = Column(Boolean) + respond_to_post_requests = Column(Boolean) + scrub_nondigest = Column(Boolean) + send_goodbye_message = Column(Boolean) + send_welcome_message = Column(Boolean) + subject_prefix = Column(Unicode) + topics = Column(PickleType) + topics_bodylines_limit = Column(Integer) + topics_enabled = Column(Boolean) + welcome_message_uri = Column(Unicode) def __init__(self, fqdn_listname): super(MailingList, self).__init__() @@ -496,10 +499,10 @@ class AcceptableAlias(Model): id = Int(primary=True) - mailing_list_id = Int() + mailing_list_id = Column(Integer) mailing_list = Reference(mailing_list_id, MailingList.id) - alias = Unicode() + alias = Column(Unicode) def __init__(self, mailing_list, alias): self.mailing_list = mailing_list @@ -548,10 +551,10 @@ class ListArchiver(Model): id = Int(primary=True) - mailing_list_id = Int() + mailing_list_id = Column(Integer) mailing_list = Reference(mailing_list_id, MailingList.id) - name = Unicode() - _is_enabled = Bool() + name = Column(Unicode) + _is_enabled = Column(Boolean) def __init__(self, mailing_list, archiver_name, system_archiver): self.mailing_list = mailing_list -- cgit v1.2.3-70-g09d2 From d95e634aa7bcf8018797923c1d90fc2eadff8ce9 Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Fri, 5 Sep 2014 10:42:52 +0530 Subject: add new UUID type --- src/mailman/database/types.py | 108 +++++++++++++++++++++++++++++++++--------- 1 file changed, 85 insertions(+), 23 deletions(-) (limited to 'src/mailman/database') diff --git a/src/mailman/database/types.py b/src/mailman/database/types.py index ba3d92df4..5ffbf3965 100644 --- a/src/mailman/database/types.py +++ b/src/mailman/database/types.py @@ -23,43 +23,105 @@ from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ 'Enum', + 'UUID' ] +import uuid -from storm.properties import SimpleProperty -from storm.variables import Variable +from sqlalchemy.types import TypeDecorator, BINARY, CHAR +from sqlalchemy.dailects import postgresql -class _EnumVariable(Variable): - """Storm variable for supporting enum types. +class Enum(TypeDecorator): + """ + Stores an integer-based Enum as an integer in the database, and converts it + on-the-fly. + """ + + impl = Integer + + def __init__(self, *args, **kw): + self.enum = kw.pop("enum") + TypeDecorator.__init__(self, *args, **kw) + + def process_bind_param(self, value, dialect): + if not isinstance(value, self.enum): + raise ValueError("{} must be a value of the {} enum".format( + self.value, self.enum.__name__)) + return value.value + - To use this, make the database column a INTEGER. + def process_result_value(self, value, dialect): + return self.enum(value) + + + +class UUID(TypeDecorator): """ + Stores a UUID in the database natively when it can and falls back to + a BINARY(16) or a CHAR(32) when it can't. - def __init__(self, *args, **kws): - self._enum = kws.pop('enum') - super(_EnumVariable, self).__init__(*args, **kws) + :: - def parse_set(self, value, from_db): - if value is None: - return None - if not from_db: - return value - return self._enum(value) + from sqlalchemy_utils import UUIDType + import uuid + + class User(Base): + __tablename__ = 'user' - def parse_get(self, value, to_db): + # Pass `binary=False` to fallback to CHAR instead of BINARY + id = sa.Column(UUIDType(binary=False), primary_key=True) + """ + impl = BINARY(16) + + python_type = uuid.UUID + + def __init__(self, binary=True, native=True): + """ + :param binary: Whether to use a BINARY(16) or CHAR(32) fallback. + """ + self.binary = binary + self.native = native + + def load_dialect_impl(self, dialect): + if dialect.name == 'postgresql' and self.native: + # Use the native UUID type. + return dialect.type_descriptor(postgresql.UUID()) + + else: + # Fallback to either a BINARY or a CHAR. + kind = self.impl if self.binary else CHAR(32) + return dialect.type_descriptor(kind) + + @staticmethod + def _coerce(value): + if value and not isinstance(value, uuid.UUID): + try: + value = uuid.UUID(value) + + except (TypeError, ValueError): + value = uuid.UUID(bytes=value) + + return value + + def process_bind_param(self, value, dialect): if value is None: - return None - if not to_db: return value - return value.value + if not isinstance(value, uuid.UUID): + value = self._coerce(value) + + if self.native and dialect.name == 'postgresql': + return str(value) -class Enum(SimpleProperty): - """Custom type for Storm supporting enums.""" + return value.bytes if self.binary else value.hex + + def process_result_value(self, value, dialect): + if value is None: + return value - variable_class = _EnumVariable + if self.native and dialect.name == 'postgresql': + return uuid.UUID(value) - def __init__(self, enum=None): - super(Enum, self).__init__(enum=enum) + return uuid.UUID(bytes=value) if self.binary else uuid.UUID(value) -- cgit v1.2.3-70-g09d2 From db1f5638fe1ab83406a305c3f108c4a1bcfd9cd7 Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Sat, 6 Sep 2014 15:43:47 +0530 Subject: * change declarative_base class to use ModelMeta class * update some queries to match SA style --- src/mailman/app/subscriptions.py | 2 +- src/mailman/database/base.py | 89 +++----------------------------------- src/mailman/database/factory.py | 2 +- src/mailman/database/model.py | 23 +++------- src/mailman/database/postgresql.py | 4 +- src/mailman/database/sqlite.py | 4 +- src/mailman/database/types.py | 5 ++- src/mailman/interfaces/database.py | 2 +- src/mailman/model/address.py | 5 ++- src/mailman/model/autorespond.py | 2 +- src/mailman/model/bounce.py | 2 +- src/mailman/model/digests.py | 2 +- src/mailman/model/domain.py | 10 ++--- src/mailman/model/listmanager.py | 22 +++++----- src/mailman/model/mailinglist.py | 23 ++++++---- src/mailman/model/member.py | 6 +-- src/mailman/model/message.py | 2 +- src/mailman/model/messagestore.py | 6 +-- src/mailman/model/mime.py | 4 +- src/mailman/model/pending.py | 10 +++-- src/mailman/model/preferences.py | 2 +- src/mailman/model/requests.py | 2 +- src/mailman/model/roster.py | 2 +- src/mailman/model/user.py | 10 +++-- src/mailman/model/version.py | 2 +- 25 files changed, 84 insertions(+), 159 deletions(-) (limited to 'src/mailman/database') diff --git a/src/mailman/app/subscriptions.py b/src/mailman/app/subscriptions.py index b2560beb5..d24a9a545 100644 --- a/src/mailman/app/subscriptions.py +++ b/src/mailman/app/subscriptions.py @@ -28,7 +28,7 @@ __all__ = [ from operator import attrgetter from passlib.utils import generate_password as generate -from storm.expr import And, Or +#from storm.expr import And, Or from uuid import UUID from zope.component import getUtility from zope.interface import implementer diff --git a/src/mailman/database/base.py b/src/mailman/database/base.py index 1577c981d..a2392bb3a 100644 --- a/src/mailman/database/base.py +++ b/src/mailman/database/base.py @@ -31,8 +31,6 @@ from lazr.config import as_boolean from pkg_resources import resource_listdir, resource_string from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from storm.cache import GenerationalCache -from storm.locals import create_database, Store from zope.interface import implementer from mailman.config import config @@ -54,7 +52,7 @@ class SABaseDatabase: """ TAG='' - def __inti__(self): + def __init__(self): self.url = None self.store = None @@ -70,56 +68,6 @@ class SABaseDatabase: def _prepare(self, url): pass - def initialize(Self, debug=None): - url = expand(config.database.url, config.paths) - log.debug('Database url: %s', url) - self.url = url - self._prepare(url) - engine = create_engine(url) - Session = sessionmaker(bind=engine) - store = Session() - self.store = session() - store.commit() - - -@implementer(IDatabase) -class StormBaseDatabase: - """The database base class for use with the Storm ORM. - - Use this as a base class for your DB-specific derived classes. - """ - - # Tag used to distinguish the database being used. Override this in base - # classes. - TAG = '' - - def __init__(self): - self.url = None - self.store = None - - def begin(self): - """See `IDatabase`.""" - # Storm takes care of this for us. - pass - - def commit(self): - """See `IDatabase`.""" - self.store.commit() - - def abort(self): - """See `IDatabase`.""" - self.store.rollback() - - def _database_exists(self): - """Return True if the database exists and is initialized. - - Return False when Mailman needs to create and initialize the - underlying database schema. - - Base classes *must* override this. - """ - raise NotImplementedError - def _pre_reset(self, store): """Clean up method for testing. @@ -137,41 +85,14 @@ class StormBaseDatabase: database-specific post-removal cleanup. """ pass - - def _prepare(self, url): - """Prepare the database for creation. - - Some database backends need to do so me prep work before letting Storm - create the database. For example, we have to touch the SQLite .db - file first so that it has the proper file modes. - """ - pass - def initialize(self, debug=None): - """See `IDatabase`.""" - # Calculate the engine url. url = expand(config.database.url, config.paths) log.debug('Database url: %s', url) - # XXX By design of SQLite, database file creation does not honor - # umask. See their ticket #1193: - # http://www.sqlite.org/cvstrac/tktview?tn=1193,31 - # - # This sucks for us because the mailman.db file /must/ be group - # writable, however even though we guarantee our umask is 002 here, it - # still gets created without the necessary g+w permission, due to - # SQLite's policy. This should only affect SQLite engines because its - # the only one that creates a little file on the local file system. - # This kludges around their bug by "touch"ing the database file before - # SQLite has any chance to create it, thus honoring the umask and - # ensuring the right permissions. We only try to do this for SQLite - # engines, and yes, we could have chmod'd the file after the fact, but - # half dozen and all... self.url = url self._prepare(url) - database = create_database(url) - store = Store(database, GenerationalCache()) - database.DEBUG = (as_boolean(config.database.debug) - if debug is None else debug) + engine = create_engine(url) + Session = sessionmaker(bind=engine) + store = Session() self.store = store store.commit() @@ -262,6 +183,8 @@ class StormBaseDatabase: # Add a marker that indicates the migration version being applied. store.add(Version(component='schema', version=version)) + + @staticmethod def _make_temporary(): raise NotImplementedError diff --git a/src/mailman/database/factory.py b/src/mailman/database/factory.py index db453ea41..426d283e1 100644 --- a/src/mailman/database/factory.py +++ b/src/mailman/database/factory.py @@ -54,7 +54,7 @@ class DatabaseFactory: database = call_name(database_class) verifyObject(IDatabase, database) database.initialize() - database.load_migrations() + #database.load_migrations() database.commit() return database diff --git a/src/mailman/database/model.py b/src/mailman/database/model.py index 5fbf4005d..4b8478fc6 100644 --- a/src/mailman/database/model.py +++ b/src/mailman/database/model.py @@ -24,26 +24,21 @@ __all__ = [ 'Model', ] - from operator import attrgetter from sqlalchemy.ext.declarative import declarative_base -from storm.properties import PropertyPublisherMeta - -Base = declerative_base() - -class ModelMeta(PropertyPublisherMeta): +class ModelMeta(object): """Do more magic on table classes.""" _class_registry = set() def __init__(self, name, bases, dict): - # Before we let the base class do it's thing, force an __storm_table__ + # Before we let the base class do it's thing, force an __tablename__ # property to enforce our table naming convention. - self.__storm_table__ = name.lower() - super(ModelMeta, self).__init__(name, bases, dict) + self.__tablename__ = name.lower() + # super(ModelMeta, self).__init__(name, bases, dict) # Register the model class so that it can be more easily cleared. # This is required by the test framework so that the corresponding # table can be reset between tests. @@ -60,12 +55,8 @@ class ModelMeta(PropertyPublisherMeta): config.db._pre_reset(store) # Make sure this is deterministic, by sorting on the storm table name. classes = sorted(ModelMeta._class_registry, - key=attrgetter('__storm_table__')) + key=attrgetter('__tablename__')) for model_class in classes: - store.find(model_class).remove() - + store.query(model_class).delete() - -class Model: - """Like Storm's `Storm` subclass, but with a bit extra.""" - __metaclass__ = Base +Model = declarative_base(cls=ModelMeta) diff --git a/src/mailman/database/postgresql.py b/src/mailman/database/postgresql.py index 48c68a937..1ee454074 100644 --- a/src/mailman/database/postgresql.py +++ b/src/mailman/database/postgresql.py @@ -32,12 +32,12 @@ from functools import partial from operator import attrgetter from urlparse import urlsplit, urlunsplit -from mailman.database.base import StormBaseDatabase +from mailman.database.base import SABaseDatabase from mailman.testing.helpers import configuration -class PostgreSQLDatabase(StormBaseDatabase): +class PostgreSQLDatabase(SABaseDatabase): """Database class for PostgreSQL.""" TAG = 'postgres' diff --git a/src/mailman/database/sqlite.py b/src/mailman/database/sqlite.py index 15629615f..ec404b9c3 100644 --- a/src/mailman/database/sqlite.py +++ b/src/mailman/database/sqlite.py @@ -34,12 +34,12 @@ import tempfile from functools import partial from urlparse import urlparse -from mailman.database.base import StormBaseDatabase +from mailman.database.base import SABaseDatabase from mailman.testing.helpers import configuration -class SQLiteDatabase(StormBaseDatabase): +class SQLiteDatabase(SABaseDatabase): """Database class for SQLite.""" TAG = 'sqlite' diff --git a/src/mailman/database/types.py b/src/mailman/database/types.py index 5ffbf3965..045065591 100644 --- a/src/mailman/database/types.py +++ b/src/mailman/database/types.py @@ -23,13 +23,14 @@ from __future__ import absolute_import, print_function, unicode_literals __metaclass__ = type __all__ = [ 'Enum', - 'UUID' + 'UUID', ] import uuid +from sqlalchemy import Integer from sqlalchemy.types import TypeDecorator, BINARY, CHAR -from sqlalchemy.dailects import postgresql +from sqlalchemy.dialects import postgresql diff --git a/src/mailman/interfaces/database.py b/src/mailman/interfaces/database.py index 21f2f71d0..d8fde2b93 100644 --- a/src/mailman/interfaces/database.py +++ b/src/mailman/interfaces/database.py @@ -61,7 +61,7 @@ class IDatabase(Interface): """Abort the current transaction.""" store = Attribute( - """The underlying Storm store on which you can do queries.""") + """The underlying SQLAlchemy store on which you can do queries.""") diff --git a/src/mailman/model/address.py b/src/mailman/model/address.py index 88e28b919..59d54aab0 100644 --- a/src/mailman/model/address.py +++ b/src/mailman/model/address.py @@ -27,7 +27,7 @@ __all__ = [ from email.utils import formataddr from sqlalchemy import (Column, Integer, String, Unicode, - ForeignKey, Datetime) + ForeignKey, DateTime) from sqlalchemy.orm import relationship, backref from zope.component import getUtility from zope.event import notify @@ -50,10 +50,11 @@ class Address(Model): email = Column(Unicode) _original = Column(Unicode) display_name = Column(Unicode) - _verified_on = Column('verified_on', Datetime) + _verified_on = Column('verified_on', DateTime) registered_on = Column(DateTime) user_id = Column(Integer, ForeignKey('user.id')) + preferences_id = Column(Integer, ForeignKey('preferences.id')) prefereces = relationship('Preferences', backref=backref('Address', uselist=False)) diff --git a/src/mailman/model/autorespond.py b/src/mailman/model/autorespond.py index 4e1f42cca..92e0b6ebe 100644 --- a/src/mailman/model/autorespond.py +++ b/src/mailman/model/autorespond.py @@ -28,7 +28,7 @@ __all__ = [ from sqlalchemy import (Column, Integer, String, Unicode, ForeignKey, Date) -from storm.locals import And, Date, Desc, Int, Reference +from sqlalchemy.orm import relationship from zope.interface import implementer from mailman.database.model import Model diff --git a/src/mailman/model/bounce.py b/src/mailman/model/bounce.py index a40178837..b3f053cba 100644 --- a/src/mailman/model/bounce.py +++ b/src/mailman/model/bounce.py @@ -45,7 +45,7 @@ class BounceEvent(Model): __tablename__ = 'bounceevent' - id = Unicode(Integer, primary_key=True) + id = Column(Integer, primary_key=True) list_id = Column(Unicode) email = Column(Unicode) timestamp = Column(DateTime) diff --git a/src/mailman/model/digests.py b/src/mailman/model/digests.py index 0794bfb4f..e94bb073e 100644 --- a/src/mailman/model/digests.py +++ b/src/mailman/model/digests.py @@ -47,7 +47,7 @@ class OneLastDigest(Model): mailing_list_id = Column(Integer, ForeignKey('mailinglist.id')) maling_list = relationship('MailingList') - address_id = Columne(Integer, ForeignKey('address.id')) + address_id = Column(Integer, ForeignKey('address.id')) address = relationship('Address') delivery_mode = Column(Enum(enum=DeliveryMode)) diff --git a/src/mailman/model/domain.py b/src/mailman/model/domain.py index 2a5391abc..860107b15 100644 --- a/src/mailman/model/domain.py +++ b/src/mailman/model/domain.py @@ -142,14 +142,14 @@ class DomainManager: def remove(self, store, mail_host): domain = self[mail_host] notify(DomainDeletingEvent(domain)) - store.remove(domain) + store.delete(domain) notify(DomainDeletedEvent(mail_host)) return domain @dbconnection def get(self, store, mail_host, default=None): """See `IDomainManager`.""" - domains = store.find(Domain, mail_host=mail_host) + domains = store.query(Domain).filter_by(mail_host=mail_host) if domains.count() < 1: return default assert domains.count() == 1, ( @@ -166,15 +166,15 @@ class DomainManager: @dbconnection def __len__(self, store): - return store.find(Domain).count() + return store.query(Domain).count() @dbconnection def __iter__(self, store): """See `IDomainManager`.""" - for domain in store.find(Domain): + for domain in store.query(Domain).all(): yield domain @dbconnection def __contains__(self, store, mail_host): """See `IDomainManager`.""" - return store.find(Domain, mail_host=mail_host).count() > 0 + return store.query(Domain).filter_by(mail_host=mail_host).count() > 0 diff --git a/src/mailman/model/listmanager.py b/src/mailman/model/listmanager.py index d648a5bde..df1a31d04 100644 --- a/src/mailman/model/listmanager.py +++ b/src/mailman/model/listmanager.py @@ -52,9 +52,7 @@ class ListManager: raise InvalidEmailAddressError(fqdn_listname) list_id = '{0}.{1}'.format(listname, hostname) notify(ListCreatingEvent(fqdn_listname)) - mlist = store.find( - MailingList, - MailingList._list_id == list_id).one() + mlist = store.query(MailingList).filter_by(_list_id=list_id).first() if mlist: raise ListAlreadyExistsError(fqdn_listname) mlist = MailingList(fqdn_listname) @@ -68,40 +66,40 @@ class ListManager: """See `IListManager`.""" listname, at, hostname = fqdn_listname.partition('@') list_id = '{0}.{1}'.format(listname, hostname) - return store.find(MailingList, MailingList._list_id == list_id).one() + return store.query(MailingList).filter_by(_list_id=list_id).one() @dbconnection def get_by_list_id(self, store, list_id): """See `IListManager`.""" - return store.find(MailingList, MailingList._list_id == list_id).one() + return store.query(MailingList).filter_by(_list_id=list_id).one() @dbconnection def delete(self, store, mlist): """See `IListManager`.""" fqdn_listname = mlist.fqdn_listname notify(ListDeletingEvent(mlist)) - store.find(ContentFilter, ContentFilter.mailing_list == mlist).remove() - store.remove(mlist) + store.query(ContentFilter).filter_by(mailing_list=mlist).delete() + store.delete(mlist) notify(ListDeletedEvent(fqdn_listname)) @property @dbconnection def mailing_lists(self, store): """See `IListManager`.""" - for mlist in store.find(MailingList): + for mlist in store.query(MailingList).all(): yield mlist @dbconnection def __iter__(self, store): """See `IListManager`.""" - for mlist in store.find(MailingList): + for mlist in store.query(MailingList).all(): yield mlist @property @dbconnection def names(self, store): """See `IListManager`.""" - result_set = store.find(MailingList) + result_set = store.query(MailingList).all() for mail_host, list_name in result_set.values(MailingList.mail_host, MailingList.list_name): yield '{0}@{1}'.format(list_name, mail_host) @@ -110,7 +108,7 @@ class ListManager: @dbconnection def list_ids(self, store): """See `IListManager`.""" - result_set = store.find(MailingList) + result_set = store.query(MailingList).all() for list_id in result_set.values(MailingList._list_id): yield list_id @@ -118,7 +116,7 @@ class ListManager: @dbconnection def name_components(self, store): """See `IListManager`.""" - result_set = store.find(MailingList) + result_set = store.query(MailingList).all() for mail_host, list_name in result_set.values(MailingList.mail_host, MailingList.list_name): yield list_name, mail_host diff --git a/src/mailman/model/mailinglist.py b/src/mailman/model/mailinglist.py index ff757aa98..324d709d6 100644 --- a/src/mailman/model/mailinglist.py +++ b/src/mailman/model/mailinglist.py @@ -27,8 +27,9 @@ __all__ = [ import os -from sqlalchemy import ( Boolean, DateTime, Float, Integer, Unicode - PickleType, Interval) +from sqlalchemy import (Column, Boolean, DateTime, Float, Integer, Unicode, + PickleType, Interval, ForeignKey) +from sqlalchemy.orm import relationship from urlparse import urljoin from zope.component import getUtility from zope.event import notify @@ -66,7 +67,6 @@ from mailman.utilities.string import expand SPACE = ' ' UNDERSCORE = '_' - @implementer(IMailingList) class MailingList(Model): @@ -114,7 +114,7 @@ class MailingList(Model): autoresponse_owner_text = Column(Unicode) autorespond_postings = Column(Enum(enum=ResponseAction)) autoresponse_postings_text = Column(Unicode) - autorespond_requests = Column(Enum(Enum=ResponseAction)) + autorespond_requests = Column(Enum(enum=ResponseAction)) autoresponse_request_text = Column(Unicode) # Content filters. filter_action = Column(Enum(enum=FilterAction)) @@ -495,10 +495,13 @@ class MailingList(Model): class AcceptableAlias(Model): """See `IAcceptableAlias`.""" - id = Int(primary=True) + __tablename__ = 'acceptablealias' + + id = Column(Integer, primary_key=True) mailing_list_id = Column(Integer) - mailing_list = Reference(mailing_list_id, MailingList.id) + mailing_list = relationship('MailingList') + #mailing_list = Reference(mailing_list_id, MailingList.id) alias = Column(Unicode) @@ -547,10 +550,12 @@ class AcceptableAliasSet: class ListArchiver(Model): """See `IListArchiver`.""" - id = Int(primary=True) + __tablename__ = 'listarchiver' - mailing_list_id = Column(Integer) - mailing_list = Reference(mailing_list_id, MailingList.id) + id = Column(Integer, primary_key=True) + + mailing_list_id = Column(Integer, ForeignKey('mailinglist.id')) + mailing_list = relationship('MailingList') name = Column(Unicode) _is_enabled = Column(Boolean) diff --git a/src/mailman/model/member.py b/src/mailman/model/member.py index f7da6b012..739e35484 100644 --- a/src/mailman/model/member.py +++ b/src/mailman/model/member.py @@ -54,14 +54,14 @@ class Member(Model): __tablename__ = 'member' id = Column(Integer, primary_key=True) - _member_id = UUID() + _member_id = Column(UUID) role = Column(Enum(enum=MemberRole)) list_id = Column(Unicode) moderation_action = Column(Enum(enum=Action)) - address_id = Column(Integer, ForegignKey('address.id')) + address_id = Column(Integer, ForeignKey('address.id')) preferences_id = Column(Integer, ForeignKey('preferences.id')) - user_id = Column(Integer, ForiegnKey('user.id')) + user_id = Column(Integer, ForeignKey('user.id')) def __init__(self, role, list_id, subscriber): self._member_id = uid_factory.new_uid() diff --git a/src/mailman/model/message.py b/src/mailman/model/message.py index 64ee2c84a..9d7623d09 100644 --- a/src/mailman/model/message.py +++ b/src/mailman/model/message.py @@ -39,7 +39,7 @@ class Message(Model): __tablename__ = 'message' - id = Column(Integer, primary_key=True, default=AutoReload)) + id = Column(Integer, primary_key=True) message_id = Column(Unicode) message_id_hash = Column(Unicode) path = Column(Unicode) # TODO : was RawStr() diff --git a/src/mailman/model/messagestore.py b/src/mailman/model/messagestore.py index a4950e8c9..69860a6a1 100644 --- a/src/mailman/model/messagestore.py +++ b/src/mailman/model/messagestore.py @@ -128,14 +128,14 @@ class MessageStore: @property @dbconnection def messages(self, store): - for row in store.find(Message): + for row in store.query(Message).all(): yield self._get_message(row) @dbconnection def delete_message(self, store, message_id): - row = store.find(Message, message_id=message_id).one() + row = store.query(Message).filter_by(message_id=message_id).one() if row is None: raise LookupError(message_id) path = os.path.join(config.MESSAGES_DIR, row.path) os.remove(path) - store.remove(row) + store.delete(row) diff --git a/src/mailman/model/mime.py b/src/mailman/model/mime.py index 3fa051f10..3eac4f07b 100644 --- a/src/mailman/model/mime.py +++ b/src/mailman/model/mime.py @@ -39,11 +39,11 @@ from mailman.interfaces.mime import IContentFilter, FilterType class ContentFilter(Model): """A single filter criteria.""" - __tablename__ == 'contentfilter' + __tablename__ = 'contentfilter' id = Column(Integer, primary_key=True) - mailing_list_id = Column(Integer, ForiegnKey('mailinglist.id')) + mailing_list_id = Column(Integer, ForeignKey('mailinglist.id')) mailing_list = relationship('MailingList') filter_type = Column(Enum(enum=FilterType)) diff --git a/src/mailman/model/pending.py b/src/mailman/model/pending.py index cc203d270..0c41a4ac6 100644 --- a/src/mailman/model/pending.py +++ b/src/mailman/model/pending.py @@ -50,6 +50,8 @@ from mailman.utilities.modules import call_name class PendedKeyValue(Model): """A pended key/value pair, tied to a token.""" + __tablename__ = 'pendedkeyvalue' + def __init__(self, key, value): self.key = key self.value = value @@ -57,7 +59,7 @@ class PendedKeyValue(Model): id = Column(Integer, primary_key=True) key = Column(Unicode) value = Column(Unicode) - pended_id = Column(Integer) + pended_id = Column(Integer, ForeignKey('pended.id')) @@ -65,15 +67,17 @@ class PendedKeyValue(Model): class Pended(Model): """A pended event, tied to a token.""" + __tablename__ = 'pended' + def __init__(self, token, expiration_date): super(Pended, self).__init__() self.token = token self.expiration_date = expiration_date - id = Column(Integer. primary_key=True) + id = Column(Integer, primary_key=True) token = Column(Unicode) # TODO : was RawStr() expiration_date = Column(DateTime) - key_values = relationship('PendedKeyValues') + key_values = relationship('PendedKeyValue') @implementer(IPendable) diff --git a/src/mailman/model/preferences.py b/src/mailman/model/preferences.py index 73bb080a9..d74b17e30 100644 --- a/src/mailman/model/preferences.py +++ b/src/mailman/model/preferences.py @@ -41,7 +41,7 @@ from mailman.interfaces.preferences import IPreferences class Preferences(Model): """See `IPreferences`.""" - __tablename__ == 'preferences' + __tablename__ = 'preferences' id = Column(Integer, primary_key=True) acknowledge_posts = Column(Boolean) diff --git a/src/mailman/model/requests.py b/src/mailman/model/requests.py index 457341557..850ba6b3b 100644 --- a/src/mailman/model/requests.py +++ b/src/mailman/model/requests.py @@ -143,7 +143,7 @@ class ListRequests: class _Request(Model): """Table for mailing list hold requests.""" - __tablename__ == 'request' + __tablename__ = 'request' id = Column(Integer, primary_key=True)# TODO: ???, default=AutoReload) key = Column(Unicode) diff --git a/src/mailman/model/roster.py b/src/mailman/model/roster.py index 5a6a13269..f641c2846 100644 --- a/src/mailman/model/roster.py +++ b/src/mailman/model/roster.py @@ -37,7 +37,7 @@ __all__ = [ ] -from storm.expr import And, Or +#from storm.expr import And, Or from zope.interface import implementer from mailman.database.transaction import dbconnection diff --git a/src/mailman/model/user.py b/src/mailman/model/user.py index 16e87bbfb..88bf62085 100644 --- a/src/mailman/model/user.py +++ b/src/mailman/model/user.py @@ -25,7 +25,7 @@ __all__ = [ ] from sqlalchemy import Column, Unicode, Integer, DateTime, ForeignKey -from sqlalchemy import relationship, backref +from sqlalchemy.orm import relationship, backref from zope.event import notify from zope.interface import implementer @@ -59,11 +59,13 @@ class User(Model): _user_id = Column(UUID) _created_on = Column(DateTime) - addresses = relationship('Address', backref='user') + addresses = relationship('Address', + backref='user', + foreign_keys='[Address.user_id]') - _preferred_address_id = Column(Integer, ForeignKey='address.id') + _preferred_address_id = Column(Integer, ForeignKey('address.id')) _preferred_address = relationship('Address', - backred=backref('user', uselist=False)) + foreign_keys=[_preferred_address_id]) preferences_id = Column(Integer, ForeignKey('preferences.id')) preferences = relationship('Preferences', diff --git a/src/mailman/model/version.py b/src/mailman/model/version.py index 9824e54d4..8dc0d4e6c 100644 --- a/src/mailman/model/version.py +++ b/src/mailman/model/version.py @@ -32,7 +32,7 @@ from mailman.database.model import Model class Version(Model): - __tablename_ = 'version' + __tablename__ = 'version' id = Column(Integer, primary_key=True) component = Column(Unicode) -- cgit v1.2.3-70-g09d2 From 4dc18daeaeaf65be50397293b54a9874bfaca228 Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Fri, 12 Sep 2014 18:38:19 +0530 Subject: modify all storm queries to work with SA --- src/mailman/database/base.py | 99 ++++++++++++++++------------- src/mailman/database/factory.py | 6 +- src/mailman/database/model.py | 15 +++-- src/mailman/model/autorespond.py | 20 +++--- src/mailman/model/bans.py | 17 ++--- src/mailman/model/bounce.py | 2 +- src/mailman/model/listmanager.py | 4 +- src/mailman/model/mailinglist.py | 7 +- src/mailman/model/roster.py | 38 +++++------ src/mailman/model/tests/test_listmanager.py | 8 +-- src/mailman/model/user.py | 6 +- src/mailman/model/usermanager.py | 14 ++-- 12 files changed, 121 insertions(+), 115 deletions(-) (limited to 'src/mailman/database') diff --git a/src/mailman/database/base.py b/src/mailman/database/base.py index a2392bb3a..0bc530e6b 100644 --- a/src/mailman/database/base.py +++ b/src/mailman/database/base.py @@ -68,6 +68,16 @@ class SABaseDatabase: def _prepare(self, url): pass + def _database_exists(self): + """Return True if the database exists and is initialized. + + Return False when Mailman needs to create and initialize the + underlying database schema. + + Base classes *must* override this. + """ + raise NotImplementedError + def _pre_reset(self, store): """Clean up method for testing. @@ -90,11 +100,10 @@ class SABaseDatabase: log.debug('Database url: %s', url) self.url = url self._prepare(url) - engine = create_engine(url) - Session = sessionmaker(bind=engine) - store = Session() - self.store = store - store.commit() + self.engine = create_engine(url) + Session = sessionmaker(bind=self.engine) + self.store = Session() + self.store.commit() def load_migrations(self, until=None): """Load schema migrations. @@ -103,45 +112,47 @@ class SABaseDatabase: With default value of None, load all migrations. :type until: string """ - migrations_path = config.database.migrations_path - if '.' in migrations_path: - parent, dot, child = migrations_path.rpartition('.') - else: - parent = migrations_path - child = '' - # If the database does not yet exist, load the base schema. - filenames = sorted(resource_listdir(parent, child)) - # Find out which schema migrations have already been loaded. - if self._database_exists(self.store): - versions = set(version.version for version in - self.store.find(Version, component='schema')) - else: - versions = set() - for filename in filenames: - module_fn, extension = os.path.splitext(filename) - if extension != '.py': - continue - parts = module_fn.split('_') - if len(parts) < 2: - continue - version = parts[1].strip() - if len(version) == 0: - # Not a schema migration file. - continue - if version in versions: - log.debug('already migrated to %s', version) - continue - if until is not None and version > until: - # We're done. - break - module_path = migrations_path + '.' + module_fn - __import__(module_path) - upgrade = getattr(sys.modules[module_path], 'upgrade', None) - if upgrade is None: - continue - log.debug('migrating db to %s: %s', version, module_path) - upgrade(self, self.store, version, module_path) - self.commit() + from mailman.database.model import Model + Model.metadata.create_all(self.engine) + # migrations_path = config.database.migrations_path + # if '.' in migrations_path: + # parent, dot, child = migrations_path.rpartition('.') + # else: + # parent = migrations_path + # child = '' + # # If the database does not yet exist, load the base schema. + # filenames = sorted(resource_listdir(parent, child)) + # # Find out which schema migrations have already been loaded. + # if self._database_exists(self.store): + # versions = set(version.version for version in + # self.store.query(Version, component='schema')) + # else: + # versions = set() + # for filename in filenames: + # module_fn, extension = os.path.splitext(filename) + # if extension != '.py': + # continue + # parts = module_fn.split('_') + # if len(parts) < 2: + # continue + # version = parts[1].strip() + # if len(version) == 0: + # # Not a schema migration file. + # continue + # if version in versions: + # log.debug('already migrated to %s', version) + # continue + # if until is not None and version > until: + # # We're done. + # break + # module_path = migrations_path + '.' + module_fn + # __import__(module_path) + # upgrade = getattr(sys.modules[module_path], 'upgrade', None) + # if upgrade is None: + # continue + # log.debug('migrating db to %s: %s', version, module_path) + # upgrade(self, self.store, version, module_path) + # self.commit() def load_sql(self, store, sql): """Load the given SQL into the store. diff --git a/src/mailman/database/factory.py b/src/mailman/database/factory.py index 426d283e1..64fcc242c 100644 --- a/src/mailman/database/factory.py +++ b/src/mailman/database/factory.py @@ -54,7 +54,7 @@ class DatabaseFactory: database = call_name(database_class) verifyObject(IDatabase, database) database.initialize() - #database.load_migrations() + database.load_migrations() database.commit() return database @@ -62,10 +62,10 @@ class DatabaseFactory: def _reset(self): """See `IDatabase`.""" - from mailman.database.model import ModelMeta + from mailman.database.model import Model self.store.rollback() self._pre_reset(self.store) - ModelMeta._reset(self.store) + Model._reset(self) self._post_reset(self.store) self.store.commit() diff --git a/src/mailman/database/model.py b/src/mailman/database/model.py index 4b8478fc6..0cb60b7cd 100644 --- a/src/mailman/database/model.py +++ b/src/mailman/database/model.py @@ -50,13 +50,14 @@ class ModelMeta(object): ModelMeta._class_registry.add(self) @staticmethod - def _reset(store): - from mailman.config import config - config.db._pre_reset(store) + def _reset(db): + Model.metadata.drop_all(db.engine) + Model.metadata.create_all(db.engine) + # Make sure this is deterministic, by sorting on the storm table name. - classes = sorted(ModelMeta._class_registry, - key=attrgetter('__tablename__')) - for model_class in classes: - store.query(model_class).delete() + # classes = sorted(ModelMeta._class_registry, + # key=attrgetter('__tablename__')) + # for model_class in classes: + # store.query(model_class).delete() Model = declarative_base(cls=ModelMeta) diff --git a/src/mailman/model/autorespond.py b/src/mailman/model/autorespond.py index 92e0b6ebe..47f15cd54 100644 --- a/src/mailman/model/autorespond.py +++ b/src/mailman/model/autorespond.py @@ -75,12 +75,11 @@ class AutoResponseSet: @dbconnection def todays_count(self, store, address, response_type): """See `IAutoResponseSet`.""" - return store.find( - AutoResponseRecord, - And(AutoResponseRecord.address == address, - AutoResponseRecord.mailing_list == self._mailing_list, - AutoResponseRecord.response_type == response_type, - AutoResponseRecord.date_sent == today())).count() + return store.find(AutoResponseRecord).filter_by( + address = address, + mailing_list = self._mailing_list, + response_type = response_type, + date_sent = today()).count() @dbconnection def response_sent(self, store, address, response_type): @@ -92,10 +91,9 @@ class AutoResponseSet: @dbconnection def last_response(self, store, address, response_type): """See `IAutoResponseSet`.""" - results = store.find( - AutoResponseRecord, - And(AutoResponseRecord.address == address, - AutoResponseRecord.mailing_list == self._mailing_list, - AutoResponseRecord.response_type == response_type) + results = store.find(AutoResponseRecord).filter_by( + address = address, + mailing_list = self._mailing_list, + response_type = response_type ).order_by(Desc(AutoResponseRecord.date_sent)) return (None if results.count() == 0 else results.first()) diff --git a/src/mailman/model/bans.py b/src/mailman/model/bans.py index 57dbdbbd5..bf02f3127 100644 --- a/src/mailman/model/bans.py +++ b/src/mailman/model/bans.py @@ -64,7 +64,7 @@ class BanManager: @dbconnection def ban(self, store, email): """See `IBanManager`.""" - bans = store.find(Ban, email=email, list_id=self._list_id) + bans = store.query(Ban).filter_by(email=email, list_id=self._list_id) if bans.count() == 0: ban = Ban(email, self._list_id) store.add(ban) @@ -72,7 +72,8 @@ class BanManager: @dbconnection def unban(self, store, email): """See `IBanManager`.""" - ban = store.find(Ban, email=email, list_id=self._list_id).one() + ban = store.query(Ban).filter_by(email=email, + list_id=self._list_id).first() if ban is not None: store.remove(ban) @@ -83,32 +84,32 @@ class BanManager: if list_id is None: # The client is asking for global bans. Look up bans on the # specific email address first. - bans = store.find(Ban, email=email, list_id=None) + bans = store.query(Ban).filter_by(email=email, list_id=None) if bans.count() > 0: return True # And now look for global pattern bans. - bans = store.find(Ban, list_id=None) + bans = store.query(Ban).filter_by(list_id=None) for ban in bans: if (ban.email.startswith('^') and re.match(ban.email, email, re.IGNORECASE) is not None): return True else: # This is a list-specific ban. - bans = store.find(Ban, email=email, list_id=list_id) + bans = store.query(Ban).filter_by(email=email, list_id=list_id) if bans.count() > 0: return True # Try global bans next. - bans = store.find(Ban, email=email, list_id=None) + bans = store.query(Ban).filter_by(email=email, list_id=None) if bans.count() > 0: return True # Now try specific mailing list bans, but with a pattern. - bans = store.find(Ban, list_id=list_id) + bans = store.query(Ban).filteR_by(list_id=list_id) for ban in bans: if (ban.email.startswith('^') and re.match(ban.email, email, re.IGNORECASE) is not None): return True # And now try global pattern bans. - bans = store.find(Ban, list_id=None) + bans = store.query(Ban).filter_by(list_id=None) for ban in bans: if (ban.email.startswith('^') and re.match(ban.email, email, re.IGNORECASE) is not None): diff --git a/src/mailman/model/bounce.py b/src/mailman/model/bounce.py index b3f053cba..7340a4824 100644 --- a/src/mailman/model/bounce.py +++ b/src/mailman/model/bounce.py @@ -85,5 +85,5 @@ class BounceProcessor: @dbconnection def unprocessed(self, store): """See `IBounceProcessor`.""" - for event in store.find(BounceEvent, BounceEvent.processed == False): + for event in store.query(BounceEvent).filter_by(processed = False): yield event diff --git a/src/mailman/model/listmanager.py b/src/mailman/model/listmanager.py index df1a31d04..a67f7b5e1 100644 --- a/src/mailman/model/listmanager.py +++ b/src/mailman/model/listmanager.py @@ -66,12 +66,12 @@ class ListManager: """See `IListManager`.""" listname, at, hostname = fqdn_listname.partition('@') list_id = '{0}.{1}'.format(listname, hostname) - return store.query(MailingList).filter_by(_list_id=list_id).one() + return store.query(MailingList).filter_by(_list_id=list_id).first() @dbconnection def get_by_list_id(self, store, list_id): """See `IListManager`.""" - return store.query(MailingList).filter_by(_list_id=list_id).one() + return store.query(MailingList).filter_by(_list_id=list_id).first() @dbconnection def delete(self, store, mlist): diff --git a/src/mailman/model/mailinglist.py b/src/mailman/model/mailinglist.py index 324d709d6..abbf370e8 100644 --- a/src/mailman/model/mailinglist.py +++ b/src/mailman/model/mailinglist.py @@ -355,10 +355,9 @@ class MailingList(Model): """See `IMailingList`.""" # First, delete all existing MIME type filter patterns. store = Store.of(self) - results = store.find( - ContentFilter, - And(ContentFilter.mailing_list == self, - ContentFilter.filter_type == FilterType.filter_mime)) + results = store.query(ContentFilter).filter( + ContentFilter.mailing_list == self, + ContentFilter.filter_type == FilterType.filter_mime) results.remove() # Now add all the new filter types. for mime_type in sequence: diff --git a/src/mailman/model/roster.py b/src/mailman/model/roster.py index f641c2846..c8bfdc582 100644 --- a/src/mailman/model/roster.py +++ b/src/mailman/model/roster.py @@ -38,6 +38,7 @@ __all__ = [ #from storm.expr import And, Or +from sqlalchemy import and_, or_ from zope.interface import implementer from mailman.database.transaction import dbconnection @@ -65,8 +66,7 @@ class AbstractRoster: @dbconnection def _query(self, store): - return store.find( - Member, + return store.query(Member).filter( Member.list_id == self._mlist.list_id, Member.role == self.role) @@ -104,8 +104,7 @@ class AbstractRoster: @dbconnection def get_member(self, store, address): """See `IRoster`.""" - results = store.find( - Member, + results = store.query(Member).filter( Member.list_id == self._mlist.list_id, Member.role == self.role, Address.email == address, @@ -160,19 +159,17 @@ class AdministratorRoster(AbstractRoster): @dbconnection def _query(self, store): - return store.find( - Member, + return store.query(Member).filter( Member.list_id == self._mlist.list_id, - Or(Member.role == MemberRole.owner, + or_(Member.role == MemberRole.owner, Member.role == MemberRole.moderator)) @dbconnection def get_member(self, store, address): """See `IRoster`.""" - results = store.find( - Member, + results = store.query(Member).filter( Member.list_id == self._mlist.list_id, - Or(Member.role == MemberRole.moderator, + or_(Member.role == MemberRole.moderator, Member.role == MemberRole.owner), Address.email == address, Member.address_id == Address.id) @@ -206,10 +203,9 @@ class DeliveryMemberRoster(AbstractRoster): :return: A generator of members. :rtype: generator """ - results = store.find( - Member, - And(Member.list_id == self._mlist.list_id, - Member.role == MemberRole.member)) + results = store.query(Member).filter( + list_id == self._mlist.list_id, + role == MemberRole.member) for member in results: if member.delivery_mode in delivery_modes: yield member @@ -250,7 +246,7 @@ class Subscribers(AbstractRoster): @dbconnection def _query(self, store): - return store.find(Member, Member.list_id == self._mlist.list_id) + return store.query(Member).filter_by(list_id = self._mlist.list_id) @@ -265,11 +261,10 @@ class Memberships: @dbconnection def _query(self, store): - results = store.find( - Member, - Or(Member.user_id == self._user.id, - And(Address.user_id == self._user.id, - Member.address_id == Address.id))) + results = store.query(Member).filter( + or_(Member.user_id == self._user.id, + and_(Member.user_id == self._user.id, + Member.address_id == Address.id))) return results.config(distinct=True) @property @@ -297,8 +292,7 @@ class Memberships: @dbconnection def get_member(self, store, address): """See `IRoster`.""" - results = store.find( - Member, + results = store.query(Member).filter( Member.address_id == Address.id, Address.user_id == self._user.id) if results.count() == 0: diff --git a/src/mailman/model/tests/test_listmanager.py b/src/mailman/model/tests/test_listmanager.py index 2d3a4e3dc..287a4dba5 100644 --- a/src/mailman/model/tests/test_listmanager.py +++ b/src/mailman/model/tests/test_listmanager.py @@ -29,7 +29,7 @@ __all__ = [ import unittest -from storm.locals import Store +from sqlalchemy.orm import sessionmaker from zope.component import getUtility from mailman.app.lifecycle import create_list @@ -139,9 +139,9 @@ Message-ID: for name in filter_names: setattr(self._ant, name, ['test-filter-1', 'test-filter-2']) getUtility(IListManager).delete(self._ant) - store = Store.of(self._ant) - filters = store.find(ContentFilter, - ContentFilter.mailing_list == self._ant) + Session = sessionmaker() + store = Session.object_session(self._ant) + filters = store.query(ContentFilter).filter_by(mailing_list = self._ant) self.assertEqual(filters.count(), 0) diff --git a/src/mailman/model/user.py b/src/mailman/model/user.py index 88bf62085..efe00c6aa 100644 --- a/src/mailman/model/user.py +++ b/src/mailman/model/user.py @@ -59,11 +59,13 @@ class User(Model): _user_id = Column(UUID) _created_on = Column(DateTime) - addresses = relationship('Address', + addresses = relationship('Address', backref='user', foreign_keys='[Address.user_id]') - _preferred_address_id = Column(Integer, ForeignKey('address.id')) + _preferred_address_id = Column(Integer, ForeignKey('address.id', + use_alter=True, + name='prefered_address_id')) _preferred_address = relationship('Address', foreign_keys=[_preferred_address_id]) diff --git a/src/mailman/model/usermanager.py b/src/mailman/model/usermanager.py index 6f4a7ff5c..a5ee40ae8 100644 --- a/src/mailman/model/usermanager.py +++ b/src/mailman/model/usermanager.py @@ -57,7 +57,7 @@ class UserManager: @dbconnection def get_user(self, store, email): """See `IUserManager`.""" - addresses = store.find(Address, email=email.lower()) + addresses = store.query(Address).filter_by(email=email.lower()) if addresses.count() == 0: return None return addresses.one().user @@ -65,7 +65,7 @@ class UserManager: @dbconnection def get_user_by_id(self, store, user_id): """See `IUserManager`.""" - users = store.find(User, _user_id=user_id) + users = store.query(User).filter_by(_user_id=user_id) if users.count() == 0: return None return users.one() @@ -74,13 +74,13 @@ class UserManager: @dbconnection def users(self, store): """See `IUserManager`.""" - for user in store.find(User): + for user in store.query(User).all(): yield user @dbconnection def create_address(self, store, email, display_name=None): """See `IUserManager`.""" - addresses = store.find(Address, email=email.lower()) + addresses = store.query(Address).filter_by(email=email.lower()) if addresses.count() == 1: found = addresses[0] raise ExistingAddressError(found.original_email) @@ -106,7 +106,7 @@ class UserManager: @dbconnection def get_address(self, store, email): """See `IUserManager`.""" - addresses = store.find(Address, email=email.lower()) + addresses = store.query(Address).filter_by(email=email.lower()) if addresses.count() == 0: return None return addresses.one() @@ -115,12 +115,12 @@ class UserManager: @dbconnection def addresses(self, store): """See `IUserManager`.""" - for address in store.find(Address): + for address in store.query(Address).all(): yield address @property @dbconnection def members(self, store): """See `IUserManager.""" - for member in store.find(Member): + for member in store.query(Member).all(): yield member -- cgit v1.2.3-70-g09d2 From f8212e1d9d32f29039b620d8805f1a53f579dd34 Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Sat, 13 Sep 2014 22:48:48 +0530 Subject: fix all tests in mailman.model.tests --- src/mailman/app/subscriptions.py | 11 ++-- src/mailman/database/types.py | 6 +- src/mailman/model/address.py | 2 +- src/mailman/model/autorespond.py | 4 +- src/mailman/model/bans.py | 2 +- src/mailman/model/bounce.py | 2 +- src/mailman/model/domain.py | 3 +- src/mailman/model/listmanager.py | 2 +- src/mailman/model/mailinglist.py | 135 ++++++++++++++++++-------------------- src/mailman/model/member.py | 8 ++- src/mailman/model/message.py | 4 +- src/mailman/model/messagestore.py | 8 +-- src/mailman/model/pending.py | 25 +++---- src/mailman/model/requests.py | 21 +++--- src/mailman/model/roster.py | 11 ++-- src/mailman/model/user.py | 5 +- 16 files changed, 123 insertions(+), 126 deletions(-) (limited to 'src/mailman/database') diff --git a/src/mailman/app/subscriptions.py b/src/mailman/app/subscriptions.py index d24a9a545..a53d22e72 100644 --- a/src/mailman/app/subscriptions.py +++ b/src/mailman/app/subscriptions.py @@ -28,7 +28,7 @@ __all__ = [ from operator import attrgetter from passlib.utils import generate_password as generate -#from storm.expr import And, Or +from sqlalchemy import and_, or_ from uuid import UUID from zope.component import getUtility from zope.interface import implementer @@ -88,8 +88,7 @@ class SubscriptionService: @dbconnection def get_member(self, store, member_id): """See `ISubscriptionService`.""" - members = store.find( - Member, + members = store.query(Member).filter( Member._member_id == member_id) if members.count() == 0: return None @@ -117,7 +116,7 @@ class SubscriptionService: # This probably could be made more efficient. if address is None or user is None: return [] - query.append(Or(Member.address_id == address.id, + query.append(or_(Member.address_id == address.id, Member.user_id == user.id)) else: # subscriber is a user id. @@ -126,7 +125,7 @@ class SubscriptionService: if address.id is not None) if len(address_ids) == 0 or user is None: return [] - query.append(Or(Member.user_id == user.id, + query.append(or_(Member.user_id == user.id, Member.address_id.is_in(address_ids))) # Calculate the rest of the query expression, which will get And'd # with the Or clause above (if there is one). @@ -134,7 +133,7 @@ class SubscriptionService: query.append(Member.list_id == list_id) if role is not None: query.append(Member.role == role) - results = store.find(Member, And(*query)) + results = store.query(Member).filter(and_(*query)) return sorted(results, key=_membership_sort_key) def __iter__(self): diff --git a/src/mailman/database/types.py b/src/mailman/database/types.py index 045065591..81721781d 100644 --- a/src/mailman/database/types.py +++ b/src/mailman/database/types.py @@ -47,13 +47,17 @@ class Enum(TypeDecorator): TypeDecorator.__init__(self, *args, **kw) def process_bind_param(self, value, dialect): + if value is None: + return None if not isinstance(value, self.enum): raise ValueError("{} must be a value of the {} enum".format( - self.value, self.enum.__name__)) + value, self.enum.__name__)) return value.value def process_result_value(self, value, dialect): + if value is None: + return None return self.enum(value) diff --git a/src/mailman/model/address.py b/src/mailman/model/address.py index 59d54aab0..7203a31a5 100644 --- a/src/mailman/model/address.py +++ b/src/mailman/model/address.py @@ -56,7 +56,7 @@ class Address(Model): user_id = Column(Integer, ForeignKey('user.id')) preferences_id = Column(Integer, ForeignKey('preferences.id')) - prefereces = relationship('Preferences', + preferences = relationship('Preferences', backref=backref('Address', uselist=False)) def __init__(self, email, display_name): diff --git a/src/mailman/model/autorespond.py b/src/mailman/model/autorespond.py index 47f15cd54..17fe5fadc 100644 --- a/src/mailman/model/autorespond.py +++ b/src/mailman/model/autorespond.py @@ -75,7 +75,7 @@ class AutoResponseSet: @dbconnection def todays_count(self, store, address, response_type): """See `IAutoResponseSet`.""" - return store.find(AutoResponseRecord).filter_by( + return store.query(AutoResponseRecord).filter_by( address = address, mailing_list = self._mailing_list, response_type = response_type, @@ -91,7 +91,7 @@ class AutoResponseSet: @dbconnection def last_response(self, store, address, response_type): """See `IAutoResponseSet`.""" - results = store.find(AutoResponseRecord).filter_by( + results = store.query(AutoResponseRecord).filter_by( address = address, mailing_list = self._mailing_list, response_type = response_type diff --git a/src/mailman/model/bans.py b/src/mailman/model/bans.py index bf02f3127..d0f3b2519 100644 --- a/src/mailman/model/bans.py +++ b/src/mailman/model/bans.py @@ -103,7 +103,7 @@ class BanManager: if bans.count() > 0: return True # Now try specific mailing list bans, but with a pattern. - bans = store.query(Ban).filteR_by(list_id=list_id) + bans = store.query(Ban).filter_by(list_id=list_id) for ban in bans: if (ban.email.startswith('^') and re.match(ban.email, email, re.IGNORECASE) is not None): diff --git a/src/mailman/model/bounce.py b/src/mailman/model/bounce.py index 7340a4824..1165fee96 100644 --- a/src/mailman/model/bounce.py +++ b/src/mailman/model/bounce.py @@ -78,7 +78,7 @@ class BounceProcessor: @dbconnection def events(self, store): """See `IBounceProcessor`.""" - for event in store.find(BounceEvent): + for event in store.query(BounceEvent).all(): yield event @property diff --git a/src/mailman/model/domain.py b/src/mailman/model/domain.py index 860107b15..585eccf3d 100644 --- a/src/mailman/model/domain.py +++ b/src/mailman/model/domain.py @@ -94,8 +94,7 @@ class Domain(Model): @dbconnection def mailing_lists(self, store): """See `IDomain`.""" - mailing_lists = store.find( - MailingList, + mailing_lists = store.query(MailingList).filter( MailingList.mail_host == self.mail_host) for mlist in mailing_lists: yield mlist diff --git a/src/mailman/model/listmanager.py b/src/mailman/model/listmanager.py index a67f7b5e1..1279de6cc 100644 --- a/src/mailman/model/listmanager.py +++ b/src/mailman/model/listmanager.py @@ -116,7 +116,7 @@ class ListManager: @dbconnection def name_components(self, store): """See `IListManager`.""" - result_set = store.query(MailingList).all() + result_set = store.query(MailingList) for mail_host, list_name in result_set.values(MailingList.mail_host, MailingList.list_name): yield list_name, mail_host diff --git a/src/mailman/model/mailinglist.py b/src/mailman/model/mailinglist.py index b1997ef95..ed7ac5553 100644 --- a/src/mailman/model/mailinglist.py +++ b/src/mailman/model/mailinglist.py @@ -28,8 +28,8 @@ __all__ = [ import os from sqlalchemy import (Column, Boolean, DateTime, Float, Integer, Unicode, - PickleType, Interval, ForeignKey) -from sqlalchemy.orm import relationship + PickleType, Interval, ForeignKey, LargeBinary) +from sqlalchemy.orm import relationship, sessionmaker from urlparse import urljoin from zope.component import getUtility from zope.event import notify @@ -67,6 +67,8 @@ from mailman.utilities.string import expand SPACE = ' ' UNDERSCORE = '_' +Session = sessionmaker() + @implementer(IMailingList) class MailingList(Model): @@ -162,7 +164,7 @@ class MailingList(Model): member_moderation_notice = Column(Unicode) mime_is_default_digest = Column(Boolean) # FIXME: There should be no moderator_password - moderator_password = Column(Unicode) # TODO : was RawStr() + moderator_password = Column(LargeBinary) # TODO : was RawStr() newsgroup_moderation = Column(Enum(enum=NewsgroupModeration)) nntp_prefix_subject_too = Column(Boolean) nondigestable = Column(Boolean) @@ -327,26 +329,24 @@ class MailingList(Model): def send_one_last_digest_to(self, address, delivery_mode): """See `IMailingList`.""" digest = OneLastDigest(self, address, delivery_mode) - Store.of(self).add(digest) + Session.object_session(self).add(digest) @property def last_digest_recipients(self): """See `IMailingList`.""" - results = Store.of(self).find( - OneLastDigest, + results = Session.object_session(self).query(OneLastDigest).filter( OneLastDigest.mailing_list == self) recipients = [(digest.address, digest.delivery_mode) for digest in results] - results.remove() + results.delete() return recipients @property def filter_types(self): """See `IMailingList`.""" - results = Store.of(self).find( - ContentFilter, - And(ContentFilter.mailing_list == self, - ContentFilter.filter_type == FilterType.filter_mime)) + results = Session.object_session(self).query(ContentFilter).filter( + ContentFilter.mailing_list == self, + ContentFilter.filter_type == FilterType.filter_mime) for content_filter in results: yield content_filter.filter_pattern @@ -354,11 +354,11 @@ class MailingList(Model): def filter_types(self, sequence): """See `IMailingList`.""" # First, delete all existing MIME type filter patterns. - store = Store.of(self) + store = Session.object_session(self) results = store.query(ContentFilter).filter( ContentFilter.mailing_list == self, ContentFilter.filter_type == FilterType.filter_mime) - results.remove() + results.delete() # Now add all the new filter types. for mime_type in sequence: content_filter = ContentFilter( @@ -368,10 +368,9 @@ class MailingList(Model): @property def pass_types(self): """See `IMailingList`.""" - results = Store.of(self).find( - ContentFilter, - And(ContentFilter.mailing_list == self, - ContentFilter.filter_type == FilterType.pass_mime)) + results = Session.object_session(self).query(ContentFilter).filter( + ContentFilter.mailing_list == self, + ContentFilter.filter_type == FilterType.pass_mime) for content_filter in results: yield content_filter.filter_pattern @@ -379,12 +378,11 @@ class MailingList(Model): def pass_types(self, sequence): """See `IMailingList`.""" # First, delete all existing MIME type pass patterns. - store = Store.of(self) - results = store.find( - ContentFilter, - And(ContentFilter.mailing_list == self, - ContentFilter.filter_type == FilterType.pass_mime)) - results.remove() + store = Session.object_session(self) + results = store.query(ContentFilter).filter( + ContentFilter.mailing_list == self, + ContentFilter.filter_type == FilterType.pass_mime) + results.delete() # Now add all the new filter types. for mime_type in sequence: content_filter = ContentFilter( @@ -394,10 +392,9 @@ class MailingList(Model): @property def filter_extensions(self): """See `IMailingList`.""" - results = Store.of(self).find( - ContentFilter, - And(ContentFilter.mailing_list == self, - ContentFilter.filter_type == FilterType.filter_extension)) + results = Session.object_session(self).query(ContentFilter).filter( + ContentFilter.mailing_list == self, + ContentFilter.filter_type == FilterType.filter_extension) for content_filter in results: yield content_filter.filter_pattern @@ -405,12 +402,11 @@ class MailingList(Model): def filter_extensions(self, sequence): """See `IMailingList`.""" # First, delete all existing file extensions filter patterns. - store = Store.of(self) - results = store.find( - ContentFilter, - And(ContentFilter.mailing_list == self, - ContentFilter.filter_type == FilterType.filter_extension)) - results.remove() + store = Session.object_session(self) + results = store.query(ContentFilter).filter( + ContentFilter.mailing_list == self, + ContentFilter.filter_type == FilterType.filter_extension) + results.delete() # Now add all the new filter types. for mime_type in sequence: content_filter = ContentFilter( @@ -420,10 +416,9 @@ class MailingList(Model): @property def pass_extensions(self): """See `IMailingList`.""" - results = Store.of(self).find( - ContentFilter, - And(ContentFilter.mailing_list == self, - ContentFilter.filter_type == FilterType.pass_extension)) + results = Session.object_session(self).query(ContentFilter).filter( + ContentFilter.mailing_list == self, + ContentFilter.filter_type == FilterType.pass_extension) for content_filter in results: yield content_filter.pass_pattern @@ -431,12 +426,11 @@ class MailingList(Model): def pass_extensions(self, sequence): """See `IMailingList`.""" # First, delete all existing file extensions pass patterns. - store = Store.of(self) - results = store.find( - ContentFilter, - And(ContentFilter.mailing_list == self, - ContentFilter.filter_type == FilterType.pass_extension)) - results.remove() + store = Session.object_session(self) + results = store.query(ContentFilter).filter( + ContentFilter.mailing_list == self, + ContentFilter.filter_type == FilterType.pass_extension) + results.delete() # Now add all the new filter types. for mime_type in sequence: content_filter = ContentFilter( @@ -457,24 +451,22 @@ class MailingList(Model): def subscribe(self, subscriber, role=MemberRole.member): """See `IMailingList`.""" - store = Store.of(self) + store = Session.object_session(self) if IAddress.providedBy(subscriber): - member = store.find( - Member, + member = store.query(Member).filter( Member.role == role, Member.list_id == self._list_id, - Member._address == subscriber).one() + Member._address == subscriber).first() if member: raise AlreadySubscribedError( self.fqdn_listname, subscriber.email, role) elif IUser.providedBy(subscriber): if subscriber.preferred_address is None: raise MissingPreferredAddressError(subscriber) - member = store.find( - Member, + member = store.query(Member).filter( Member.role == role, Member.list_id == self._list_id, - Member._user == subscriber).one() + Member._user == subscriber).first() if member: raise AlreadySubscribedError( self.fqdn_listname, subscriber, role) @@ -518,27 +510,27 @@ class AcceptableAliasSet: def clear(self): """See `IAcceptableAliasSet`.""" - Store.of(self._mailing_list).find( - AcceptableAlias, - AcceptableAlias.mailing_list == self._mailing_list).remove() + Session.object_session(self._mailing_list).query( + AcceptableAlias).filter( + AcceptableAlias.mailing_list == self._mailing_list).delete() def add(self, alias): if not (alias.startswith('^') or '@' in alias): raise ValueError(alias) alias = AcceptableAlias(self._mailing_list, alias.lower()) - Store.of(self._mailing_list).add(alias) + Session.object_session(self._mailing_list).add(alias) def remove(self, alias): - Store.of(self._mailing_list).find( - AcceptableAlias, - And(AcceptableAlias.mailing_list == self._mailing_list, - AcceptableAlias.alias == alias.lower())).remove() + Session.object_session(self._mailing_list).query( + AcceptableAlias).filter( + AcceptableAlias.mailing_list == self._mailing_list, + AcceptableAlias.alias == alias.lower()).delete() @property def aliases(self): - aliases = Store.of(self._mailing_list).find( - AcceptableAlias, - AcceptableAlias.mailing_list == self._mailing_list) + aliases = Session.object_session(self._mailing_list).query( + AcceptableAlias).filter( + AcceptableAlias.mailing_list == self._mailing_list) for alias in aliases: yield alias.alias @@ -587,25 +579,24 @@ class ListArchiverSet: system_archivers[archiver.name] = archiver # Add any system enabled archivers which aren't already associated # with the mailing list. - store = Store.of(self._mailing_list) + store = Session.object_session(self._mailing_list) for archiver_name in system_archivers: - exists = store.find( - ListArchiver, - And(ListArchiver.mailing_list == mailing_list, - ListArchiver.name == archiver_name)).one() + exists = store.query(ListArchiver).filter( + ListArchiver.mailing_list == mailing_list, + ListArchiver.name == archiver_name).first() if exists is None: store.add(ListArchiver(mailing_list, archiver_name, system_archivers[archiver_name])) @property def archivers(self): - entries = Store.of(self._mailing_list).find( - ListArchiver, ListArchiver.mailing_list == self._mailing_list) + entries = Session.object_session(self._mailing_list).query( + ListArchiver).filter(ListArchiver.mailing_list == self._mailing_list) for entry in entries: yield entry def get(self, archiver_name): - return Store.of(self._mailing_list).find( - ListArchiver, - And(ListArchiver.mailing_list == self._mailing_list, - ListArchiver.name == archiver_name)).one() + return Session.object_session(self._mailing_list).query( + ListArchiver).filter( + ListArchiver.mailing_list == self._mailing_list, + ListArchiver.name == archiver_name).first() diff --git a/src/mailman/model/member.py b/src/mailman/model/member.py index 739e35484..f1007c311 100644 --- a/src/mailman/model/member.py +++ b/src/mailman/model/member.py @@ -25,6 +25,7 @@ __all__ = [ ] from sqlalchemy import Integer, Unicode, ForeignKey, Column +from sqlalchemy.orm import relationship from zope.component import getUtility from zope.event import notify from zope.interface import implementer @@ -60,8 +61,11 @@ class Member(Model): moderation_action = Column(Enum(enum=Action)) address_id = Column(Integer, ForeignKey('address.id')) + _address = relationship('Address') preferences_id = Column(Integer, ForeignKey('preferences.id')) + preferences = relationship('Preferences') user_id = Column(Integer, ForeignKey('user.id')) + _user = relationship('User') def __init__(self, role, list_id, subscriber): self._member_id = uid_factory.new_uid() @@ -196,5 +200,5 @@ class Member(Model): """See `IMember`.""" # Yes, this must get triggered before self is deleted. notify(UnsubscriptionEvent(self.mailing_list, self)) - store.remove(self.preferences) - store.remove(self) + store.delete(self.preferences) + store.delete(self) diff --git a/src/mailman/model/message.py b/src/mailman/model/message.py index 9d7623d09..b153d4909 100644 --- a/src/mailman/model/message.py +++ b/src/mailman/model/message.py @@ -24,7 +24,7 @@ __all__ = [ 'Message', ] -from sqlalchemy import Column, Integer, Unicode +from sqlalchemy import Column, Integer, Unicode, LargeBinary from zope.interface import implementer from mailman.database.model import Model @@ -42,7 +42,7 @@ class Message(Model): id = Column(Integer, primary_key=True) message_id = Column(Unicode) message_id_hash = Column(Unicode) - path = Column(Unicode) # TODO : was RawStr() + path = Column(LargeBinary) # TODO : was RawStr() # This is a Messge-ID field representation, not a database row id. @dbconnection diff --git a/src/mailman/model/messagestore.py b/src/mailman/model/messagestore.py index 69860a6a1..f9f224dd6 100644 --- a/src/mailman/model/messagestore.py +++ b/src/mailman/model/messagestore.py @@ -59,7 +59,7 @@ class MessageStore: # Calculate and insert the X-Message-ID-Hash. message_id = message_ids[0] # Complain if the Message-ID already exists in the storage. - existing = store.find(Message, Message.message_id == message_id).one() + existing = store.query(Message).filter(Message.message_id == message_id).first() if existing is not None: raise ValueError( 'Message ID already exists in message store: {0}'.format( @@ -107,7 +107,7 @@ class MessageStore: @dbconnection def get_message_by_id(self, store, message_id): - row = store.find(Message, message_id=message_id).one() + row = store.query(Message).filter_by(message_id=message_id).first() if row is None: return None return self._get_message(row) @@ -120,7 +120,7 @@ class MessageStore: # US-ASCII. if isinstance(message_id_hash, unicode): message_id_hash = message_id_hash.encode('ascii') - row = store.find(Message, message_id_hash=message_id_hash).one() + row = store.query(Message).filter_by(message_id_hash=message_id_hash).first() if row is None: return None return self._get_message(row) @@ -133,7 +133,7 @@ class MessageStore: @dbconnection def delete_message(self, store, message_id): - row = store.query(Message).filter_by(message_id=message_id).one() + row = store.query(Message).filter_by(message_id=message_id).first() if row is None: raise LookupError(message_id) path = os.path.join(config.MESSAGES_DIR, row.path) diff --git a/src/mailman/model/pending.py b/src/mailman/model/pending.py index 0c41a4ac6..30aae074c 100644 --- a/src/mailman/model/pending.py +++ b/src/mailman/model/pending.py @@ -31,7 +31,8 @@ import random import hashlib from lazr.config import as_timedelta -from sqlalchemy import Column, Integer, Unicode, ForeignKey, DateTime +from sqlalchemy import ( + Column, Integer, Unicode, ForeignKey, DateTime, LargeBinary) from sqlalchemy.orm import relationship from zope.interface import implementer from zope.interface.verify import verifyObject @@ -75,7 +76,7 @@ class Pended(Model): self.expiration_date = expiration_date id = Column(Integer, primary_key=True) - token = Column(Unicode) # TODO : was RawStr() + token = Column(LargeBinary) # TODO : was RawStr() expiration_date = Column(DateTime) key_values = relationship('PendedKeyValue') @@ -109,7 +110,7 @@ class Pendings: token = hashlib.sha1(repr(x)).hexdigest() # In practice, we'll never get a duplicate, but we'll be anal # about checking anyway. - if store.find(Pended, token=token).count() == 0: + if store.query(Pended).filter_by(token=token).count() == 0: break else: raise AssertionError('Could not find a valid pendings token') @@ -133,7 +134,7 @@ class Pendings: value = ('mailman.model.pending.unpack_list\1' + '\2'.join(value)) keyval = PendedKeyValue(key=key, value=value) - pending.key_values.add(keyval) + pending.key_values.append(keyval) store.add(pending) return token @@ -141,7 +142,7 @@ class Pendings: def confirm(self, store, token, expunge=True): # Token can come in as a unicode, but it's stored in the database as # bytes. They must be ascii. - pendings = store.find(Pended, token=str(token)) + pendings = store.query(Pended).filter_by(token=str(token)) if pendings.count() == 0: return None assert pendings.count() == 1, ( @@ -150,7 +151,7 @@ class Pendings: pendable = UnpendedPendable() # Find all PendedKeyValue entries that are associated with the pending # object's ID. Watch out for type conversions. - for keyvalue in store.find(PendedKeyValue, + for keyvalue in store.query(PendedKeyValue).filter( PendedKeyValue.pended_id == pending.id): if keyvalue.value is not None and '\1' in keyvalue.value: type_name, value = keyvalue.value.split('\1', 1) @@ -158,23 +159,23 @@ class Pendings: else: pendable[keyvalue.key] = keyvalue.value if expunge: - store.remove(keyvalue) + store.delete(keyvalue) if expunge: - store.remove(pending) + store.delete(pending) return pendable @dbconnection def evict(self, store): right_now = now() - for pending in store.find(Pended): + for pending in store.query(Pended).all(): if pending.expiration_date < right_now: # Find all PendedKeyValue entries that are associated with the # pending object's ID. - q = store.find(PendedKeyValue, + q = store.query(PendedKeyValue).filter( PendedKeyValue.pended_id == pending.id) for keyvalue in q: - store.remove(keyvalue) - store.remove(pending) + store.delete(keyvalue) + store.delete(pending) diff --git a/src/mailman/model/requests.py b/src/mailman/model/requests.py index 850ba6b3b..1b72f78f3 100644 --- a/src/mailman/model/requests.py +++ b/src/mailman/model/requests.py @@ -26,7 +26,7 @@ __all__ = [ from cPickle import dumps, loads from datetime import timedelta -from sqlalchemy import Column, Unicode, Integer, ForeignKey +from sqlalchemy import Column, Unicode, Integer, ForeignKey, LargeBinary from sqlalchemy.orm import relationship from zope.component import getUtility from zope.interface import implementer @@ -69,25 +69,23 @@ class ListRequests: @property @dbconnection def count(self, store): - return store.find(_Request, mailing_list=self.mailing_list).count() + return store.query(_Request).filter_by(mailing_list=self.mailing_list).count() @dbconnection def count_of(self, store, request_type): - return store.find( - _Request, + return store.query(_Request).filter_by( mailing_list=self.mailing_list, request_type=request_type).count() @property @dbconnection def held_requests(self, store): - results = store.find(_Request, mailing_list=self.mailing_list) + results = store.query(_Request).filter_by(mailing_list=self.mailing_list) for request in results: yield request @dbconnection def of_type(self, store, request_type): - results = store.find( - _Request, + results = store.query(_Request).filter_by( mailing_list=self.mailing_list, request_type=request_type) for request in results: yield request @@ -105,11 +103,12 @@ class ListRequests: data_hash = token request = _Request(key, request_type, self.mailing_list, data_hash) store.add(request) + store.flush() return request.id @dbconnection def get_request(self, store, request_id, request_type=None): - result = store.get(_Request, request_id) + result = store.query(_Request).get(request_id) if result is None: return None if request_type is not None and result.request_type != request_type: @@ -131,12 +130,12 @@ class ListRequests: @dbconnection def delete_request(self, store, request_id): - request = store.get(_Request, request_id) + request = store.query(_Request).get(request_id) if request is None: raise KeyError(request_id) # Throw away the pended data. getUtility(IPendings).confirm(request.data_hash) - store.remove(request) + store.delete(request) @@ -148,7 +147,7 @@ class _Request(Model): id = Column(Integer, primary_key=True)# TODO: ???, default=AutoReload) key = Column(Unicode) request_type = Column(Enum(enum=RequestType)) - data_hash = Column(Unicode) # TODO : was RawStr() + data_hash = Column(LargeBinary) mailing_list_id = Column(Integer, ForeignKey('mailinglist.id')) mailing_list = relationship('MailingList') diff --git a/src/mailman/model/roster.py b/src/mailman/model/roster.py index c8bfdc582..a9a396523 100644 --- a/src/mailman/model/roster.py +++ b/src/mailman/model/roster.py @@ -37,7 +37,6 @@ __all__ = [ ] -#from storm.expr import And, Or from sqlalchemy import and_, or_ from zope.interface import implementer @@ -203,9 +202,9 @@ class DeliveryMemberRoster(AbstractRoster): :return: A generator of members. :rtype: generator """ - results = store.query(Member).filter( - list_id == self._mlist.list_id, - role == MemberRole.member) + results = store.query(Member).filter_by( + list_id = self._mlist.list_id, + role = MemberRole.member) for member in results: if member.delivery_mode in delivery_modes: yield member @@ -263,9 +262,9 @@ class Memberships: def _query(self, store): results = store.query(Member).filter( or_(Member.user_id == self._user.id, - and_(Member.user_id == self._user.id, + and_(Address.user_id == self._user.id, Member.address_id == Address.id))) - return results.config(distinct=True) + return results.distinct() @property def member_count(self): diff --git a/src/mailman/model/user.py b/src/mailman/model/user.py index 0ba690805..12f4643f1 100644 --- a/src/mailman/model/user.py +++ b/src/mailman/model/user.py @@ -24,7 +24,8 @@ __all__ = [ 'User', ] -from sqlalchemy import Column, Unicode, Integer, DateTime, ForeignKey +from sqlalchemy import ( + Column, Unicode, Integer, DateTime, ForeignKey, LargeBinary) from sqlalchemy.orm import relationship, backref from zope.event import notify from zope.interface import implementer @@ -55,7 +56,7 @@ class User(Model): id = Column(Integer, primary_key=True) display_name = Column(Unicode) - _password = Column('password', Unicode) # TODO : was RawStr() + _password = Column('password', LargeBinary) # TODO : was RawStr() _user_id = Column(UUID) _created_on = Column(DateTime) -- cgit v1.2.3-70-g09d2 From f774c7d7fc1f67b80ac701b3d640e2fab24422b4 Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Thu, 18 Sep 2014 19:02:08 +0530 Subject: all tests except for importer working(ignoring test_migrations.py) --- src/mailman/database/base.py | 57 +++++++++++----------------------------- src/mailman/database/model.py | 1 + src/mailman/database/sqlite.py | 2 +- src/mailman/model/address.py | 1 + src/mailman/model/listmanager.py | 4 +-- 5 files changed, 21 insertions(+), 44 deletions(-) (limited to 'src/mailman/database') diff --git a/src/mailman/database/base.py b/src/mailman/database/base.py index 0bc530e6b..b66513a2c 100644 --- a/src/mailman/database/base.py +++ b/src/mailman/database/base.py @@ -31,6 +31,7 @@ from lazr.config import as_boolean from pkg_resources import resource_listdir, resource_string from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker +from sqlalchemy.orm.session import Session from zope.interface import implementer from mailman.config import config @@ -55,6 +56,7 @@ class SABaseDatabase: def __init__(self): self.url = None self.store = None + self.transaction = None def begin(self): pass @@ -101,10 +103,22 @@ class SABaseDatabase: self.url = url self._prepare(url) self.engine = create_engine(url) - Session = sessionmaker(bind=self.engine) - self.store = Session() + session = sessionmaker(bind=self.engine) + self.store = session() self.store.commit() + # def initialize_testing(self): + # url = expand(config.database.url, config.paths) + # log.debug('Database url: %s', url) + # self.url = url + # self._prepare(url) + # self.engine = create_engine(url) + # connection = self.engine.connect() + # self.transaction = connection.begin_nested() + # self.store = Session(connection) + # self.store.commit() + + def load_migrations(self, until=None): """Load schema migrations. @@ -114,45 +128,6 @@ class SABaseDatabase: """ from mailman.database.model import Model Model.metadata.create_all(self.engine) - # migrations_path = config.database.migrations_path - # if '.' in migrations_path: - # parent, dot, child = migrations_path.rpartition('.') - # else: - # parent = migrations_path - # child = '' - # # If the database does not yet exist, load the base schema. - # filenames = sorted(resource_listdir(parent, child)) - # # Find out which schema migrations have already been loaded. - # if self._database_exists(self.store): - # versions = set(version.version for version in - # self.store.query(Version, component='schema')) - # else: - # versions = set() - # for filename in filenames: - # module_fn, extension = os.path.splitext(filename) - # if extension != '.py': - # continue - # parts = module_fn.split('_') - # if len(parts) < 2: - # continue - # version = parts[1].strip() - # if len(version) == 0: - # # Not a schema migration file. - # continue - # if version in versions: - # log.debug('already migrated to %s', version) - # continue - # if until is not None and version > until: - # # We're done. - # break - # module_path = migrations_path + '.' + module_fn - # __import__(module_path) - # upgrade = getattr(sys.modules[module_path], 'upgrade', None) - # if upgrade is None: - # continue - # log.debug('migrating db to %s: %s', version, module_path) - # upgrade(self, self.store, version, module_path) - # self.commit() def load_sql(self, store, sql): """Load the given SQL into the store. diff --git a/src/mailman/database/model.py b/src/mailman/database/model.py index 0cb60b7cd..ab071e1d3 100644 --- a/src/mailman/database/model.py +++ b/src/mailman/database/model.py @@ -57,6 +57,7 @@ class ModelMeta(object): # Make sure this is deterministic, by sorting on the storm table name. # classes = sorted(ModelMeta._class_registry, # key=attrgetter('__tablename__')) + # print("\n\n" + str(classes) + "\n\n") # for model_class in classes: # store.query(model_class).delete() diff --git a/src/mailman/database/sqlite.py b/src/mailman/database/sqlite.py index ec404b9c3..0594d9091 100644 --- a/src/mailman/database/sqlite.py +++ b/src/mailman/database/sqlite.py @@ -72,7 +72,7 @@ def _cleanup(self, tempdir): def make_temporary(database): """Adapts by monkey patching an existing SQLite IDatabase.""" tempdir = tempfile.mkdtemp() - url = 'sqlite:///' + os.path.join(tempdir, 'mailman.db') + url = 'sqlite:///' + os.path.join(tempdir, 'mailman.db') with configuration('database', url=url): database.initialize() database._cleanup = types.MethodType( diff --git a/src/mailman/model/address.py b/src/mailman/model/address.py index 7203a31a5..fbe862829 100644 --- a/src/mailman/model/address.py +++ b/src/mailman/model/address.py @@ -60,6 +60,7 @@ class Address(Model): backref=backref('Address', uselist=False)) def __init__(self, email, display_name): + super(Address, self).__init__() getUtility(IEmailValidator).validate(email) lower_case = email.lower() self.email = lower_case diff --git a/src/mailman/model/listmanager.py b/src/mailman/model/listmanager.py index 1279de6cc..3806f9497 100644 --- a/src/mailman/model/listmanager.py +++ b/src/mailman/model/listmanager.py @@ -99,7 +99,7 @@ class ListManager: @dbconnection def names(self, store): """See `IListManager`.""" - result_set = store.query(MailingList).all() + result_set = store.query(MailingList) for mail_host, list_name in result_set.values(MailingList.mail_host, MailingList.list_name): yield '{0}@{1}'.format(list_name, mail_host) @@ -108,7 +108,7 @@ class ListManager: @dbconnection def list_ids(self, store): """See `IListManager`.""" - result_set = store.query(MailingList).all() + result_set = store.query(MailingList) for list_id in result_set.values(MailingList._list_id): yield list_id -- cgit v1.2.3-70-g09d2 From c928104c5e49f443ecbe6c21581ebd98175f2451 Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Thu, 18 Sep 2014 19:21:57 +0530 Subject: reset database by purging all data instead of dropping schema and recreating it --- src/mailman/database/model.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'src/mailman/database') diff --git a/src/mailman/database/model.py b/src/mailman/database/model.py index ab071e1d3..dedd7a34b 100644 --- a/src/mailman/database/model.py +++ b/src/mailman/database/model.py @@ -24,10 +24,13 @@ __all__ = [ 'Model', ] + +import contextlib from operator import attrgetter from sqlalchemy.ext.declarative import declarative_base +from mailman.config import config class ModelMeta(object): """Do more magic on table classes.""" @@ -51,14 +54,12 @@ class ModelMeta(object): @staticmethod def _reset(db): - Model.metadata.drop_all(db.engine) - Model.metadata.create_all(db.engine) - - # Make sure this is deterministic, by sorting on the storm table name. - # classes = sorted(ModelMeta._class_registry, - # key=attrgetter('__tablename__')) - # print("\n\n" + str(classes) + "\n\n") - # for model_class in classes: - # store.query(model_class).delete() + meta = Model.metadata + engine = config.db.engine + with contextlib.closing(engine.connect()) as con: + trans = con.begin() + for table in reversed(meta.sorted_tables): + con.execute(table.delete()) + trans.commit() Model = declarative_base(cls=ModelMeta) -- cgit v1.2.3-70-g09d2 From 6dd2ac32ee1f1e8f588c08fd5363f0f794d0a6b1 Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Fri, 19 Sep 2014 06:59:12 +0530 Subject: * fix the circular dependecy problem between User and Adress * fix almost all the errors relating to doctests --- src/mailman/app/subscriptions.py | 2 +- src/mailman/database/base.py | 12 ------------ src/mailman/database/types.py | 4 +--- src/mailman/model/autorespond.py | 3 ++- src/mailman/model/bans.py | 2 +- src/mailman/model/user.py | 9 ++++++--- src/mailman/utilities/importer.py | 20 ++++++++++++++++++++ 7 files changed, 31 insertions(+), 21 deletions(-) (limited to 'src/mailman/database') diff --git a/src/mailman/app/subscriptions.py b/src/mailman/app/subscriptions.py index a53d22e72..303303e70 100644 --- a/src/mailman/app/subscriptions.py +++ b/src/mailman/app/subscriptions.py @@ -126,7 +126,7 @@ class SubscriptionService: if len(address_ids) == 0 or user is None: return [] query.append(or_(Member.user_id == user.id, - Member.address_id.is_in(address_ids))) + Member.address_id.in_(address_ids))) # Calculate the rest of the query expression, which will get And'd # with the Or clause above (if there is one). if list_id is not None: diff --git a/src/mailman/database/base.py b/src/mailman/database/base.py index b66513a2c..5eec853d6 100644 --- a/src/mailman/database/base.py +++ b/src/mailman/database/base.py @@ -107,18 +107,6 @@ class SABaseDatabase: self.store = session() self.store.commit() - # def initialize_testing(self): - # url = expand(config.database.url, config.paths) - # log.debug('Database url: %s', url) - # self.url = url - # self._prepare(url) - # self.engine = create_engine(url) - # connection = self.engine.connect() - # self.transaction = connection.begin_nested() - # self.store = Session(connection) - # self.store.commit() - - def load_migrations(self, until=None): """Load schema migrations. diff --git a/src/mailman/database/types.py b/src/mailman/database/types.py index 81721781d..a6f0b32ca 100644 --- a/src/mailman/database/types.py +++ b/src/mailman/database/types.py @@ -49,9 +49,7 @@ class Enum(TypeDecorator): def process_bind_param(self, value, dialect): if value is None: return None - if not isinstance(value, self.enum): - raise ValueError("{} must be a value of the {} enum".format( - value, self.enum.__name__)) + return value.value diff --git a/src/mailman/model/autorespond.py b/src/mailman/model/autorespond.py index 17fe5fadc..c3aff174a 100644 --- a/src/mailman/model/autorespond.py +++ b/src/mailman/model/autorespond.py @@ -28,6 +28,7 @@ __all__ = [ from sqlalchemy import (Column, Integer, String, Unicode, ForeignKey, Date) +from sqlalchemy import desc from sqlalchemy.orm import relationship from zope.interface import implementer @@ -95,5 +96,5 @@ class AutoResponseSet: address = address, mailing_list = self._mailing_list, response_type = response_type - ).order_by(Desc(AutoResponseRecord.date_sent)) + ).order_by(desc(AutoResponseRecord.date_sent)) return (None if results.count() == 0 else results.first()) diff --git a/src/mailman/model/bans.py b/src/mailman/model/bans.py index d0f3b2519..fbbecaebd 100644 --- a/src/mailman/model/bans.py +++ b/src/mailman/model/bans.py @@ -75,7 +75,7 @@ class BanManager: ban = store.query(Ban).filter_by(email=email, list_id=self._list_id).first() if ban is not None: - store.remove(ban) + store.delete(ban) @dbconnection def is_banned(self, store, email): diff --git a/src/mailman/model/user.py b/src/mailman/model/user.py index 12f4643f1..37fb29e65 100644 --- a/src/mailman/model/user.py +++ b/src/mailman/model/user.py @@ -62,13 +62,16 @@ class User(Model): addresses = relationship('Address', backref='user', - foreign_keys='[Address.user_id]') + primaryjoin= + id==Address.user_id) _preferred_address_id = Column(Integer, ForeignKey('address.id', use_alter=True, - name='prefered_address_id')) + name='_preferred_address')) _preferred_address = relationship('Address', - foreign_keys=[_preferred_address_id]) + primaryjoin= + _preferred_address_id==Address.id, + post_update=True) preferences_id = Column(Integer, ForeignKey('preferences.id')) preferences = relationship('Preferences', diff --git a/src/mailman/utilities/importer.py b/src/mailman/utilities/importer.py index cf22af24f..9856a8223 100644 --- a/src/mailman/utilities/importer.py +++ b/src/mailman/utilities/importer.py @@ -198,6 +198,14 @@ NAME_MAPPINGS = dict( send_welcome_msg='send_welcome_message', ) +# Datetime Fields that need a type conversion to python datetime +# object for SQLite database. +DATETIME_OBJECTS = [ + 'created_at', + 'digest_last_sent_at', + 'last_post_time', +] + EXCLUDES = set(( 'digest_members', 'members', @@ -217,6 +225,9 @@ def import_config_pck(mlist, config_dict): # Some attributes must not be directly imported. if key in EXCLUDES: continue + # Created at must not be set, it needs a type conversion + if key in DATETIME_OBJECTS: + continue # Some attributes from Mailman 2 were renamed in Mailman 3. key = NAME_MAPPINGS.get(key, key) # Handle the simple case where the key is an attribute of the @@ -238,6 +249,15 @@ def import_config_pck(mlist, config_dict): except (TypeError, KeyError): print('Type conversion error for key "{}": {}'.format( key, value), file=sys.stderr) + for key in DATETIME_OBJECTS: + try: + value = datetime.datetime.utcfromtimestamp(config_dict[key]) + except KeyError: + continue + if key == 'last_post_time': + setattr(mlist, 'last_post_at', value) + continue + setattr(mlist, key, value) # Handle the archiving policy. In MM2.1 there were two boolean options # but only three of the four possible states were valid. Now there's just # an enum. -- cgit v1.2.3-70-g09d2 From c339f06cca6ddf1d28cde2614a94c2a0c905957a Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Fri, 19 Sep 2014 22:41:56 +0530 Subject: * remove some unused code * add left out documentation * remov super().__init__() calls in models as it was useless now. * remove schema_migrate func in mailman/database/base.py --- src/mailman/database/base.py | 57 ++++++++++++++++++++----------------------- src/mailman/database/model.py | 17 ------------- src/mailman/model/address.py | 1 - src/mailman/model/message.py | 1 - src/mailman/model/pending.py | 1 - src/mailman/model/requests.py | 1 - src/mailman/model/uid.py | 1 - src/mailman/model/user.py | 1 - src/mailman/model/version.py | 1 - 9 files changed, 26 insertions(+), 55 deletions(-) (limited to 'src/mailman/database') diff --git a/src/mailman/database/base.py b/src/mailman/database/base.py index 5eec853d6..f379b3124 100644 --- a/src/mailman/database/base.py +++ b/src/mailman/database/base.py @@ -49,9 +49,12 @@ NL = '\n' class SABaseDatabase: """The database base class for use with SQLAlchemy. - Use this as a base class for your DB_Specific derived classes. + Use this as a base class for your DB-Specific derived classes. """ - TAG='' + # Tag used to distinguish the database being used. Override this in base + # classes. + + TAG = '' def __init__(self): self.url = None @@ -59,17 +62,18 @@ class SABaseDatabase: self.transaction = None def begin(self): + """See `IDatabase`.""" + # SA does this for us. pass def commit(self): + """See `IDatabase`.""" self.store.commit() def abort(self): + """See `IDatabase`.""" self.store.rollback() - def _prepare(self, url): - pass - def _database_exists(self): """Return True if the database exists and is initialized. @@ -97,11 +101,27 @@ class SABaseDatabase: database-specific post-removal cleanup. """ pass + def initialize(self, debug=None): + """See `IDatabase`""" + # Calculate the engine url url = expand(config.database.url, config.paths) log.debug('Database url: %s', url) + # XXX By design of SQLite, database file creation does not honor + # umask. See their ticket #1193: + # http://www.sqlite.org/cvstrac/tktview?tn=1193,31 + # + # This sucks for us because the mailman.db file /must/ be group + # writable, however even though we guarantee our umask is 002 here, it + # still gets created without the necessary g+w permission, due to + # SQLite's policy. This should only affect SQLite engines because its + # the only one that creates a little file on the local file system. + # This kludges around their bug by "touch"ing the database file before + # SQLite has any chance to create it, thus honoring the umask and + # ensuring the right permissions. We only try to do this for SQLite + # engines, and yes, we could have chmod'd the file after the fact, but + # half dozen and all... self.url = url - self._prepare(url) self.engine = create_engine(url) session = sessionmaker(bind=self.engine) self.store = session() @@ -133,31 +153,6 @@ class SABaseDatabase: if statement.strip() != '': store.execute(statement + ';') - def load_schema(self, store, version, filename, module_path): - """Load the schema from a file. - - This is a helper method for migration classes to call. - - :param store: The Storm store to load the schema into. - :type store: storm.locals.Store` - :param version: The schema version identifier of the form - YYYYMMDDHHMMSS. - :type version: string - :param filename: The file name containing the schema to load. Pass - `None` if there is no schema file to load. - :type filename: string - :param module_path: The fully qualified Python module path to the - migration module being loaded. This is used to record information - for use by the test suite. - :type module_path: string - """ - if filename is not None: - contents = resource_string('mailman.database.schema', filename) - self.load_sql(store, contents) - # Add a marker that indicates the migration version being applied. - store.add(Version(component='schema', version=version)) - - @staticmethod def _make_temporary(): diff --git a/src/mailman/database/model.py b/src/mailman/database/model.py index dedd7a34b..d86ebb80e 100644 --- a/src/mailman/database/model.py +++ b/src/mailman/database/model.py @@ -35,23 +35,6 @@ from mailman.config import config class ModelMeta(object): """Do more magic on table classes.""" - _class_registry = set() - - def __init__(self, name, bases, dict): - # Before we let the base class do it's thing, force an __tablename__ - # property to enforce our table naming convention. - self.__tablename__ = name.lower() - # super(ModelMeta, self).__init__(name, bases, dict) - # Register the model class so that it can be more easily cleared. - # This is required by the test framework so that the corresponding - # table can be reset between tests. - # - # The PRESERVE flag indicates whether the table should be reset or - # not. We have to handle the actual Model base class explicitly - # because it does not correspond to a table in the database. - if not getattr(self, 'PRESERVE', False) and name != 'Model': - ModelMeta._class_registry.add(self) - @staticmethod def _reset(db): meta = Model.metadata diff --git a/src/mailman/model/address.py b/src/mailman/model/address.py index fbe862829..7203a31a5 100644 --- a/src/mailman/model/address.py +++ b/src/mailman/model/address.py @@ -60,7 +60,6 @@ class Address(Model): backref=backref('Address', uselist=False)) def __init__(self, email, display_name): - super(Address, self).__init__() getUtility(IEmailValidator).validate(email) lower_case = email.lower() self.email = lower_case diff --git a/src/mailman/model/message.py b/src/mailman/model/message.py index b153d4909..39f33aa89 100644 --- a/src/mailman/model/message.py +++ b/src/mailman/model/message.py @@ -47,7 +47,6 @@ class Message(Model): @dbconnection def __init__(self, store, message_id, message_id_hash, path): - super(Message, self).__init__() self.message_id = message_id self.message_id_hash = message_id_hash self.path = path diff --git a/src/mailman/model/pending.py b/src/mailman/model/pending.py index 30aae074c..97d394721 100644 --- a/src/mailman/model/pending.py +++ b/src/mailman/model/pending.py @@ -71,7 +71,6 @@ class Pended(Model): __tablename__ = 'pended' def __init__(self, token, expiration_date): - super(Pended, self).__init__() self.token = token self.expiration_date = expiration_date diff --git a/src/mailman/model/requests.py b/src/mailman/model/requests.py index 1b72f78f3..88ad0e407 100644 --- a/src/mailman/model/requests.py +++ b/src/mailman/model/requests.py @@ -153,7 +153,6 @@ class _Request(Model): mailing_list = relationship('MailingList') def __init__(self, key, request_type, mailing_list, data_hash): - super(_Request, self).__init__() self.key = key self.request_type = request_type self.mailing_list = mailing_list diff --git a/src/mailman/model/uid.py b/src/mailman/model/uid.py index 6486089fa..77f1b59bb 100644 --- a/src/mailman/model/uid.py +++ b/src/mailman/model/uid.py @@ -54,7 +54,6 @@ class UID(Model): @dbconnection def __init__(self, store, uid): - super(UID, self).__init__() self.uid = uid store.add(self) diff --git a/src/mailman/model/user.py b/src/mailman/model/user.py index 37fb29e65..cd47a5dac 100644 --- a/src/mailman/model/user.py +++ b/src/mailman/model/user.py @@ -79,7 +79,6 @@ class User(Model): @dbconnection def __init__(self, store, display_name=None, preferences=None): - super(User, self).__init__() self._created_on = date_factory.now() user_id = uid_factory.new_uid() assert store.query(User).filter_by(_user_id=user_id).count() == 0, ( diff --git a/src/mailman/model/version.py b/src/mailman/model/version.py index 8dc0d4e6c..95cf03dac 100644 --- a/src/mailman/model/version.py +++ b/src/mailman/model/version.py @@ -43,6 +43,5 @@ class Version(Model): PRESERVE = True def __init__(self, component, version): - super(Version, self).__init__() self.component = component self.version = version -- cgit v1.2.3-70-g09d2