diff options
| author | Barry Warsaw | 2008-12-23 17:33:37 -0500 |
|---|---|---|
| committer | Barry Warsaw | 2008-12-23 17:33:37 -0500 |
| commit | d4de7996e6d4fb5db04dfed3b3fd12747622b164 (patch) | |
| tree | da5485447a128a0dd857a2a492168deec838362b | |
| parent | fd600d3952393dc9808fefb9be871f78cdbdff39 (diff) | |
| download | mailman-d4de7996e6d4fb5db04dfed3b3fd12747622b164.tar.gz mailman-d4de7996e6d4fb5db04dfed3b3fd12747622b164.tar.zst mailman-d4de7996e6d4fb5db04dfed3b3fd12747622b164.zip | |
Use my lazr.config megamerge branch for now, even though it's still under
development.
Completely rework the way switchboards and queue runners are initialized,
i.e. driven from the configuration file instead of hard coded.
The various queue runner directories are no longer available thorugh the
config object directly. Get them from config.switchboards.
Provide minimal mailman.cfg and testing.cfg configuration files.
Neuter styles for now until they can be consolidated with lazr.config.
Diffstat (limited to '')
28 files changed, 366 insertions, 358 deletions
diff --git a/buildout.cfg b/buildout.cfg index 8c03f89d3..d4ba46351 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -4,7 +4,7 @@ parts = tags test unzip = true -develop = . +develop = . /Users/barry/projects/lazr/megamerge [interpreter] recipe = zc.recipe.egg diff --git a/mailman/config/config.py b/mailman/config/config.py index fd4345203..f3bbed316 100644 --- a/mailman/config/config.py +++ b/mailman/config/config.py @@ -29,13 +29,11 @@ import errno import logging from StringIO import StringIO -from lazr.config import ConfigSchema -from lazr.config.interfaces import NoCategoryError -from pkg_resources import resource_filename +from lazr.config import ConfigSchema, as_boolean +from pkg_resources import resource_string from mailman import Defaults from mailman import version -from mailman.config.helpers import as_boolean from mailman.core import errors from mailman.domain import Domain from mailman.languages import LanguageManager @@ -52,6 +50,7 @@ class Configuration(object): self.domains = {} # email host -> IDomain self.qrunners = {} self.qrunner_shortcuts = {} + self.switchboards = {} self.languages = LanguageManager() self.QFILE_SCHEMA_VERSION = version.QFILE_SCHEMA_VERSION self._config = None @@ -65,7 +64,11 @@ class Configuration(object): def _clear(self): """Clear the cached configuration variables.""" + # First, stop all registered qrunners. + for runner in self.qrunners.values(): + runner.stop() self.domains.clear() + self.switchboards.clear() self.qrunners.clear() self.qrunner_shortcuts.clear() self.languages = LanguageManager() @@ -76,13 +79,16 @@ class Configuration(object): def load(self, filename=None): """Load the configuration from the schema and config files.""" - schema_file = resource_filename('mailman.config', 'schema.cfg') - schema = ConfigSchema(schema_file) - # If a configuration file was given, load it now too. - if filename is None: - self._config = schema.loadFile(StringIO(''), '<default>') - else: - self._config = schema.load(filename) + schema_string = resource_string('mailman.config', 'schema.cfg') + schema = ConfigSchema('schema.cfg', StringIO(schema_string)) + # If a configuration file was given, load it now too. First, load the + # absolute minimum default configuration, then if a configuration + # filename was given by the user, push it. + config_string = resource_string('mailman.config', 'mailman.cfg') + self._config = schema.loadFile(StringIO(config_string), 'mailman.cfg') + if filename is not None: + with open(filename) as user_config: + self._config.push(user_config.read()) self._post_process() def push(self, config_name, config_string): @@ -100,10 +106,7 @@ class Configuration(object): def _post_process(self): """Perform post-processing after loading the configuration files.""" # Set up the domains. - try: - domains = self._config.getByCategory('domain') - except NoCategoryError: - domains = [] + domains = self._config.getByCategory('domain', []) for section in domains: domain = Domain(section.email_host, section.base_url, section.description, section.contact_address) @@ -117,79 +120,45 @@ class Configuration(object): # We'll do the reverse mappings on-demand. There shouldn't be too # many virtual hosts that it will really matter that much. self.domains[domain.email_host] = domain - # Set up the queue runners. - try: - qrunners = self._config.getByCategory('qrunner') - except NoCategoryError: - qrunners = [] - for section in qrunners: - if not as_boolean(section.start): - continue - name = section['class'] - self.qrunners[name] = section.count - # Calculate the queue runner shortcut name. - classname = name.rsplit('.', 1)[1] - if classname.endswith('Runner'): - shortname = classname[:-6].lower() - else: - shortname = classname - self.qrunner_shortcuts[shortname] = section['class'] # Set up directories. self.BIN_DIR = os.path.abspath(os.path.dirname(sys.argv[0])) - VAR_DIR = self._config.mailman.var_dir + self.VAR_DIR = var_dir = self._config.mailman.var_dir # Now that we've loaded all the configuration files we're going to # load, set up some useful directories. join = os.path.join - self.LIST_DATA_DIR = join(VAR_DIR, 'lists') - self.LOG_DIR = join(VAR_DIR, 'logs') - self.LOCK_DIR = lockdir = join(VAR_DIR, 'locks') - self.DATA_DIR = datadir = join(VAR_DIR, 'data') - self.ETC_DIR = etcdir = join(VAR_DIR, 'etc') - self.SPAM_DIR = join(VAR_DIR, 'spam') - self.EXT_DIR = join(VAR_DIR, 'ext') - self.PUBLIC_ARCHIVE_FILE_DIR = join(VAR_DIR, 'archives', 'public') - self.PRIVATE_ARCHIVE_FILE_DIR = join(VAR_DIR, 'archives', 'private') - # Directories used by the qrunner subsystem - self.QUEUE_DIR = qdir = join(VAR_DIR, 'qfiles') - self.ARCHQUEUE_DIR = join(qdir, 'archive') - self.BADQUEUE_DIR = join(qdir, 'bad') - self.BOUNCEQUEUE_DIR = join(qdir, 'bounces') - self.CMDQUEUE_DIR = join(qdir, 'commands') - self.INQUEUE_DIR = join(qdir, 'in') - self.MAILDIR_DIR = join(qdir, 'maildir') - self.NEWSQUEUE_DIR = join(qdir, 'news') - self.OUTQUEUE_DIR = join(qdir, 'out') - self.PIPELINEQUEUE_DIR = join(qdir, 'pipeline') - self.RETRYQUEUE_DIR = join(qdir, 'retry') - self.SHUNTQUEUE_DIR = join(qdir, 'shunt') - self.VIRGINQUEUE_DIR = join(qdir, 'virgin') - self.MESSAGES_DIR = join(VAR_DIR, 'messages') + self.LIST_DATA_DIR = join(var_dir, 'lists') + self.LOG_DIR = join(var_dir, 'logs') + self.LOCK_DIR = lockdir = join(var_dir, 'locks') + self.DATA_DIR = datadir = join(var_dir, 'data') + self.ETC_DIR = etcdir = join(var_dir, 'etc') + self.SPAM_DIR = join(var_dir, 'spam') + self.EXT_DIR = join(var_dir, 'ext') + self.QUEUE_DIR = join(var_dir, 'qfiles') + self.MESSAGES_DIR = join(var_dir, 'messages') + self.PUBLIC_ARCHIVE_FILE_DIR = join(var_dir, 'archives', 'public') + self.PRIVATE_ARCHIVE_FILE_DIR = join(var_dir, 'archives', 'private') # Other useful files self.PIDFILE = join(datadir, 'master-qrunner.pid') self.SITE_PW_FILE = join(datadir, 'adm.pw') self.LISTCREATOR_PW_FILE = join(datadir, 'creator.pw') self.CONFIG_FILE = join(etcdir, 'mailman.cfg') self.LOCK_FILE = join(lockdir, 'master-qrunner') + # Set up the switchboards. + from mailman.queue import Switchboard + Switchboard.initialize() # Set up all the languages. - try: - languages = self._config.getByCategory('language') - except NoCategoryError: - languages = [] - for section in languages: - code = section.name.split('.')[1] - self.languages.add_language(code, section.description, - section.charset, section.enable) + languages = self._config.getByCategory('language', []) + for language in languages: + code = language.name.split('.')[1] + self.languages.add_language(code, language.description, + language.charset, language.enabled) # Always enable the server default language, which must be defined. self.languages.enable_language(self._config.mailman.default_language) @property def logger_configs(self): """Return all log config sections.""" - try: - loggers = self._config.getByCategory('logging') - except NoCategoryError: - loggers = [] - return loggers + return self._config.getByCategory('logging', []) @property def paths(self): @@ -208,14 +177,16 @@ class Configuration(object): raise @property + def qrunner_configs(self): + for section in self._config.getByCategory('qrunner', []): + yield section + + @property def header_matches(self): """Iterate over all spam matching headers. Values are 3-tuples of (header, pattern, chain) """ - try: - matches = self._config.getByCategory('spam.headers') - except NoCategoryError: - matches = [] + matches = self._config.getByCategory('spam.headers', []) for match in matches: yield (matches.header, matches.pattern, matches.chain) diff --git a/mailman/config/helpers.py b/mailman/config/helpers.py deleted file mode 100644 index 5c5cecc67..000000000 --- a/mailman/config/helpers.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright (C) 2008 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -"""Configuration helpers.""" - -__metaclass__ = type -__all__ = [ - 'as_boolean', - 'as_log_level', - ] - - -import logging - -# XXX BAW 20-Dec-2008 Donate these back to the lazr.config project. - - - -def as_boolean(value): - """Turn a string into a boolean. - - :param value: A string with one of the following values - (case-insensitive): true, yes, 1, on, enable, enabled (for True), or - false, no, 0, off, disable, disabled (for False). Everything else is - an error. - :type value: string - :return: True or False. - :rtype: boolean - """ - value = value.lower() - if value in ('true', 'yes', '1', 'on', 'enabled', 'enable'): - return True - if value in ('false', 'no', '0', 'off', 'disabled', 'disable'): - return False - raise ValueError('Invalid boolean value: %s' % value) - - - -def as_log_level(value): - """Turn a string into a log level. - - :param value: A string with a value (case-insensitive) equal to one of the - symbolic logging levels. - :type value: string - :return: A logging level constant. - :rtype: int - """ - value = value.upper() - return getattr(logging, value) diff --git a/mailman/config/mailman.cfg b/mailman/config/mailman.cfg new file mode 100644 index 000000000..126017790 --- /dev/null +++ b/mailman/config/mailman.cfg @@ -0,0 +1,66 @@ +# Copyright (C) 2008 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +# This is the absolute bare minimum base configuration file. User supplied +# configurations are pushed onto this. + +[language.en] + +[qrunner.archive] +class: mailman.queue.archive.ArchiveRunner + +[qrunner.bad] +class: mailman.queue.fake.BadRunner +# The shunt runner is just a placeholder for its switchboard. +start: no + +[qrunner.bounces] +class: mailman.queue.bounce.BounceRunner + +[qrunner.commands] +class: mailman.queue.command.CommandRunner + +[qrunner.in] +class: mailman.queue.incoming.IncomingRunner + +[qrunner.lmtp] +class: mailman.queue.lmtp.LMTPRunner + +[qrunner.maildir] +class: mailman.queue.maildir.MaildirRunner +# This is still experimental. +start: no + +[qrunner.news] +class: mailman.queue.news.NewsRunner + +[qrunner.out] +class: mailman.queue.outgoing.OutgoingRunner + +[qrunner.pipeline] +class: mailman.queue.pipeline.PipelineRunner + +[qrunner.retry] +class: mailman.queue.retry.RetryRunner + +[qrunner.shunt] +class: mailman.queue.fake.ShuntRunner +# The shunt runner is just a placeholder for its switchboard. +start: no + +[qrunner.virgin] +class: mailman.queue.virgin.VirginRunner diff --git a/mailman/config/schema.cfg b/mailman/config/schema.cfg index 5a50b6845..0a27c28f4 100644 --- a/mailman/config/schema.cfg +++ b/mailman/config/schema.cfg @@ -18,9 +18,7 @@ # This is the GNU Mailman configuration schema. It defines the default # configuration options for the core system and plugins. It uses ini-style # formats under the lazr.config regime to define all system configuration -# options. See <https://launchpad.net/lazr.config> for details. You can -# override the defaults by creating a mailman.cfg file in your etc directory. -# See mailman.cfg.sample as an example. +# options. See <https://launchpad.net/lazr.config> for details. [mailman] # This address is the "site owner" address. Certain messages which must be @@ -60,43 +58,29 @@ use_envelope_sender: no email_commands_max_lines: 10 -[qrunner.template] +[qrunner.master] # Define which process queue runners, and how many of them, to start. -class: mailman.queue.runner.Runner -count: 1 -start: yes -max_restarts: 10 - -[qrunner.archive] -class: mailman.queue.archive.ArchiveRunner - -[qrunner.bounce] -class: mailman.queue.bounce.BounceRunner - -[qrunner.command] -class: mailman.queue.command.CommandRunner - -[qrunner.incoming] -class: mailman.queue.incoming.IncomingRunner -[qrunner.news] -class: mailman.queue.news.NewsRunner - -[qrunner.outgoing] -class: mailman.queue.outgoing.OutgoingRunner +# The full import path to the class for this queue runner. +class: mailman.queue.runner.Runner -[qrunner.pipeline] -class: mailman.queue.pipeline.PipelineRunner +# The directory path that this queue runner scans. +path: $VAR_DIR/qfiles/$name -[qrunner.retry] -class: mailman.queue.retry.RetryRunner +# The number of parallel queue runners. This must be a power of 2. +instances: 1 -[qrunner.virgin] -class: mailman.queue.virgin.VirginRunner +# Whether to start this queue runner or not. +start: yes -[qrunner.lmtp] -class: mailman.queue.lmtp.LMTPRunner +# The maximum number of restarts for this queue runner. When the runner exits +# because of an error or other unexpected problem, it is automatically +# restarted, until the maximum number of restarts has been reached. +max_restarts: 10 +# The sleep interval for the queue runner. It wakes up once every interval to +# process the files in its slice of the queue directory. +sleep_time: 1s [database] # Use this to set the Storm database engine URL. You generally have one @@ -231,11 +215,7 @@ description: English (USA) # And the default character set for the language. charset: us-ascii # Whether the language is enabled or not. -enable: yes - -[language.en] -description: English (USA) -charset: us-ascii +enabled: yes [spam.headers.template] diff --git a/mailman/core/initialize.py b/mailman/core/initialize.py index 16dcecf94..39e924545 100644 --- a/mailman/core/initialize.py +++ b/mailman/core/initialize.py @@ -93,6 +93,7 @@ def initialize_2(debug=False): from mailman.core.chains import initialize as initialize_chains from mailman.core.pipelines import initialize as initialize_pipelines from mailman.core.rules import initialize as initialize_rules + # Order here is somewhat important. initialize_archivers() initialize_rules() initialize_chains() diff --git a/mailman/core/logging.py b/mailman/core/logging.py index 608b59c2c..ed400215b 100644 --- a/mailman/core/logging.py +++ b/mailman/core/logging.py @@ -25,8 +25,9 @@ import codecs import logging import ConfigParser +from lazr.config import as_boolean, as_log_level + from mailman.config import config -from mailman.config.helpers import as_boolean, as_log_level _handlers = [] diff --git a/mailman/core/styles.py b/mailman/core/styles.py index d264143f5..76428a955 100644 --- a/mailman/core/styles.py +++ b/mailman/core/styles.py @@ -23,12 +23,15 @@ __all__ = [ 'style_manager', ] +# XXX Styles need to be reconciled with lazr.config. + import datetime from operator import attrgetter from zope.interface import implements from zope.interface.verify import verifyObject +from mailman import Defaults from mailman import Utils from mailman.config import config from mailman.core.plugins import get_plugins @@ -54,75 +57,75 @@ class DefaultStyle: # Most of these were ripped from the old MailList.InitVars() method. mlist.volume = 1 mlist.post_id = 1 - mlist.new_member_options = config.DEFAULT_NEW_MEMBER_OPTIONS + mlist.new_member_options = Defaults.DEFAULT_NEW_MEMBER_OPTIONS # This stuff is configurable mlist.real_name = mlist.list_name.capitalize() mlist.respond_to_post_requests = True - mlist.advertised = config.DEFAULT_LIST_ADVERTISED - mlist.max_num_recipients = config.DEFAULT_MAX_NUM_RECIPIENTS - mlist.max_message_size = config.DEFAULT_MAX_MESSAGE_SIZE - mlist.reply_goes_to_list = config.DEFAULT_REPLY_GOES_TO_LIST + mlist.advertised = Defaults.DEFAULT_LIST_ADVERTISED + mlist.max_num_recipients = Defaults.DEFAULT_MAX_NUM_RECIPIENTS + mlist.max_message_size = Defaults.DEFAULT_MAX_MESSAGE_SIZE + mlist.reply_goes_to_list = Defaults.DEFAULT_REPLY_GOES_TO_LIST mlist.reply_to_address = u'' - mlist.first_strip_reply_to = config.DEFAULT_FIRST_STRIP_REPLY_TO - mlist.admin_immed_notify = config.DEFAULT_ADMIN_IMMED_NOTIFY + mlist.first_strip_reply_to = Defaults.DEFAULT_FIRST_STRIP_REPLY_TO + mlist.admin_immed_notify = Defaults.DEFAULT_ADMIN_IMMED_NOTIFY mlist.admin_notify_mchanges = ( - config.DEFAULT_ADMIN_NOTIFY_MCHANGES) + Defaults.DEFAULT_ADMIN_NOTIFY_MCHANGES) mlist.require_explicit_destination = ( - config.DEFAULT_REQUIRE_EXPLICIT_DESTINATION) - mlist.acceptable_aliases = config.DEFAULT_ACCEPTABLE_ALIASES - mlist.send_reminders = config.DEFAULT_SEND_REMINDERS - mlist.send_welcome_msg = config.DEFAULT_SEND_WELCOME_MSG - mlist.send_goodbye_msg = config.DEFAULT_SEND_GOODBYE_MSG + Defaults.DEFAULT_REQUIRE_EXPLICIT_DESTINATION) + mlist.acceptable_aliases = Defaults.DEFAULT_ACCEPTABLE_ALIASES + mlist.send_reminders = Defaults.DEFAULT_SEND_REMINDERS + mlist.send_welcome_msg = Defaults.DEFAULT_SEND_WELCOME_MSG + mlist.send_goodbye_msg = Defaults.DEFAULT_SEND_GOODBYE_MSG mlist.bounce_matching_headers = ( - config.DEFAULT_BOUNCE_MATCHING_HEADERS) + Defaults.DEFAULT_BOUNCE_MATCHING_HEADERS) mlist.header_matches = [] - mlist.anonymous_list = config.DEFAULT_ANONYMOUS_LIST + mlist.anonymous_list = Defaults.DEFAULT_ANONYMOUS_LIST mlist.description = u'' mlist.info = u'' mlist.welcome_msg = u'' mlist.goodbye_msg = u'' - mlist.subscribe_policy = config.DEFAULT_SUBSCRIBE_POLICY - mlist.subscribe_auto_approval = config.DEFAULT_SUBSCRIBE_AUTO_APPROVAL - mlist.unsubscribe_policy = config.DEFAULT_UNSUBSCRIBE_POLICY - mlist.private_roster = config.DEFAULT_PRIVATE_ROSTER - mlist.obscure_addresses = config.DEFAULT_OBSCURE_ADDRESSES - mlist.admin_member_chunksize = config.DEFAULT_ADMIN_MEMBER_CHUNKSIZE - mlist.administrivia = config.DEFAULT_ADMINISTRIVIA - mlist.preferred_language = config.DEFAULT_SERVER_LANGUAGE + mlist.subscribe_policy = Defaults.DEFAULT_SUBSCRIBE_POLICY + mlist.subscribe_auto_approval = Defaults.DEFAULT_SUBSCRIBE_AUTO_APPROVAL + mlist.unsubscribe_policy = Defaults.DEFAULT_UNSUBSCRIBE_POLICY + mlist.private_roster = Defaults.DEFAULT_PRIVATE_ROSTER + mlist.obscure_addresses = Defaults.DEFAULT_OBSCURE_ADDRESSES + mlist.admin_member_chunksize = Defaults.DEFAULT_ADMIN_MEMBER_CHUNKSIZE + mlist.administrivia = Defaults.DEFAULT_ADMINISTRIVIA + mlist.preferred_language = Defaults.DEFAULT_SERVER_LANGUAGE mlist.include_rfc2369_headers = True mlist.include_list_post_header = True - mlist.filter_mime_types = config.DEFAULT_FILTER_MIME_TYPES - mlist.pass_mime_types = config.DEFAULT_PASS_MIME_TYPES + mlist.filter_mime_types = Defaults.DEFAULT_FILTER_MIME_TYPES + mlist.pass_mime_types = Defaults.DEFAULT_PASS_MIME_TYPES mlist.filter_filename_extensions = ( - config.DEFAULT_FILTER_FILENAME_EXTENSIONS) - mlist.pass_filename_extensions = config.DEFAULT_PASS_FILENAME_EXTENSIONS - mlist.filter_content = config.DEFAULT_FILTER_CONTENT - mlist.collapse_alternatives = config.DEFAULT_COLLAPSE_ALTERNATIVES + Defaults.DEFAULT_FILTER_FILENAME_EXTENSIONS) + mlist.pass_filename_extensions = Defaults.DEFAULT_PASS_FILENAME_EXTENSIONS + mlist.filter_content = Defaults.DEFAULT_FILTER_CONTENT + mlist.collapse_alternatives = Defaults.DEFAULT_COLLAPSE_ALTERNATIVES mlist.convert_html_to_plaintext = ( - config.DEFAULT_CONVERT_HTML_TO_PLAINTEXT) - mlist.filter_action = config.DEFAULT_FILTER_ACTION + Defaults.DEFAULT_CONVERT_HTML_TO_PLAINTEXT) + mlist.filter_action = Defaults.DEFAULT_FILTER_ACTION # Digest related variables - mlist.digestable = config.DEFAULT_DIGESTABLE - mlist.digest_is_default = config.DEFAULT_DIGEST_IS_DEFAULT - mlist.mime_is_default_digest = config.DEFAULT_MIME_IS_DEFAULT_DIGEST - mlist.digest_size_threshhold = config.DEFAULT_DIGEST_SIZE_THRESHHOLD - mlist.digest_send_periodic = config.DEFAULT_DIGEST_SEND_PERIODIC - mlist.digest_header = config.DEFAULT_DIGEST_HEADER - mlist.digest_footer = config.DEFAULT_DIGEST_FOOTER - mlist.digest_volume_frequency = config.DEFAULT_DIGEST_VOLUME_FREQUENCY + mlist.digestable = Defaults.DEFAULT_DIGESTABLE + mlist.digest_is_default = Defaults.DEFAULT_DIGEST_IS_DEFAULT + mlist.mime_is_default_digest = Defaults.DEFAULT_MIME_IS_DEFAULT_DIGEST + mlist.digest_size_threshhold = Defaults.DEFAULT_DIGEST_SIZE_THRESHHOLD + mlist.digest_send_periodic = Defaults.DEFAULT_DIGEST_SEND_PERIODIC + mlist.digest_header = Defaults.DEFAULT_DIGEST_HEADER + mlist.digest_footer = Defaults.DEFAULT_DIGEST_FOOTER + mlist.digest_volume_frequency = Defaults.DEFAULT_DIGEST_VOLUME_FREQUENCY mlist.one_last_digest = {} mlist.digest_members = {} mlist.next_digest_number = 1 - mlist.nondigestable = config.DEFAULT_NONDIGESTABLE + mlist.nondigestable = Defaults.DEFAULT_NONDIGESTABLE mlist.personalize = Personalization.none # New sender-centric moderation (privacy) options mlist.default_member_moderation = ( - config.DEFAULT_DEFAULT_MEMBER_MODERATION) + Defaults.DEFAULT_DEFAULT_MEMBER_MODERATION) # Archiver - mlist.archive = config.DEFAULT_ARCHIVE - mlist.archive_private = config.DEFAULT_ARCHIVE_PRIVATE + mlist.archive = Defaults.DEFAULT_ARCHIVE + mlist.archive_private = Defaults.DEFAULT_ARCHIVE_PRIVATE mlist.archive_volume_frequency = ( - config.DEFAULT_ARCHIVE_VOLUME_FREQUENCY) + Defaults.DEFAULT_ARCHIVE_VOLUME_FREQUENCY) mlist.emergency = False mlist.member_moderation_action = Action.hold mlist.member_moderation_notice = u'' @@ -130,9 +133,9 @@ class DefaultStyle: mlist.hold_these_nonmembers = [] mlist.reject_these_nonmembers = [] mlist.discard_these_nonmembers = [] - mlist.forward_auto_discards = config.DEFAULT_FORWARD_AUTO_DISCARDS + mlist.forward_auto_discards = Defaults.DEFAULT_FORWARD_AUTO_DISCARDS mlist.generic_nonmember_action = ( - config.DEFAULT_GENERIC_NONMEMBER_ACTION) + Defaults.DEFAULT_GENERIC_NONMEMBER_ACTION) mlist.nonmember_rejection_notice = u'' # Ban lists mlist.ban_list = [] @@ -140,9 +143,9 @@ class DefaultStyle: # 2-tuple of the date of the last autoresponse and the number of # autoresponses sent on that date. mlist.hold_and_cmd_autoresponses = {} - mlist.subject_prefix = _(config.DEFAULT_SUBJECT_PREFIX) - mlist.msg_header = config.DEFAULT_MSG_HEADER - mlist.msg_footer = config.DEFAULT_MSG_FOOTER + mlist.subject_prefix = _(Defaults.DEFAULT_SUBJECT_PREFIX) + mlist.msg_header = Defaults.DEFAULT_MSG_HEADER + mlist.msg_footer = Defaults.DEFAULT_MSG_FOOTER # Set this to Never if the list's preferred language uses us-ascii, # otherwise set it to As Needed if Utils.GetCharSet(mlist.preferred_language) == 'us-ascii': @@ -150,9 +153,9 @@ class DefaultStyle: else: mlist.encode_ascii_prefixes = 2 # scrub regular delivery - mlist.scrub_nondigest = config.DEFAULT_SCRUB_NONDIGEST + mlist.scrub_nondigest = Defaults.DEFAULT_SCRUB_NONDIGEST # automatic discarding - mlist.max_days_to_hold = config.DEFAULT_MAX_DAYS_TO_HOLD + mlist.max_days_to_hold = Defaults.DEFAULT_MAX_DAYS_TO_HOLD # Autoresponder mlist.autorespond_postings = False mlist.autorespond_admin = False @@ -169,19 +172,19 @@ class DefaultStyle: mlist.admin_responses = {} mlist.request_responses = {} # Bounces - mlist.bounce_processing = config.DEFAULT_BOUNCE_PROCESSING - mlist.bounce_score_threshold = config.DEFAULT_BOUNCE_SCORE_THRESHOLD - mlist.bounce_info_stale_after = config.DEFAULT_BOUNCE_INFO_STALE_AFTER + mlist.bounce_processing = Defaults.DEFAULT_BOUNCE_PROCESSING + mlist.bounce_score_threshold = Defaults.DEFAULT_BOUNCE_SCORE_THRESHOLD + mlist.bounce_info_stale_after = Defaults.DEFAULT_BOUNCE_INFO_STALE_AFTER mlist.bounce_you_are_disabled_warnings = ( - config.DEFAULT_BOUNCE_YOU_ARE_DISABLED_WARNINGS) + Defaults.DEFAULT_BOUNCE_YOU_ARE_DISABLED_WARNINGS) mlist.bounce_you_are_disabled_warnings_interval = ( - config.DEFAULT_BOUNCE_YOU_ARE_DISABLED_WARNINGS_INTERVAL) + Defaults.DEFAULT_BOUNCE_YOU_ARE_DISABLED_WARNINGS_INTERVAL) mlist.bounce_unrecognized_goes_to_list_owner = ( - config.DEFAULT_BOUNCE_UNRECOGNIZED_GOES_TO_LIST_OWNER) + Defaults.DEFAULT_BOUNCE_UNRECOGNIZED_GOES_TO_LIST_OWNER) mlist.bounce_notify_owner_on_disable = ( - config.DEFAULT_BOUNCE_NOTIFY_OWNER_ON_DISABLE) + Defaults.DEFAULT_BOUNCE_NOTIFY_OWNER_ON_DISABLE) mlist.bounce_notify_owner_on_removal = ( - config.DEFAULT_BOUNCE_NOTIFY_OWNER_ON_REMOVAL) + Defaults.DEFAULT_BOUNCE_NOTIFY_OWNER_ON_REMOVAL) # This holds legacy member related information. It's keyed by the # member address, and the value is an object containing the bounce # score, the date of the last received bounce, and a count of the @@ -190,7 +193,7 @@ class DefaultStyle: # New style delivery status mlist.delivery_status = {} # NNTP gateway - mlist.nntp_host = config.DEFAULT_NNTP_HOST + mlist.nntp_host = Defaults.DEFAULT_NNTP_HOST mlist.linked_newsgroup = u'' mlist.gateway_to_news = False mlist.gateway_to_mail = False diff --git a/mailman/database/__init__.py b/mailman/database/__init__.py index 429fea953..65022ffe4 100644 --- a/mailman/database/__init__.py +++ b/mailman/database/__init__.py @@ -23,6 +23,7 @@ __all__ = [ import os from locknix.lockfile import Lock +from lazr.config import as_boolean from pkg_resources import resource_string from storm.locals import create_database, Store from string import Template @@ -32,7 +33,6 @@ from zope.interface import implements import mailman.version from mailman.config import config -from mailman.config.helpers import as_boolean from mailman.database.listmanager import ListManager from mailman.database.messagestore import MessageStore from mailman.database.pending import Pendings diff --git a/mailman/interfaces/runner.py b/mailman/interfaces/runner.py index 28b15968c..8ad54f445 100644 --- a/mailman/interfaces/runner.py +++ b/mailman/interfaces/runner.py @@ -30,9 +30,10 @@ class IRunner(Interface): def stop(): """Stop the queue runner on the next iteration through the loop.""" - QDIR = Attribute('The queue directory. Overridden in subclasses.') + queue_directory = Attribute( + 'The queue directory. Overridden in subclasses.') - SLEEPTIME = Attribute("""\ + sleep_time = Attribute("""\ The number of seconds this queue runner will sleep between iterations through the main loop. If given, overrides `config.QRUNNER_SLEEP_TIME` diff --git a/mailman/queue/__init__.py b/mailman/queue/__init__.py index 27ea83320..5aa72bb14 100644 --- a/mailman/queue/__init__.py +++ b/mailman/queue/__init__.py @@ -32,6 +32,7 @@ __all__ = [ import os +import sys import time import email import errno @@ -43,13 +44,16 @@ import marshal import traceback from cStringIO import StringIO +from lazr.config import as_boolean, as_timedelta +from string import Template from zope.interface import implements -from mailman import i18n from mailman import Message from mailman import Utils +from mailman import i18n from mailman.config import config -from mailman.interfaces import IRunner, ISwitchboard +from mailman.interfaces.runner import IRunner +from mailman.interfaces.switchboard import ISwitchboard # 20 bytes of all bits set, maximum hashlib.sha.digest() value shamax = 0xffffffffffffffffffffffffffffffffffffffffL @@ -57,6 +61,7 @@ shamax = 0xffffffffffffffffffffffffffffffffffffffffL # Small increment to add to time in case two entries have the same time. This # prevents skipping one of two entries with the same time until the next pass. DELTA = .0001 +DOT = '.' elog = logging.getLogger('mailman.error') dlog = logging.getLogger('mailman.debug') @@ -66,11 +71,24 @@ dlog = logging.getLogger('mailman.debug') class Switchboard: implements(ISwitchboard) - def __init__(self, whichq, slice=None, numslices=1, recover=False): + @staticmethod + def initialize(): + """Initialize the global switchboards for input/output.""" + for conf in config.qrunner_configs: + name = conf.name.split('.')[-1] + assert name not in config.switchboards, ( + 'Duplicate qrunner name: %s' % name) + substitutions = config.paths + substitutions['name'] = name + path = Template(conf.path).safe_substitute(substitutions) + config.switchboards[name] = Switchboard(path) + + def __init__(self, queue_directory, + slice=None, numslices=1, recover=False): """Create a switchboard object. - :param whichq: The queue directory. - :type whichq: str + :param queue_directory: The queue directory. + :type queue_directory: str :param slice: The slice number for this switchboard, or None. If not None, it must be [0..`numslices`). :type slice: int or None @@ -80,9 +98,11 @@ class Switchboard: :param recover: True if backup files should be recovered. :type recover: bool """ - self._whichq = whichq + assert (numslices & (numslices - 1)) == 0, ( + 'Not a power of 2: %s' % numslices) + self.queue_directory = queue_directory # Create the directory if it doesn't yet exist. - Utils.makedirs(self._whichq, 0770) + Utils.makedirs(self.queue_directory, 0770) # Fast track for no slices self._lower = None self._upper = None @@ -93,11 +113,6 @@ class Switchboard: if recover: self.recover_backup_files() - @property - def queue_directory(self): - """See `ISwitchboard`.""" - return self._whichq - def enqueue(self, _msg, _metadata=None, **_kws): """See `ISwitchboard`.""" if _metadata is None: @@ -125,7 +140,7 @@ class Switchboard: # and the sha hex digest. rcvtime = data.setdefault('received_time', now) filebase = repr(rcvtime) + '+' + hashlib.sha1(hashfood).hexdigest() - filename = os.path.join(self._whichq, filebase + '.pck') + filename = os.path.join(self.queue_directory, filebase + '.pck') tmpfile = filename + '.tmp' # Always add the metadata schema version number data['version'] = config.QFILE_SCHEMA_VERSION @@ -148,8 +163,8 @@ class Switchboard: def dequeue(self, filebase): """See `ISwitchboard`.""" # Calculate the filename from the given filebase. - filename = os.path.join(self._whichq, filebase + '.pck') - backfile = os.path.join(self._whichq, filebase + '.bak') + filename = os.path.join(self.queue_directory, filebase + '.pck') + backfile = os.path.join(self.queue_directory, filebase + '.bak') # Read the message object and metadata. with open(filename) as fp: # Move the file to the backup file name for processing. If this @@ -172,13 +187,13 @@ class Switchboard: return msg, data def finish(self, filebase, preserve=False): - bakfile = os.path.join(self._whichq, filebase + '.bak') + bakfile = os.path.join(self.queue_directory, filebase + '.bak') try: if preserve: - psvfile = os.path.join(config.SHUNTQUEUE_DIR, - filebase + '.psv') + shunt_dir = config.switchboards['shunt'].queue_directory + psvfile = os.path.join(shunt_dir, filebase + '.psv') # Create the directory if it doesn't yet exist. - Utils.makedirs(config.SHUNTQUEUE_DIR, 0770) + Utils.makedirs(shunt_dir, 0770) os.rename(bakfile, psvfile) else: os.unlink(bakfile) @@ -196,7 +211,7 @@ class Switchboard: times = {} lower = self._lower upper = self._upper - for f in os.listdir(self._whichq): + for f in os.listdir(self.queue_directory): # By ignoring anything that doesn't end in .pck, we ignore # tempfiles and avoid a race condition. filebase, ext = os.path.splitext(f) @@ -220,8 +235,8 @@ class Switchboard: # to exist at the same time, so the move is enough to ensure that our # normal dequeuing process will handle them. for filebase in self.get_files('.bak'): - src = os.path.join(self._whichq, filebase + '.bak') - dst = os.path.join(self._whichq, filebase + '.pck') + src = os.path.join(self.queue_directory, filebase + '.bak') + dst = os.path.join(self.queue_directory, filebase + '.pck') os.rename(src, dst) @@ -229,27 +244,31 @@ class Switchboard: class Runner: implements(IRunner) - QDIR = None - SLEEPTIME = None - - def __init__(self, slice=None, numslices=1): + def __init__(self, name, slice=None): """Create a queue runner. :param slice: The slice number for this queue runner. This is passed directly to the underlying `ISwitchboard` object. :type slice: int or None - :param numslices: The number of slices for this queue. Must be a - power of 2. - :type numslices: int """ - # Create our own switchboard. Don't use the switchboard cache because - # we want to provide slice and numslice arguments. - self._switchboard = Switchboard(self.QDIR, slice, numslices, True) - # Create the shunt switchboard - self._shunt = Switchboard(config.SHUNTQUEUE_DIR) + # Grab the configuration section. + self.name = name + section = getattr(config, 'qrunner.' + name) + substitutions = config.paths + substitutions['name'] = name + self.queue_directory = Template(section.path).safe_substitute( + substitutions) + numslices = int(section.instances) + self.switchboard = Switchboard( + self.queue_directory, slice, numslices, True) + self.sleep_time = as_timedelta(section.sleep_time) + # sleep_time is a timedelta; turn it into a float for time.sleep(). + self.sleep_float = (86400 * self.sleep_time.days + + self.sleep_time.seconds + + self.sleep_time.microseconds / 1000000.0) + self.max_restarts = int(section.max_restarts) + self.start = as_boolean(section.start) self._stop = False - if self.SLEEPTIME is None: - self.SLEEPTIME = config.QRUNNER_SLEEP_TIME def __repr__(self): return '<%s at %s>' % (self.__class__.__name__, id(self)) @@ -286,13 +305,13 @@ class Runner: dlog.debug('[%s] starting oneloop', me) # List all the files in our queue directory. The switchboard is # guaranteed to hand us the files in FIFO order. - files = self._switchboard.files + files = self.switchboard.files for filebase in files: dlog.debug('[%s] processing filebase: %s', me, filebase) try: # Ask the switchboard for the message and metadata objects # associated with this queue file. - msg, msgdata = self._switchboard.dequeue(filebase) + msg, msgdata = self.switchboard.dequeue(filebase) except Exception, e: # This used to just catch email.Errors.MessageParseError, but # other problems can occur in message parsing, e.g. @@ -302,14 +321,14 @@ class Runner: self._log(e) elog.error('Skipping and preserving unparseable message: %s', filebase) - self._switchboard.finish(filebase, preserve=True) + self.switchboard.finish(filebase, preserve=True) config.db.abort() continue try: dlog.debug('[%s] processing onefile', me) self._process_one_file(msg, msgdata) dlog.debug('[%s] finishing filebase: %s', me, filebase) - self._switchboard.finish(filebase) + self.switchboard.finish(filebase) except Exception, e: # All runners that implement _dispose() must guarantee that # exceptions are caught and dealt with properly. Still, there @@ -319,14 +338,14 @@ class Runner: # intervention. self._log(e) # Put a marker in the metadata for unshunting. - msgdata['whichq'] = self._switchboard.queue_directory + msgdata['whichq'] = self.switchboard.queue_directory # It is possible that shunting can throw an exception, e.g. a # permissions problem or a MemoryError due to a really large # message. Try to be graceful. try: new_filebase = self._shunt.enqueue(msg, msgdata) elog.error('SHUNTING: %s', new_filebase) - self._switchboard.finish(filebase) + self.switchboard.finish(filebase) except Exception, e: # The message wasn't successfully shunted. Log the # exception and try to preserve the original queue entry @@ -335,13 +354,13 @@ class Runner: elog.error( 'SHUNTING FAILED, preserving original entry: %s', filebase) - self._switchboard.finish(filebase, preserve=True) + self.switchboard.finish(filebase, preserve=True) config.db.abort() # Other work we want to do each time through the loop. dlog.debug('[%s] doing periodic', me) self._do_periodic() dlog.debug('[%s] checking short circuit', me) - if self._short_curcuit(): + if self._short_circuit(): dlog.debug('[%s] short circuiting', me) break dlog.debug('[%s] commiting', me) @@ -380,7 +399,7 @@ class Runner: msgdata['lang'] = language keepqueued = self._dispose(mlist, msg, msgdata) if keepqueued: - self._switchboard.enqueue(msg, msgdata) + self.switchboard.enqueue(msg, msgdata) def _log(self, exc): elog.error('Uncaught runner exception: %s', exc) @@ -401,10 +420,10 @@ class Runner: def _snooze(self, filecnt): """See `IRunner`.""" - if filecnt or float(self.SLEEPTIME) <= 0: + if filecnt or self.sleep_float <= 0: return - time.sleep(float(self.SLEEPTIME)) + time.sleep(self.sleep_float) - def _short_curcuit(self): + def _short_circuit(self): """See `IRunner`.""" return self._stop diff --git a/mailman/queue/archive.py b/mailman/queue/archive.py index 6dda70387..e9fd5f7ad 100644 --- a/mailman/queue/archive.py +++ b/mailman/queue/archive.py @@ -30,14 +30,14 @@ from datetime import datetime from email.Utils import parsedate_tz, mktime_tz, formatdate from locknix.lockfile import Lock -from mailman.configuration import config +from mailman import Defaults from mailman.core.plugins import get_plugins from mailman.queue import Runner class ArchiveRunner(Runner): - QDIR = config.ARCHQUEUE_DIR + """The archive runner.""" def _dispose(self, mlist, msg, msgdata): # Support clobber_date, i.e. setting the date in the archive to the @@ -48,9 +48,9 @@ class ArchiveRunner(Runner): received_time = formatdate(msgdata['received_time']) if not original_date: clobber = True - elif config.ARCHIVER_CLOBBER_DATE_POLICY == 1: + elif Defaults.ARCHIVER_CLOBBER_DATE_POLICY == 1: clobber = True - elif config.ARCHIVER_CLOBBER_DATE_POLICY == 2: + elif Defaults.ARCHIVER_CLOBBER_DATE_POLICY == 2: # What's the timestamp on the original message? timetup = parsedate_tz(original_date) now = datetime.now() @@ -60,7 +60,7 @@ class ArchiveRunner(Runner): else: utc_timestamp = datetime.fromtimestamp(mktime_tz(timetup)) clobber = (abs(now - utc_timestamp) > - config.ARCHIVER_ALLOWABLE_SANE_DATE_SKEW) + Defaults.ARCHIVER_ALLOWABLE_SANE_DATE_SKEW) except (ValueError, OverflowError): # The likely cause of this is that the year in the Date: field # is horribly incorrect, e.g. (from SF bug # 571634): diff --git a/mailman/queue/bounce.py b/mailman/queue/bounce.py index 0c5788174..18fa08437 100644 --- a/mailman/queue/bounce.py +++ b/mailman/queue/bounce.py @@ -30,7 +30,7 @@ from email.Utils import parseaddr from mailman import Utils from mailman.Bouncers import BouncerAPI from mailman.Message import UserNotification -from mailman.configuration import config +from mailman.config import config from mailman.i18n import _ from mailman.queue import Runner, Switchboard diff --git a/mailman/queue/command.py b/mailman/queue/command.py index a02827de0..986758ce1 100644 --- a/mailman/queue/command.py +++ b/mailman/queue/command.py @@ -43,7 +43,7 @@ from zope.interface import implements from mailman import Message from mailman import Utils from mailman.app.replybot import autorespond_to_sender -from mailman.configuration import config +from mailman.config import config from mailman.i18n import _ from mailman.interfaces import ContinueProcessing, IEmailResults from mailman.queue import Runner diff --git a/mailman/queue/docs/archiver.txt b/mailman/queue/docs/archiver.txt index 98ead0aa7..ed7c26d45 100644 --- a/mailman/queue/docs/archiver.txt +++ b/mailman/queue/docs/archiver.txt @@ -18,8 +18,7 @@ interface. By default, there's a Pipermail archiver. ... First post! ... """) - >>> from mailman.queue import Switchboard - >>> archiver_queue = Switchboard(config.ARCHQUEUE_DIR) + >>> archiver_queue = config.switchboards['archive'] >>> ignore = archiver_queue.enqueue(msg, {}, listname=mlist.fqdn_listname) >>> from mailman.queue.archive import ArchiveRunner diff --git a/mailman/queue/http.py b/mailman/queue/http.py index cb6940b29..c626e766e 100644 --- a/mailman/queue/http.py +++ b/mailman/queue/http.py @@ -25,7 +25,7 @@ from cStringIO import StringIO from wsgiref.simple_server import make_server, WSGIRequestHandler from mailman.Cgi.wsgi_app import mailman_app -from mailman.configuration import config +from mailman.config import config from mailman.queue import Runner hlog = logging.getLogger('mailman.http') diff --git a/mailman/queue/incoming.py b/mailman/queue/incoming.py index d4decd435..aaab56b21 100644 --- a/mailman/queue/incoming.py +++ b/mailman/queue/incoming.py @@ -26,7 +26,7 @@ prepared for delivery. Rejections, discards, and holds are processed immediately. """ -from mailman.configuration import config +from mailman.config import config from mailman.core.chains import process from mailman.queue import Runner diff --git a/mailman/queue/lmtp.py b/mailman/queue/lmtp.py index b4688653b..18f430e55 100644 --- a/mailman/queue/lmtp.py +++ b/mailman/queue/lmtp.py @@ -43,7 +43,7 @@ import asyncore from email.utils import parseaddr from mailman.Message import Message -from mailman.configuration import config +from mailman.config import config from mailman.database.transaction import txn from mailman.queue import Runner, Switchboard diff --git a/mailman/queue/maildir.py b/mailman/queue/maildir.py index 71bac67dc..3f7dd5556 100644 --- a/mailman/queue/maildir.py +++ b/mailman/queue/maildir.py @@ -57,7 +57,7 @@ from email.Parser import Parser from email.Utils import parseaddr from mailman.Message import Message -from mailman.configuration import config +from mailman.config import config from mailman.queue import Runner log = logging.getLogger('mailman.error') diff --git a/mailman/queue/news.py b/mailman/queue/news.py index 8415f7d95..70ffae71b 100644 --- a/mailman/queue/news.py +++ b/mailman/queue/news.py @@ -29,7 +29,7 @@ from email.utils import getaddresses, make_msgid COMMASPACE = ', ' from mailman import Utils -from mailman.configuration import config +from mailman.config import config from mailman.interfaces import NewsModeration from mailman.queue import Runner diff --git a/mailman/queue/outgoing.py b/mailman/queue/outgoing.py index 3ab67eaad..9eb287e6b 100644 --- a/mailman/queue/outgoing.py +++ b/mailman/queue/outgoing.py @@ -27,7 +27,7 @@ import logging from datetime import datetime from mailman import Message -from mailman.configuration import config +from mailman.config import config from mailman.core import errors from mailman.queue import Runner, Switchboard from mailman.queue.bounce import BounceMixin diff --git a/mailman/queue/pipeline.py b/mailman/queue/pipeline.py index 665bfce77..4d4be7790 100644 --- a/mailman/queue/pipeline.py +++ b/mailman/queue/pipeline.py @@ -23,7 +23,7 @@ headers, calculates message recipients, and more. """ from mailman.app.pipelines import process -from mailman.configuration import config +from mailman.config import config from mailman.queue import Runner diff --git a/mailman/queue/retry.py b/mailman/queue/retry.py index f8e1b4665..03cf86848 100644 --- a/mailman/queue/retry.py +++ b/mailman/queue/retry.py @@ -17,7 +17,7 @@ import time -from mailman.configuration import config +from mailman.config import config from mailman.queue import Runner, Switchboard diff --git a/mailman/queue/virgin.py b/mailman/queue/virgin.py index 503337ea1..917d702ca 100644 --- a/mailman/queue/virgin.py +++ b/mailman/queue/virgin.py @@ -24,7 +24,7 @@ recipient. """ from mailman.app.pipelines import process -from mailman.configuration import config +from mailman.config import config from mailman.queue import Runner diff --git a/mailman/testing/helpers.py b/mailman/testing/helpers.py index 0f9c3edae..2f5b4af01 100644 --- a/mailman/testing/helpers.py +++ b/mailman/testing/helpers.py @@ -58,14 +58,19 @@ def make_testable_runner(runner_class): :return: A runner instance. """ + assert runner_class.__name__.endswith('Runner'), ( + 'Unparseable runner class name: %s' % runner_class.__name__) + + name = runner_class.__name__[:-6].lower() + class EmptyingRunner(runner_class): """Stop processing when the queue is empty.""" def _do_periodic(self): """Stop when the queue is empty.""" - self._stop = (len(self._switchboard.files) == 0) + self._stop = (len(self.switchboard.files) == 0) - return EmptyingRunner() + return EmptyingRunner(name) diff --git a/mailman/testing/layers.py b/mailman/testing/layers.py index 97d6ade23..bda18289c 100644 --- a/mailman/testing/layers.py +++ b/mailman/testing/layers.py @@ -27,7 +27,9 @@ __all__ = [ import os import shutil import tempfile -import textwrap + +from pkg_resources import resource_string +from textwrap import dedent from mailman.config import config from mailman.core.initialize import initialize @@ -47,45 +49,19 @@ class ConfigLayer: def setUp(cls): initialize() assert cls.var_dir is None, 'Layer already set up' - test_config = [] # Calculate a temporary VAR_DIR directory so that run-time artifacts # of the tests won't tread on the installation's data. This also # makes it easier to clean up after the tests are done, and insures # isolation of test suite runs. cls.var_dir = tempfile.mkdtemp() - # lazr.config says it doesn't care about indentation, but we've - # actually got mixed indentation above because of the for-loop. It's - # just safer to dedent the whole string now. - test_config.append(textwrap.dedent(""" + # Create a section with the var directory. + test_config = dedent(""" [mailman] var_dir: %s - """ % cls.var_dir)) - # Push a high port for our test SMTP server. - test_config.append(textwrap.dedent(""" - [mta] - smtp_port: 9025 - """)) - # Set the qrunners to exit after one error. - for qrunner in config.qrunner_shortcuts: - test_config.append(textwrap.dedent(""" - [qrunner.%s] - max_restarts: 1 - """ % qrunner)) - # Add stuff for the archiver and a sample domain. - test_config.append(textwrap.dedent(""" - [archiver.mail_archive] - base_url: http://go.mail-archive.dev/ - recipient: archive@mail-archive.dev - - [domain.example_dot_com] - email_host: example.com - base_url: http://www.example.com - contact_address: postmaster@example.com - """)) - config_string = NL.join(test_config) - import pdb; pdb.set_trace() - config.push('test config', config_string) - config._config.getByCategory('domain') + """ % cls.var_dir) + # Read the testing config, but don't push it yet. + test_config += resource_string('mailman.testing', 'testing.cfg') + config.push('test config', test_config) @classmethod def tearDown(cls): diff --git a/mailman/testing/testing.cfg b/mailman/testing/testing.cfg new file mode 100644 index 000000000..baac7d803 --- /dev/null +++ b/mailman/testing/testing.cfg @@ -0,0 +1,66 @@ +# Copyright (C) 2008 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +# A testing configuration. + +[mta] +smtp_port: 9025 + +[qrunner.archive] +max_restarts: 1 + +[qrunner.bounces] +max_restarts: 1 + +[qrunner.commands] +max_restarts: 1 + +[qrunner.in] +max_restarts: 1 + +[qrunner.lmtp] +max_restarts: 1 + +[qrunner.maildir] +max_restarts: 1 + +[qrunner.news] +max_restarts: 1 + +[qrunner.out] +max_restarts: 1 + +[qrunner.pipeline] +max_restarts: 1 + +[qrunner.retry] +max_restarts: 1 + +[qrunner.shunt] +max_restarts: 1 + +[qrunner.virgin] +max_restarts: 1 + +[archiver.mail_archive] +base_url: http://go.mail-archive.dev/ +recipient: archive@mail-archive.dev + +[domain.example_dot_com] +email_host: example.com +base_url: http://www.example.com +contact_address: postmaster@example.com diff --git a/mailman/testing/testing.cfg.in b/mailman/testing/testing.cfg.in deleted file mode 100644 index e544242d0..000000000 --- a/mailman/testing/testing.cfg.in +++ /dev/null @@ -1,17 +0,0 @@ -# -*- python -*- - -# Configuration file template for the unit test suite. We need this because -# both the process running the tests and all sub-processes (e.g. qrunners) -# must share the same configuration file. - -MAX_RESTARTS = 1 -MTA = None -USE_LMTP = Yes - -# Make sure these goes to fake domains. -MAIL_ARCHIVE_BASEURL = 'http://go.mail-archive.dev/' -MAIL_ARCHIVE_RECIPIENT = 'archive@mail-archive.dev' - -add_domain('example.com', 'http://www.example.com') - -# bin/testall will add additional runtime configuration variables here. |
