diff options
| author | bwarsaw | 2006-07-08 17:37:55 +0000 |
|---|---|---|
| committer | bwarsaw | 2006-07-08 17:37:55 +0000 |
| commit | cbef3114de3e80b9436d909b11568858e3a1cf42 (patch) | |
| tree | f567fe3fbc331fe399b92e93f80068e8995a7821 | |
| parent | 60b723291e592ff7925e1b15b79161d1cdac5938 (diff) | |
| download | mailman-cbef3114de3e80b9436d909b11568858e3a1cf42.tar.gz mailman-cbef3114de3e80b9436d909b11568858e3a1cf42.tar.zst mailman-cbef3114de3e80b9436d909b11568858e3a1cf42.zip | |
Massive conversion process so that Mailman can be run from a user specified
configuration file. While the full conversion is not yet complete, everything
that seems to be required to run mailmanctl, qrunner, rmlist, and newlist have
been updated.
Basically, modules should no longer import mm_cfg, but instead they should
import Mailman.configuration.config. The latter is an object that's
guaranteed to exist, but not guaranteed to be initialized until some top-level
script calls config.load(). The latter should be called with the argument to
-C/--config which is a new convention the above scripts have been given.
In most cases, where mm_cfg.<variable> is used config.<variable> can be used,
but the exceptions are where the default value must be available before
config.load() is called. Sometimes you can import Mailman.Default and get the
variable from there, but other times the code has to be changed to work around
this limitation. Take each on a case-by-case basis.
Note that the various directories calculated from VAR_PREFIX, EXEC_PREFIX, and
PREFIX are now calculated in config.py, not in Defaults.py. This way a
configuration file can override the base directories and everything should
work correctly.
Other changes here include:
- mailmanctl, qrunner, and update are switched to optparse and $-strings, and
changed to the mmshell architecture
- An etc directory has been added to /usr/local/mailman and a
mailman.cfg.sample file is installed there. Sites should now edit an
etc/mailman.cfg file to do their configurations, although the mm_cfg file is
still honored. The formats of the two files are identical.
- list_lists is given the -C/--config option
- Some coding style fixes in bin/update, but not extensive
- Get rid of nested scope hacks in qrunner.py
- A start on getting EmailBase tests working (specifically test_message),
although not yet complete.
40 files changed, 2029 insertions, 1943 deletions
diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in index 27eb73ee5..d8248a3cf 100644 --- a/Mailman/Defaults.py.in +++ b/Mailman/Defaults.py.in @@ -1264,44 +1264,10 @@ AuthListAdmin = 3 # List Administrator (total control over list) AuthListModerator = 4 # List Moderator (can only handle held requests) AuthSiteAdmin = 5 # Site Administrator (total control over everything) -# Useful directories -LIST_DATA_DIR = os.path.join(VAR_PREFIX, 'lists') -LOG_DIR = os.path.join(VAR_PREFIX, 'logs') -LOCK_DIR = os.path.join(VAR_PREFIX, 'locks') -DATA_DIR = os.path.join(VAR_PREFIX, 'data') -SPAM_DIR = os.path.join(VAR_PREFIX, 'spam') -WRAPPER_DIR = os.path.join(EXEC_PREFIX, 'mail') -BIN_DIR = os.path.join(PREFIX, 'bin') -SCRIPTS_DIR = os.path.join(PREFIX, 'scripts') -TEMPLATE_DIR = os.path.join(PREFIX, 'templates') -MESSAGES_DIR = os.path.join(PREFIX, 'messages') -PUBLIC_ARCHIVE_FILE_DIR = os.path.join(VAR_PREFIX, 'archives', 'public') -PRIVATE_ARCHIVE_FILE_DIR = os.path.join(VAR_PREFIX, 'archives', 'private') - -# Directories used by the qrunner subsystem -QUEUE_DIR = os.path.join(VAR_PREFIX, 'qfiles') -INQUEUE_DIR = os.path.join(QUEUE_DIR, 'in') -OUTQUEUE_DIR = os.path.join(QUEUE_DIR, 'out') -CMDQUEUE_DIR = os.path.join(QUEUE_DIR, 'commands') -BOUNCEQUEUE_DIR = os.path.join(QUEUE_DIR, 'bounces') -NEWSQUEUE_DIR = os.path.join(QUEUE_DIR, 'news') -ARCHQUEUE_DIR = os.path.join(QUEUE_DIR, 'archive') -SHUNTQUEUE_DIR = os.path.join(QUEUE_DIR, 'shunt') -VIRGINQUEUE_DIR = os.path.join(QUEUE_DIR, 'virgin') -BADQUEUE_DIR = os.path.join(QUEUE_DIR, 'bad') -RETRYQUEUE_DIR = os.path.join(QUEUE_DIR, 'retry') -MAILDIR_DIR = os.path.join(QUEUE_DIR, 'maildir') - -# Other useful files -PIDFILE = os.path.join(DATA_DIR, 'master-qrunner.pid') -SITE_PW_FILE = os.path.join(DATA_DIR, 'adm.pw') -LISTCREATOR_PW_FILE = os.path.join(DATA_DIR, 'creator.pw') - # Import a bunch of version numbers from Version import * -MAILMAN_VERSION = 'GNU Mailman ' + VERSION # Vgg: Language descriptions and charsets dictionary, any new supported # language must have a corresponding entry here. Key is the name of the diff --git a/Mailman/Handlers/SMTPDirect.py b/Mailman/Handlers/SMTPDirect.py index e4fa6ec3b..60fb52359 100644 --- a/Mailman/Handlers/SMTPDirect.py +++ b/Mailman/Handlers/SMTPDirect.py @@ -38,19 +38,19 @@ from email.Header import Header from email.Utils import formataddr from Mailman import Errors -from Mailman import mm_cfg from Mailman import Utils from Mailman.Handlers import Decorate from Mailman.SafeDict import MsgSafeDict +from Mailman.configuration import config DOT = '.' log = logging.getLogger('mailman.smtp') flog = logging.getLogger('mailman.smtp-failure') -every_log = logging.getLogger('mailman.' + mm_cfg.SMTP_LOG_EVERY_MESSAGE[0]) -success_log = logging.getLogger('mailman.' + mm_cfg.SMTP_LOG_SUCCESS[0]) -refused_log = logging.getLogger('mailman.' + mm_cfg.SMTP_LOG_REFUSED[0]) -failure_log = logging.getLogger('mailman.' + mm_cfg.SMTP_LOG_EACH_FAILURE[0]) +every_log = logging.getLogger('mailman.' + config.SMTP_LOG_EVERY_MESSAGE[0]) +success_log = logging.getLogger('mailman.' + config.SMTP_LOG_SUCCESS[0]) +refused_log = logging.getLogger('mailman.' + config.SMTP_LOG_REFUSED[0]) +failure_log = logging.getLogger('mailman.' + config.SMTP_LOG_EACH_FAILURE[0]) @@ -61,8 +61,8 @@ class Connection: def __connect(self): self.__conn = smtplib.SMTP() - self.__conn.connect(mm_cfg.SMTPHOST, mm_cfg.SMTPPORT) - self.__numsessions = mm_cfg.SMTP_MAX_SESSIONS_PER_CONNECTION + self.__conn.connect(config.SMTPHOST, config.SMTPPORT) + self.__numsessions = config.SMTP_MAX_SESSIONS_PER_CONNECTION def sendmail(self, envsender, recips, msgtext): if self.__conn is None: @@ -118,10 +118,10 @@ def process(mlist, msg, msgdata): chunks = [[recip] for recip in recips] msgdata['personalize'] = 1 deliveryfunc = verpdeliver - elif mm_cfg.SMTP_MAX_RCPTS <= 0: + elif config.SMTP_MAX_RCPTS <= 0: chunks = [recips] else: - chunks = chunkify(recips, mm_cfg.SMTP_MAX_RCPTS) + chunks = chunkify(recips, config.SMTP_MAX_RCPTS) # See if this is an unshunted message for which some were undelivered if msgdata.has_key('undelivered'): chunks = msgdata['undelivered'] @@ -181,12 +181,12 @@ def process(mlist, msg, msgdata): # We have to use the copy() method because extended call syntax requires a # concrete dictionary object; it does not allow a generic mapping (XXX is # this still true in Python 2.3?). - if mm_cfg.SMTP_LOG_EVERY_MESSAGE: - every_log.info('%s', mm_cfg.SMTP_LOG_EVERY_MESSAGE[1] % d) + if config.SMTP_LOG_EVERY_MESSAGE: + every_log.info('%s', config.SMTP_LOG_EVERY_MESSAGE[1] % d) if refused: - if mm_cfg.SMTP_LOG_REFUSED: - refused_log.info('%s', mm_cfg.SMTP_LOG_REFUSED[1] % d) + if config.SMTP_LOG_REFUSED: + refused_log.info('%s', config.SMTP_LOG_REFUSED[1] % d) elif msgdata.get('tolist'): # Log the successful post, but only if it really was a post to the @@ -194,8 +194,8 @@ def process(mlist, msg, msgdata): # -request addrs should never get here. BAW: it may be useful to log # the other messages, but in that case, we should probably have a # separate configuration variable to control that. - if mm_cfg.SMTP_LOG_SUCCESS: - success_log.info('%s', mm_cfg.SMTP_LOG_SUCCESS[1] % d) + if config.SMTP_LOG_SUCCESS: + success_log.info('%s', config.SMTP_LOG_SUCCESS[1] % d) # Process any failed deliveries. tempfailures = [] @@ -217,11 +217,11 @@ def process(mlist, msg, msgdata): # Deal with persistent transient failures by queuing them up for # future delivery. TBD: this could generate lots of log entries! tempfailures.append(recip) - if mm_cfg.SMTP_LOG_EACH_FAILURE: + if config.SMTP_LOG_EACH_FAILURE: d.update({'recipient': recip, 'failcode' : code, 'failmsg' : smtpmsg}) - failure_log.info('%s', mm_cfg.SMTP_LOG_EACH_FAILURE[1] % d) + failure_log.info('%s', config.SMTP_LOG_EACH_FAILURE[1] % d) # Return the results if tempfailures or permfailures: raise Errors.SomeRecipientsFailed(tempfailures, permfailures) @@ -300,7 +300,7 @@ def verpdeliver(mlist, msg, msgdata, envsender, failures, conn): 'mailbox': rmailbox, 'host' : DOT.join(rdomain), } - envsender = '%s@%s' % ((mm_cfg.VERP_FORMAT % d), DOT.join(bdomain)) + envsender = '%s@%s' % ((config.VERP_FORMAT % d), DOT.join(bdomain)) if mlist.personalize == 2: # When fully personalizing, we want the To address to point to the # recipient, not to the mailing list diff --git a/Mailman/MTA/Utils.py b/Mailman/MTA/Utils.py index 14562de66..ef3df0cb8 100644 --- a/Mailman/MTA/Utils.py +++ b/Mailman/MTA/Utils.py @@ -1,25 +1,26 @@ -# Copyright (C) 2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2006 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. """Utilities for list creation/deletion hooks.""" import os import pwd -from Mailman import mm_cfg +from Mailman.configuration import config @@ -35,7 +36,7 @@ def getusername(): def _makealiases_mailprog(listname): - wrapper = os.path.join(mm_cfg.WRAPPER_DIR, 'mailman') + wrapper = os.path.join(config.WRAPPER_DIR, 'mailman') # Most of the list alias extensions are quite regular. I.e. if the # message is delivered to listname-foobar, it will be filtered to a # program called foobar. There are two exceptions: @@ -57,7 +58,7 @@ def _makealiases_mailprog(listname): def _makealiases_maildir(listname): - maildir = mm_cfg.MAILDIR_DIR + maildir = config.MAILDIR_DIR if not maildir.endswith('/'): maildir += '/' # Deliver everything using maildir style. This way there's no mail @@ -73,7 +74,9 @@ def _makealiases_maildir(listname): -if mm_cfg.USE_MAILDIR: +# XXX This won't work if Mailman.MTA.Utils is imported before the +# configuration is loaded. +if config.USE_MAILDIR: makealiases = _makealiases_maildir else: makealiases = _makealiases_mailprog diff --git a/Mailman/MailList.py b/Mailman/MailList.py index 04fb0f22c..ecf449246 100644 --- a/Mailman/MailList.py +++ b/Mailman/MailList.py @@ -45,8 +45,8 @@ from email.Utils import getaddresses, formataddr, parseaddr from Mailman import Errors from Mailman import LockFile from Mailman import Utils -from Mailman import mm_cfg from Mailman.UserDesc import UserDesc +from Mailman.configuration import config # Base classes from Mailman import Pending @@ -196,19 +196,19 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, return self.getListAddress('owner') def GetRequestEmail(self, cookie=''): - if mm_cfg.VERP_CONFIRMATIONS and cookie: + if config.VERP_CONFIRMATIONS and cookie: return self.GetConfirmEmail(cookie) else: return self.getListAddress('request') def GetConfirmEmail(self, cookie): - return mm_cfg.VERP_CONFIRM_FORMAT % { + return config.VERP_CONFIRM_FORMAT % { 'addr' : '%s-confirm' % self.internal_name(), 'cookie': cookie, } + '@' + self.host_name def GetConfirmJoinSubject(self, listname, cookie): - if mm_cfg.VERP_CONFIRMATIONS and cookie: + if config.VERP_CONFIRMATIONS and cookie: cset = i18n.get_translation().charset() or \ Utils.GetCharSet(self.preferred_language) subj = Header( @@ -219,7 +219,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, return 'confirm ' + cookie def GetConfirmLeaveSubject(self, listname, cookie): - if mm_cfg.VERP_CONFIRMATIONS and cookie: + if config.VERP_CONFIRMATIONS and cookie: cset = i18n.get_translation().charset() or \ Utils.GetCharSet(self.preferred_language) subj = Header( @@ -269,8 +269,8 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # need to reload, otherwise... we do. self.__timestamp = 0 self.__lock = LockFile.LockFile( - os.path.join(mm_cfg.LOCK_DIR, name or '<site>') + '.lock', - lifetime=mm_cfg.LIST_LOCK_LIFETIME) + os.path.join(config.LOCK_DIR, name or '<site>') + '.lock', + lifetime=config.LIST_LOCK_LIFETIME) self._internal_name = name if name: self._full_path = Site.get_listpath(name) @@ -299,7 +299,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # Must save this state, even though it isn't configurable self.volume = 1 self.members = {} # self.digest_members is initted in mm_digest - self.data_version = mm_cfg.DATA_FILE_VERSION + self.data_version = config.DATA_FILE_VERSION self.last_post_time = 0 self.post_id = 1. # A float so it never has a chance to overflow. @@ -307,76 +307,76 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, self.language = {} self.usernames = {} self.passwords = {} - self.new_member_options = mm_cfg.DEFAULT_NEW_MEMBER_OPTIONS + self.new_member_options = config.DEFAULT_NEW_MEMBER_OPTIONS # This stuff is configurable self.respond_to_post_requests = 1 - self.advertised = mm_cfg.DEFAULT_LIST_ADVERTISED - self.max_num_recipients = mm_cfg.DEFAULT_MAX_NUM_RECIPIENTS - self.max_message_size = mm_cfg.DEFAULT_MAX_MESSAGE_SIZE + self.advertised = config.DEFAULT_LIST_ADVERTISED + self.max_num_recipients = config.DEFAULT_MAX_NUM_RECIPIENTS + self.max_message_size = config.DEFAULT_MAX_MESSAGE_SIZE # See the note in Defaults.py concerning DEFAULT_HOST_NAME # vs. DEFAULT_EMAIL_HOST. - self.host_name = mm_cfg.DEFAULT_HOST_NAME or mm_cfg.DEFAULT_EMAIL_HOST + self.host_name = config.DEFAULT_HOST_NAME or config.DEFAULT_EMAIL_HOST self.web_page_url = ( - mm_cfg.DEFAULT_URL or - mm_cfg.DEFAULT_URL_PATTERN % mm_cfg.DEFAULT_URL_HOST) + config.DEFAULT_URL or + config.DEFAULT_URL_PATTERN % config.DEFAULT_URL_HOST) self.owner = [admin] self.moderator = [] - self.reply_goes_to_list = mm_cfg.DEFAULT_REPLY_GOES_TO_LIST + self.reply_goes_to_list = config.DEFAULT_REPLY_GOES_TO_LIST self.reply_to_address = '' - self.first_strip_reply_to = mm_cfg.DEFAULT_FIRST_STRIP_REPLY_TO - self.admin_immed_notify = mm_cfg.DEFAULT_ADMIN_IMMED_NOTIFY + self.first_strip_reply_to = config.DEFAULT_FIRST_STRIP_REPLY_TO + self.admin_immed_notify = config.DEFAULT_ADMIN_IMMED_NOTIFY self.admin_notify_mchanges = \ - mm_cfg.DEFAULT_ADMIN_NOTIFY_MCHANGES + config.DEFAULT_ADMIN_NOTIFY_MCHANGES self.require_explicit_destination = \ - mm_cfg.DEFAULT_REQUIRE_EXPLICIT_DESTINATION - self.acceptable_aliases = mm_cfg.DEFAULT_ACCEPTABLE_ALIASES - self.umbrella_list = mm_cfg.DEFAULT_UMBRELLA_LIST + config.DEFAULT_REQUIRE_EXPLICIT_DESTINATION + self.acceptable_aliases = config.DEFAULT_ACCEPTABLE_ALIASES + self.umbrella_list = config.DEFAULT_UMBRELLA_LIST self.umbrella_member_suffix = \ - mm_cfg.DEFAULT_UMBRELLA_MEMBER_ADMIN_SUFFIX - self.send_reminders = mm_cfg.DEFAULT_SEND_REMINDERS - self.send_welcome_msg = mm_cfg.DEFAULT_SEND_WELCOME_MSG - self.send_goodbye_msg = mm_cfg.DEFAULT_SEND_GOODBYE_MSG + config.DEFAULT_UMBRELLA_MEMBER_ADMIN_SUFFIX + self.send_reminders = config.DEFAULT_SEND_REMINDERS + self.send_welcome_msg = config.DEFAULT_SEND_WELCOME_MSG + self.send_goodbye_msg = config.DEFAULT_SEND_GOODBYE_MSG self.bounce_matching_headers = \ - mm_cfg.DEFAULT_BOUNCE_MATCHING_HEADERS + config.DEFAULT_BOUNCE_MATCHING_HEADERS self.header_filter_rules = [] - self.anonymous_list = mm_cfg.DEFAULT_ANONYMOUS_LIST + self.anonymous_list = config.DEFAULT_ANONYMOUS_LIST internalname = self.internal_name() self.real_name = internalname[0].upper() + internalname[1:] self.description = '' self.info = '' self.welcome_msg = '' self.goodbye_msg = '' - self.subscribe_policy = mm_cfg.DEFAULT_SUBSCRIBE_POLICY - self.subscribe_auto_approval = mm_cfg.DEFAULT_SUBSCRIBE_AUTO_APPROVAL - self.unsubscribe_policy = mm_cfg.DEFAULT_UNSUBSCRIBE_POLICY - self.private_roster = mm_cfg.DEFAULT_PRIVATE_ROSTER - self.obscure_addresses = mm_cfg.DEFAULT_OBSCURE_ADDRESSES - self.admin_member_chunksize = mm_cfg.DEFAULT_ADMIN_MEMBER_CHUNKSIZE - self.administrivia = mm_cfg.DEFAULT_ADMINISTRIVIA - self.preferred_language = mm_cfg.DEFAULT_SERVER_LANGUAGE + self.subscribe_policy = config.DEFAULT_SUBSCRIBE_POLICY + self.subscribe_auto_approval = config.DEFAULT_SUBSCRIBE_AUTO_APPROVAL + self.unsubscribe_policy = config.DEFAULT_UNSUBSCRIBE_POLICY + self.private_roster = config.DEFAULT_PRIVATE_ROSTER + self.obscure_addresses = config.DEFAULT_OBSCURE_ADDRESSES + self.admin_member_chunksize = config.DEFAULT_ADMIN_MEMBER_CHUNKSIZE + self.administrivia = config.DEFAULT_ADMINISTRIVIA + self.preferred_language = config.DEFAULT_SERVER_LANGUAGE self.available_languages = [] self.include_rfc2369_headers = 1 self.include_list_post_header = 1 - self.filter_mime_types = mm_cfg.DEFAULT_FILTER_MIME_TYPES - self.pass_mime_types = mm_cfg.DEFAULT_PASS_MIME_TYPES + self.filter_mime_types = config.DEFAULT_FILTER_MIME_TYPES + self.pass_mime_types = config.DEFAULT_PASS_MIME_TYPES self.filter_filename_extensions = \ - mm_cfg.DEFAULT_FILTER_FILENAME_EXTENSIONS - self.pass_filename_extensions = mm_cfg.DEFAULT_PASS_FILENAME_EXTENSIONS - self.filter_content = mm_cfg.DEFAULT_FILTER_CONTENT - self.collapse_alternatives = mm_cfg.DEFAULT_COLLAPSE_ALTERNATIVES + config.DEFAULT_FILTER_FILENAME_EXTENSIONS + self.pass_filename_extensions = config.DEFAULT_PASS_FILENAME_EXTENSIONS + self.filter_content = config.DEFAULT_FILTER_CONTENT + self.collapse_alternatives = config.DEFAULT_COLLAPSE_ALTERNATIVES self.convert_html_to_plaintext = \ - mm_cfg.DEFAULT_CONVERT_HTML_TO_PLAINTEXT - self.filter_action = mm_cfg.DEFAULT_FILTER_ACTION + config.DEFAULT_CONVERT_HTML_TO_PLAINTEXT + self.filter_action = config.DEFAULT_FILTER_ACTION # Analogs to these are initted in Digester.InitVars - self.nondigestable = mm_cfg.DEFAULT_NONDIGESTABLE + self.nondigestable = config.DEFAULT_NONDIGESTABLE self.personalize = 0 # New sender-centric moderation (privacy) options self.default_member_moderation = \ - mm_cfg.DEFAULT_DEFAULT_MEMBER_MODERATION + config.DEFAULT_DEFAULT_MEMBER_MODERATION # Emergency moderation bit self.emergency = 0 - # This really ought to default to mm_cfg.HOLD, but that doesn't work + # This really ought to default to config.HOLD, but that doesn't work # with the current GUI description model. So, 0==Hold, 1==Reject, # 2==Discard self.member_moderation_action = 0 @@ -385,8 +385,8 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, self.hold_these_nonmembers = [] self.reject_these_nonmembers = [] self.discard_these_nonmembers = [] - self.forward_auto_discards = mm_cfg.DEFAULT_FORWARD_AUTO_DISCARDS - self.generic_nonmember_action = mm_cfg.DEFAULT_GENERIC_NONMEMBER_ACTION + self.forward_auto_discards = config.DEFAULT_FORWARD_AUTO_DISCARDS + self.generic_nonmember_action = config.DEFAULT_GENERIC_NONMEMBER_ACTION self.nonmember_rejection_notice = '' # Ban lists self.ban_list = [] @@ -403,9 +403,9 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # These need to come near the bottom because they're dependent on # other settings. - self.subject_prefix = mm_cfg.DEFAULT_SUBJECT_PREFIX % self.__dict__ - self.msg_header = mm_cfg.DEFAULT_MSG_HEADER - self.msg_footer = mm_cfg.DEFAULT_MSG_FOOTER + self.subject_prefix = config.DEFAULT_SUBJECT_PREFIX % self.__dict__ + self.msg_header = config.DEFAULT_MSG_HEADER + self.msg_footer = config.DEFAULT_MSG_FOOTER # Set this to Never if the list's preferred language uses us-ascii, # otherwise set it to As Needed if Utils.GetCharSet(self.preferred_language) == 'us-ascii': @@ -413,9 +413,9 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, else: self.encode_ascii_prefixes = 2 # scrub regular delivery - self.scrub_nondigest = mm_cfg.DEFAULT_SCRUB_NONDIGEST + self.scrub_nondigest = config.DEFAULT_SCRUB_NONDIGEST # automatic discarding - self.max_days_to_hold = mm_cfg.DEFAULT_MAX_DAYS_TO_HOLD + self.max_days_to_hold = config.DEFAULT_MAX_DAYS_TO_HOLD # @@ -425,17 +425,17 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, class CategoryDict(UserDict): def __init__(self): UserDict.__init__(self) - self.keysinorder = mm_cfg.ADMIN_CATEGORIES[:] + self.keysinorder = config.ADMIN_CATEGORIES[:] def keys(self): return self.keysinorder def items(self): items = [] - for k in mm_cfg.ADMIN_CATEGORIES: + for k in config.ADMIN_CATEGORIES: items.append((k, self.data[k])) return items def values(self): values = [] - for k in mm_cfg.ADMIN_CATEGORIES: + for k in config.ADMIN_CATEGORIES: values.append(self.data[k]) return values @@ -476,7 +476,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # However, most scripts already catch MMBadEmailError as exceptions on # the admin's email address, so transform the exception. if emailhost is None: - emailhost = mm_cfg.DEFAULT_EMAIL_HOST + emailhost = config.DEFAULT_EMAIL_HOST postingaddr = '%s@%s' % (name, emailhost) try: Utils.ValidateEmail(postingaddr) @@ -515,7 +515,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # Use a binary format... it's more efficient. cPickle.dump(dict, fp, 1) fp.flush() - if mm_cfg.SYNC_AFTER_WRITE: + if config.SYNC_AFTER_WRITE: os.fsync(fp.fileno()) fp.close() except IOError, e: @@ -697,7 +697,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # def CheckVersion(self, stored_state): """Auto-update schema if necessary.""" - if self.data_version >= mm_cfg.DATA_FILE_VERSION: + if self.data_version >= config.DATA_FILE_VERSION: return # Initialize any new variables self.InitVars() @@ -712,7 +712,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, try: from versions import Update Update(self, stored_state) - self.data_version = mm_cfg.DATA_FILE_VERSION + self.data_version = config.DATA_FILE_VERSION self.Save() finally: if not waslocked: @@ -725,8 +725,8 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # URL is empty; substitute faulty value with (hopefully sane) # default. Note that DEFAULT_URL is obsolete. self.web_page_url = ( - mm_cfg.DEFAULT_URL or - mm_cfg.DEFAULT_URL_PATTERN % mm_cfg.DEFAULT_URL_HOST) + config.DEFAULT_URL or + config.DEFAULT_URL_PATTERN % config.DEFAULT_URL_HOST) if self.web_page_url and self.web_page_url[-1] <> '/': self.web_page_url = self.web_page_url + '/' # Legacy reply_to_address could be an illegal value. We now verify @@ -958,9 +958,9 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # Do the actual addition self.addNewMember(email, realname=name, digest=digest, password=password, language=lang) - self.setMemberOption(email, mm_cfg.DisableMime, + self.setMemberOption(email, config.DisableMime, 1 - self.mime_is_default_digest) - self.setMemberOption(email, mm_cfg.Moderate, + self.setMemberOption(email, config.Moderate, self.default_member_moderation) # Now send and log results if digest: @@ -1283,17 +1283,17 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, if approved is not None: # Does it match the list password? Note that we purposefully # do not allow the site password here. - if self.Authenticate([mm_cfg.AuthListAdmin, - mm_cfg.AuthListModerator], - approved) <> mm_cfg.UnAuthorized: - action = mm_cfg.APPROVE + if self.Authenticate([config.AuthListAdmin, + config.AuthListModerator], + approved) <> config.UnAuthorized: + action = config.APPROVE else: # The password didn't match. Re-pend the message and # inform the list moderators about the problem. self.pend_repend(cookie, rec) raise Errors.MMBadPasswordError else: - action = mm_cfg.DISCARD + action = config.DISCARD try: self.HandleRequest(id, action) except KeyError: @@ -1459,7 +1459,7 @@ bad regexp in bounce_matching_header line: %s lang = self.preferred_language i18n.set_language(lang) # No limit - if mm_cfg.MAX_AUTORESPONSES_PER_DAY == 0: + if config.MAX_AUTORESPONSES_PER_DAY == 0: return 1 today = time.localtime()[:3] info = self.hold_and_cmd_autoresponses.get(sender) @@ -1473,7 +1473,7 @@ bad regexp in bounce_matching_header line: %s # They've already hit the limit for today. vlog.info('-request/hold autoresponse discarded for: %s', sender) return 0 - if count >= mm_cfg.MAX_AUTORESPONSES_PER_DAY: + if count >= config.MAX_AUTORESPONSES_PER_DAY: vlog.info('-request/hold autoresponse limit hit for: %s', sender) self.hold_and_cmd_autoresponses[sender] = (today, -1) # Send this notification message instead @@ -1544,8 +1544,8 @@ bad regexp in bounce_matching_header line: %s # language support to the list, then the general admin page may have a # blank field where the list owner is supposed to chose the list's # preferred language. - if mm_cfg.DEFAULT_SERVER_LANGUAGE not in langs: - langs.append(mm_cfg.DEFAULT_SERVER_LANGUAGE) + if config.DEFAULT_SERVER_LANGUAGE not in langs: + langs.append(config.DEFAULT_SERVER_LANGUAGE) # When testing, it's possible we've disabled a language, so just # filter things out so we don't get tracebacks. - return [lang for lang in langs if mm_cfg.LC_DESCRIPTIONS.has_key(lang)] + return [lang for lang in langs if config.LC_DESCRIPTIONS.has_key(lang)] diff --git a/Mailman/Message.py b/Mailman/Message.py index 7216d7c9a..b39bcb3fe 100644 --- a/Mailman/Message.py +++ b/Mailman/Message.py @@ -29,8 +29,8 @@ import email.Utils from email.Charset import Charset from email.Header import Header -from Mailman import mm_cfg from Mailman import Utils +from Mailman.configuration import config COMMASPACE = ', ' @@ -113,7 +113,7 @@ class Message(email.Message.Message): This method differs from get_senders() in that it returns one and only one address, and uses a different search order. """ - senderfirst = mm_cfg.USE_ENVELOPE_SENDER + senderfirst = config.USE_ENVELOPE_SENDER if use_envelope is not None: senderfirst = use_envelope if senderfirst: @@ -165,7 +165,7 @@ class Message(email.Message.Message): names without the trailing colon. """ if headers is None: - headers = mm_cfg.SENDER_HEADERS + headers = config.SENDER_HEADERS pairs = [] for h in headers: if h is None: @@ -245,7 +245,7 @@ class UserNotification(Message): def _enqueue(self, mlist, **_kws): # Not imported at module scope to avoid import loop from Mailman.Queue.sbcache import get_switchboard - virginq = get_switchboard(mm_cfg.VIRGINQUEUE_DIR) + virginq = get_switchboard(config.VIRGINQUEUE_DIR) # The message metadata better have a `recip' attribute virginq.enqueue(self, listname = mlist.internal_name(), @@ -276,7 +276,7 @@ class OwnerNotification(UserNotification): def _enqueue(self, mlist, **_kws): # Not imported at module scope to avoid import loop from Mailman.Queue.sbcache import get_switchboard - virginq = get_switchboard(mm_cfg.VIRGINQUEUE_DIR) + virginq = get_switchboard(config.VIRGINQUEUE_DIR) # The message metadata better have a `recip' attribute virginq.enqueue(self, listname = mlist.internal_name(), diff --git a/Mailman/Queue/ArchRunner.py b/Mailman/Queue/ArchRunner.py index 627145372..f3c90f5c3 100644 --- a/Mailman/Queue/ArchRunner.py +++ b/Mailman/Queue/ArchRunner.py @@ -1,4 +1,4 @@ -# Copyright (C) 2000,2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 2000-2006 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -12,21 +12,22 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. """Archive queue runner.""" import time from email.Utils import parsedate_tz, mktime_tz, formatdate -from Mailman import mm_cfg from Mailman import LockFile from Mailman.Queue.Runner import Runner +from Mailman.configuration import config class ArchRunner(Runner): - QDIR = mm_cfg.ARCHQUEUE_DIR + QDIR = config.ARCHQUEUE_DIR def _dispose(self, mlist, msg, msgdata): # Support clobber_date, i.e. setting the date in the archive to the @@ -37,9 +38,9 @@ class ArchRunner(Runner): receivedtime = formatdate(msgdata['received_time']) if not originaldate: clobber = 1 - elif mm_cfg.ARCHIVER_CLOBBER_DATE_POLICY == 1: + elif config.ARCHIVER_CLOBBER_DATE_POLICY == 1: clobber = 1 - elif mm_cfg.ARCHIVER_CLOBBER_DATE_POLICY == 2: + elif config.ARCHIVER_CLOBBER_DATE_POLICY == 2: # what's the timestamp on the original message? tup = parsedate_tz(originaldate) now = time.time() @@ -47,7 +48,7 @@ class ArchRunner(Runner): if not tup: clobber = 1 elif abs(now - mktime_tz(tup)) > \ - mm_cfg.ARCHIVER_ALLOWABLE_SANE_DATE_SKEW: + config.ARCHIVER_ALLOWABLE_SANE_DATE_SKEW: clobber = 1 except (ValueError, OverflowError): # The likely cause of this is that the year in the Date: field @@ -65,7 +66,7 @@ class ArchRunner(Runner): msg['X-List-Received-Date'] = receivedtime # Now try to get the list lock try: - mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT) + mlist.Lock(timeout=config.LIST_LOCK_TIMEOUT) except LockFile.TimeOutError: # oh well, try again later return 1 diff --git a/Mailman/Queue/BounceRunner.py b/Mailman/Queue/BounceRunner.py index 0063635ac..93bee2062 100644 --- a/Mailman/Queue/BounceRunner.py +++ b/Mailman/Queue/BounceRunner.py @@ -28,13 +28,13 @@ from email.MIMEText import MIMEText from email.Utils import parseaddr from Mailman import LockFile -from Mailman import mm_cfg from Mailman import Utils from Mailman.Bouncers import BouncerAPI -from Mailman.i18n import _ from Mailman.Message import UserNotification from Mailman.Queue.Runner import Runner from Mailman.Queue.sbcache import get_switchboard +from Mailman.configuration import config +from Mailman.i18n import _ COMMASPACE = ', ' @@ -78,10 +78,10 @@ class BounceMixin: # their lists. So now we ignore site list bounces. Ce La Vie for # password reminder bounces. self._bounce_events_file = os.path.join( - mm_cfg.DATA_DIR, 'bounce-events-%05d.pck' % os.getpid()) + config.DATA_DIR, 'bounce-events-%05d.pck' % os.getpid()) self._bounce_events_fp = None self._bouncecnt = 0 - self._nextaction = time.time() + mm_cfg.REGISTER_BOUNCES_EVERY + self._nextaction = time.time() + config.REGISTER_BOUNCES_EVERY def _queue_bounces(self, listname, addrs, msg): today = time.localtime()[:3] @@ -137,7 +137,7 @@ class BounceMixin: if self._nextaction > now or self._bouncecnt == 0: return # Let's go ahead and register the bounces we've got stored up - self._nextaction = now + mm_cfg.REGISTER_BOUNCES_EVERY + self._nextaction = now + config.REGISTER_BOUNCES_EVERY self._register_bounces() def _probe_bounce(self, mlist, token): @@ -158,7 +158,7 @@ class BounceMixin: class BounceRunner(Runner, BounceMixin): - QDIR = mm_cfg.BOUNCEQUEUE_DIR + QDIR = config.BOUNCEQUEUE_DIR def __init__(self, slice=None, numslices=1): Runner.__init__(self, slice, numslices) @@ -167,7 +167,7 @@ class BounceRunner(Runner, BounceMixin): def _dispose(self, mlist, msg, msgdata): # Make sure we have the most up-to-date state mlist.Load() - outq = get_switchboard(mm_cfg.OUTQUEUE_DIR) + outq = get_switchboard(config.OUTQUEUE_DIR) # There are a few possibilities here: # # - the message could have been VERP'd in which case, we know exactly @@ -245,7 +245,7 @@ def verp_bounce(mlist, msg): to = parseaddr(field)[1] if not to: continue # empty header - mo = re.search(mm_cfg.VERP_REGEXP, to) + mo = re.search(config.VERP_REGEXP, to) if not mo: continue # no match of regexp try: @@ -255,7 +255,7 @@ def verp_bounce(mlist, msg): addr = '%s@%s' % mo.group('mailbox', 'host') except IndexError: elog.error("VERP_REGEXP doesn't yield the right match groups: %s", - mm_cfg.VERP_REGEXP) + config.VERP_REGEXP) return [] return [addr] @@ -276,7 +276,7 @@ def verp_probe(mlist, msg): to = parseaddr(field)[1] if not to: continue # empty header - mo = re.search(mm_cfg.VERP_PROBE_REGEXP, to) + mo = re.search(config.VERP_PROBE_REGEXP, to) if not mo: continue # no match of regexp try: @@ -290,7 +290,7 @@ def verp_probe(mlist, msg): except IndexError: elog.error( "VERP_PROBE_REGEXP doesn't yield the right match groups: %s", - mm_cfg.VERP_PROBE_REGEXP) + config.VERP_PROBE_REGEXP) return None diff --git a/Mailman/Queue/CommandRunner.py b/Mailman/Queue/CommandRunner.py index fe30a3cd3..2d1fc1031 100644 --- a/Mailman/Queue/CommandRunner.py +++ b/Mailman/Queue/CommandRunner.py @@ -33,11 +33,11 @@ from email.MIMEText import MIMEText from Mailman import LockFile from Mailman import Message -from Mailman import mm_cfg from Mailman import Utils from Mailman.Handlers import Replybot -from Mailman.i18n import _ from Mailman.Queue.Runner import Runner +from Mailman.configuration import config +from Mailman.i18n import _ NL = '\n' @@ -88,8 +88,8 @@ class Results: assert isinstance(body, basestring) lines = body.splitlines() # Use no more lines than specified - self.commands.extend(lines[:mm_cfg.DEFAULT_MAIL_COMMANDS_MAX_LINES]) - self.ignored.extend(lines[mm_cfg.DEFAULT_MAIL_COMMANDS_MAX_LINES:]) + self.commands.extend(lines[:config.DEFAULT_MAIL_COMMANDS_MAX_LINES]) + self.ignored.extend(lines[config.DEFAULT_MAIL_COMMANDS_MAX_LINES:]) def process(self): # Now, process each line until we find an error. The first @@ -192,7 +192,7 @@ To obtain instructions, send a message containing just the word "help". class CommandRunner(Runner): - QDIR = mm_cfg.CMDQUEUE_DIR + QDIR = config.CMDQUEUE_DIR def _dispose(self, mlist, msg, msgdata): # The policy here is similar to the Replybot policy. If a message has @@ -217,7 +217,7 @@ class CommandRunner(Runner): # locked. Still, it's more convenient to lock it here and now and # deal with lock failures in one place. try: - mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT) + mlist.Lock(timeout=config.LIST_LOCK_TIMEOUT) except LockFile.TimeOutError: # Oh well, try again later return True @@ -232,7 +232,7 @@ class CommandRunner(Runner): elif msgdata.get('toleave'): res.do_command('leave') elif msgdata.get('toconfirm'): - mo = re.match(mm_cfg.VERP_CONFIRM_REGEXP, msg.get('to', '')) + mo = re.match(config.VERP_CONFIRM_REGEXP, msg.get('to', '')) if mo: res.do_command('confirm', (mo.group('cookie'),)) res.send_response() diff --git a/Mailman/Queue/IncomingRunner.py b/Mailman/Queue/IncomingRunner.py index 19a315040..332c9f129 100644 --- a/Mailman/Queue/IncomingRunner.py +++ b/Mailman/Queue/IncomingRunner.py @@ -103,8 +103,8 @@ from cStringIO import StringIO from Mailman import Errors from Mailman import LockFile -from Mailman import mm_cfg from Mailman.Queue.Runner import Runner +from Mailman.configuration import config log = logging.getLogger('mailman.error') vlog = logging.getLogger('mailman.vette') @@ -112,12 +112,12 @@ vlog = logging.getLogger('mailman.vette') class IncomingRunner(Runner): - QDIR = mm_cfg.INQUEUE_DIR + QDIR = config.INQUEUE_DIR def _dispose(self, mlist, msg, msgdata): # Try to get the list lock. try: - mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT) + mlist.Lock(timeout=config.LIST_LOCK_TIMEOUT) except LockFile.TimeOutError: # Oh well, try again later return 1 @@ -146,7 +146,7 @@ class IncomingRunner(Runner): # flows through the pipeline will empty it out! return msgdata.get('pipeline', getattr(mlist, 'pipeline', - mm_cfg.GLOBAL_PIPELINE))[:] + config.GLOBAL_PIPELINE))[:] def _dopipeline(self, mlist, msg, msgdata, pipeline): while pipeline: diff --git a/Mailman/Queue/MaildirRunner.py b/Mailman/Queue/MaildirRunner.py index 97b71c7ef..9e5e08be5 100644 --- a/Mailman/Queue/MaildirRunner.py +++ b/Mailman/Queue/MaildirRunner.py @@ -57,11 +57,11 @@ import logging from email.Parser import Parser from email.Utils import parseaddr -from Mailman import mm_cfg from Mailman import Utils from Mailman.Message import Message from Mailman.Queue.Runner import Runner from Mailman.Queue.sbcache import get_switchboard +from Mailman.configuration import config # We only care about the listname and the subq as in listname@ or # listname-request@ @@ -88,8 +88,8 @@ class MaildirRunner(Runner): # Don't call the base class constructor, but build enough of the # underlying attributes to use the base class's implementation. self._stop = 0 - self._dir = os.path.join(mm_cfg.MAILDIR_DIR, 'new') - self._cur = os.path.join(mm_cfg.MAILDIR_DIR, 'cur') + self._dir = os.path.join(config.MAILDIR_DIR, 'new') + self._cur = os.path.join(config.MAILDIR_DIR, 'cur') self._parser = Parser(Message) def _oneloop(self): @@ -150,29 +150,29 @@ class MaildirRunner(Runner): msgdata = {'listname': listname} # -admin is deprecated if subq in ('bounces', 'admin'): - queue = get_switchboard(mm_cfg.BOUNCEQUEUE_DIR) + queue = get_switchboard(config.BOUNCEQUEUE_DIR) elif subq == 'confirm': msgdata['toconfirm'] = 1 - queue = get_switchboard(mm_cfg.CMDQUEUE_DIR) + queue = get_switchboard(config.CMDQUEUE_DIR) elif subq in ('join', 'subscribe'): msgdata['tojoin'] = 1 - queue = get_switchboard(mm_cfg.CMDQUEUE_DIR) + queue = get_switchboard(config.CMDQUEUE_DIR) elif subq in ('leave', 'unsubscribe'): msgdata['toleave'] = 1 - queue = get_switchboard(mm_cfg.CMDQUEUE_DIR) + queue = get_switchboard(config.CMDQUEUE_DIR) elif subq == 'owner': msgdata.update({ 'toowner': 1, 'envsender': Utils.get_site_email(extra='bounces'), - 'pipeline': mm_cfg.OWNER_PIPELINE, + 'pipeline': config.OWNER_PIPELINE, }) - queue = get_switchboard(mm_cfg.INQUEUE_DIR) + queue = get_switchboard(config.INQUEUE_DIR) elif subq is None: msgdata['tolist'] = 1 - queue = get_switchboard(mm_cfg.INQUEUE_DIR) + queue = get_switchboard(config.INQUEUE_DIR) elif subq == 'request': msgdata['torequest'] = 1 - queue = get_switchboard(mm_cfg.CMDQUEUE_DIR) + queue = get_switchboard(config.CMDQUEUE_DIR) else: log.error('Unknown sub-queue: %s', subq) os.rename(dstname, xdstname) diff --git a/Mailman/Queue/NewsRunner.py b/Mailman/Queue/NewsRunner.py index 7a71f5bf2..31e88da9a 100644 --- a/Mailman/Queue/NewsRunner.py +++ b/Mailman/Queue/NewsRunner.py @@ -12,7 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. """NNTP queue runner.""" @@ -27,9 +28,9 @@ from email.Utils import getaddresses COMMASPACE = ', ' -from Mailman import mm_cfg from Mailman import Utils from Mailman.Queue.Runner import Runner +from Mailman.configuration import config log = logging.getLogger('mailman.error') @@ -48,7 +49,7 @@ mcre = re.compile(r""" class NewsRunner(Runner): - QDIR = mm_cfg.NEWSQUEUE_DIR + QDIR = config.NEWSQUEUE_DIR def _dispose(self, mlist, msg, msgdata): # Make sure we have the most up-to-date state @@ -64,8 +65,8 @@ class NewsRunner(Runner): nntp_host, nntp_port = Utils.nntpsplit(mlist.nntp_host) conn = nntplib.NNTP(nntp_host, nntp_port, readermode=True, - user=mm_cfg.NNTP_USERNAME, - password=mm_cfg.NNTP_PASSWORD) + user=config.NNTP_USERNAME, + password=config.NNTP_PASSWORD) conn.post(fp) except nntplib.error_temp, e: log.error('(NNTPDirect) NNTP error for list "%s": %s', @@ -146,9 +147,9 @@ def prepare_message(mlist, msg, msgdata): # woon't completely sanitize the message, but it will eliminate the bulk # of the rejections based on message headers. The NNTP server may still # reject the message because of other problems. - for header in mm_cfg.NNTP_REMOVE_HEADERS: + for header in config.NNTP_REMOVE_HEADERS: del msg[header] - for header, rewrite in mm_cfg.NNTP_REWRITE_DUPLICATE_HEADERS: + for header, rewrite in config.NNTP_REWRITE_DUPLICATE_HEADERS: values = msg.get_all(header, []) if len(values) < 2: # We only care about duplicates diff --git a/Mailman/Queue/OutgoingRunner.py b/Mailman/Queue/OutgoingRunner.py index 3d4575ed2..351f24da6 100644 --- a/Mailman/Queue/OutgoingRunner.py +++ b/Mailman/Queue/OutgoingRunner.py @@ -12,7 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. """Outgoing queue runner.""" @@ -27,10 +28,10 @@ import logging from Mailman import Errors from Mailman import LockFile from Mailman import Message -from Mailman import mm_cfg from Mailman.Queue.BounceRunner import BounceMixin from Mailman.Queue.Runner import Runner from Mailman.Queue.Switchboard import Switchboard +from Mailman.configuration import config # This controls how often _doperiodic() will try to deal with deferred # permanent failures. It is a count of calls to _doperiodic() @@ -41,20 +42,20 @@ log = logging.getLogger('mailman.error') class OutgoingRunner(Runner, BounceMixin): - QDIR = mm_cfg.OUTQUEUE_DIR + QDIR = config.OUTQUEUE_DIR def __init__(self, slice=None, numslices=1): Runner.__init__(self, slice, numslices) BounceMixin.__init__(self) # We look this function up only at startup time - modname = 'Mailman.Handlers.' + mm_cfg.DELIVERY_MODULE + modname = 'Mailman.Handlers.' + config.DELIVERY_MODULE mod = __import__(modname) self._func = getattr(sys.modules[modname], 'process') # This prevents smtp server connection problems from filling up the # error log. It gets reset if the message was successfully sent, and # set if there was a socket.error. self.__logged = False - self.__retryq = Switchboard(mm_cfg.RETRYQUEUE_DIR) + self.__retryq = Switchboard(config.RETRYQUEUE_DIR) def _dispose(self, mlist, msg, msgdata): # See if we should retry delivery of this message again. @@ -75,13 +76,13 @@ class OutgoingRunner(Runner, BounceMixin): # There was a problem connecting to the SMTP server. Log this # once, but crank up our sleep time so we don't fill the error # log. - port = mm_cfg.SMTPPORT + port = config.SMTPPORT if port == 0: port = 'smtp' # Log this just once. if not self.__logged: log.error('Cannot connect to SMTP server %s on port %s', - mm_cfg.SMTPHOST, port) + config.SMTPHOST, port) self.__logged = True return True except Errors.SomeRecipientsFailed, e: @@ -115,7 +116,7 @@ class OutgoingRunner(Runner, BounceMixin): return False else: # Keep trying to delivery this message for a while - deliver_until = now + mm_cfg.DELIVERY_RETRY_PERIOD + deliver_until = now + config.DELIVERY_RETRY_PERIOD msgdata['last_recip_count'] = len(recips) msgdata['deliver_until'] = deliver_until msgdata['recips'] = recips diff --git a/Mailman/Queue/RetryRunner.py b/Mailman/Queue/RetryRunner.py index b6c0eac6b..da87982c1 100644 --- a/Mailman/Queue/RetryRunner.py +++ b/Mailman/Queue/RetryRunner.py @@ -12,23 +12,24 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. import time -from Mailman import mm_cfg from Mailman.Queue.Runner import Runner from Mailman.Queue.Switchboard import Switchboard +from Mailman.configuration import config class RetryRunner(Runner): - QDIR = mm_cfg.RETRYQUEUE_DIR - SLEEPTIME = mm_cfg.minutes(15) + QDIR = config.RETRYQUEUE_DIR + SLEEPTIME = config.minutes(15) def __init__(self, slice=None, numslices=1): Runner.__init__(self, slice, numslices) - self.__outq = Switchboard(mm_cfg.OUTQUEUE_DIR) + self.__outq = Switchboard(config.OUTQUEUE_DIR) def _dispose(self, mlist, msg, msgdata): # Move it to the out queue for another retry diff --git a/Mailman/Queue/Runner.py b/Mailman/Queue/Runner.py index 56baf431b..95cb3fd43 100644 --- a/Mailman/Queue/Runner.py +++ b/Mailman/Queue/Runner.py @@ -12,7 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. """Generic queue runner class.""" @@ -25,11 +26,11 @@ import email.Errors from cStringIO import StringIO from Mailman import Errors -from Mailman import i18n from Mailman import MailList -from Mailman import mm_cfg from Mailman import Utils +from Mailman import i18n from Mailman.Queue.Switchboard import Switchboard +from Mailman.configuration import config log = logging.getLogger('mailman.error') @@ -37,7 +38,7 @@ log = logging.getLogger('mailman.error') class Runner: QDIR = None - SLEEPTIME = mm_cfg.QRUNNER_SLEEP_TIME + SLEEPTIME = config.QRUNNER_SLEEP_TIME def __init__(self, slice=None, numslices=1): self._kids = {} @@ -45,7 +46,7 @@ class Runner: # we want to provide slice and numslice arguments. self._switchboard = Switchboard(self.QDIR, slice, numslices) # Create the shunt switchboard - self._shunt = Switchboard(mm_cfg.SHUNTQUEUE_DIR) + self._shunt = Switchboard(config.SHUNTQUEUE_DIR) self._stop = False def __repr__(self): @@ -132,7 +133,7 @@ class Runner: # Find out which mailing list this message is destined for. listname = msgdata.get('listname') if not listname: - listname = mm_cfg.MAILMAN_SITE_LIST + listname = config.MAILMAN_SITE_LIST mlist = self._open_list(listname) if not mlist: log.error('Dequeuing message destined for missing list: %s', @@ -153,7 +154,7 @@ class Runner: if mlist: lang = mlist.getMemberLanguage(sender) else: - lang = mm_cfg.DEFAULT_SERVER_LANGUAGE + lang = config.DEFAULT_SERVER_LANGUAGE i18n.set_language(lang) msgdata['lang'] = lang try: diff --git a/Mailman/Queue/Switchboard.py b/Mailman/Queue/Switchboard.py index 36b69283e..d9fc5dc27 100644 --- a/Mailman/Queue/Switchboard.py +++ b/Mailman/Queue/Switchboard.py @@ -12,7 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. """Reading and writing message objects and message metadata.""" @@ -41,8 +42,8 @@ import cPickle import marshal from Mailman import Message -from Mailman import mm_cfg from Mailman import Utils +from Mailman.configuration import config # 20 bytes of all bits set, maximum sha.digest() value shamax = 0xffffffffffffffffffffffffffffffffffffffffL @@ -104,7 +105,7 @@ class Switchboard: filename = os.path.join(self.__whichq, filebase + '.pck') tmpfile = filename + '.tmp' # Always add the metadata schema version number - data['version'] = mm_cfg.QFILE_SCHEMA_VERSION + data['version'] = config.QFILE_SCHEMA_VERSION # Filter out volatile entries for k in data.keys(): if k.startswith('_'): diff --git a/Mailman/Queue/VirginRunner.py b/Mailman/Queue/VirginRunner.py index 7d09ef8e9..bdb8a26bf 100644 --- a/Mailman/Queue/VirginRunner.py +++ b/Mailman/Queue/VirginRunner.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2006 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -12,7 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. """Virgin message queue runner. @@ -22,14 +23,14 @@ to go through some minimal processing before they can be sent out to the recipient. """ -from Mailman import mm_cfg -from Mailman.Queue.Runner import Runner from Mailman.Queue.IncomingRunner import IncomingRunner +from Mailman.Queue.Runner import Runner +from Mailman.configuration import config class VirginRunner(IncomingRunner): - QDIR = mm_cfg.VIRGINQUEUE_DIR + QDIR = config.VIRGINQUEUE_DIR def _dispose(self, mlist, msg, msgdata): # We need to fasttrack this message through any handlers that touch diff --git a/Mailman/Queue/__init__.py b/Mailman/Queue/__init__.py index 7fef22451..e69de29bb 100644 --- a/Mailman/Queue/__init__.py +++ b/Mailman/Queue/__init__.py @@ -1,15 +0,0 @@ -# Copyright (C) 2000,2001,2002 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. diff --git a/Mailman/Queue/sbcache.py b/Mailman/Queue/sbcache.py index dee1bca7f..6fe499d0d 100644 --- a/Mailman/Queue/sbcache.py +++ b/Mailman/Queue/sbcache.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2006 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -12,7 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. """A factory of Switchboards with caching.""" diff --git a/Mailman/Site.py b/Mailman/Site.py index 691d23870..72becb224 100644 --- a/Mailman/Site.py +++ b/Mailman/Site.py @@ -12,7 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. """Provide some customization for site-wide behavior. @@ -23,7 +24,7 @@ implementation should work for standard Mailman. import os import errno -from Mailman import mm_cfg +from Mailman.configuration import config @@ -54,7 +55,7 @@ def get_listpath(listname, domain=None, create=0): should not attempt to create the path heirarchy (and in fact the absence of the path might be significant). """ - path = os.path.join(mm_cfg.LIST_DATA_DIR, listname) + path = os.path.join(config.LIST_DATA_DIR, listname) if create: _makedir(path) return path @@ -79,9 +80,9 @@ def get_archpath(listname, domain=None, create=False, public=False): is usually a symlink instead of a directory). """ if public: - subdir = mm_cfg.PUBLIC_ARCHIVE_FILE_DIR + subdir = config.PUBLIC_ARCHIVE_FILE_DIR else: - subdir = mm_cfg.PRIVATE_ARCHIVE_FILE_DIR + subdir = config.PRIVATE_ARCHIVE_FILE_DIR path = os.path.join(subdir, listname) if create: _makedir(path) @@ -101,7 +102,7 @@ def get_listnames(domain=None): from Mailman.Utils import list_exists # We don't currently support separate virtual domain directories got = [] - for fn in os.listdir(mm_cfg.LIST_DATA_DIR): + for fn in os.listdir(config.LIST_DATA_DIR): if list_exists(fn): got.append(fn) return got diff --git a/Mailman/Utils.py b/Mailman/Utils.py index 42bacc16a..5e594be8d 100644 --- a/Mailman/Utils.py +++ b/Mailman/Utils.py @@ -15,7 +15,6 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, # USA. - """Miscellaneous essential routines. This includes actual message transmission routines, address checking and @@ -42,8 +41,8 @@ from string import ascii_letters, digits, whitespace from Mailman import Errors from Mailman import Site -from Mailman import mm_cfg from Mailman.SafeDict import SafeDict +from Mailman.configuration import config EMPTYSTRING = '' UEMPTYSTRING = u'' @@ -228,7 +227,7 @@ def ScriptURL(target, web_page_url=None, absolute=False): absolute - a flag which if set, generates an absolute url """ if web_page_url is None: - web_page_url = mm_cfg.DEFAULT_URL_PATTERN % get_domain() + web_page_url = config.DEFAULT_URL_PATTERN % get_domain() if web_page_url[-1] <> '/': web_page_url = web_page_url + '/' fullpath = os.environ.get('REQUEST_URI') @@ -247,7 +246,7 @@ def ScriptURL(target, web_page_url=None, absolute=False): path = ('../' * count) + target else: path = web_page_url + target - return path + mm_cfg.CGIEXT + return path + config.CGIEXT @@ -334,8 +333,10 @@ def Secure_MakeRandomPassword(length): os.close(fd) -def MakeRandomPassword(length=mm_cfg.MEMBER_PASSWORD_LENGTH): - if mm_cfg.USER_FRIENDLY_PASSWORDS: +def MakeRandomPassword(length=None): + if length is None: + length = config.MEMBER_PASSWORD_LENGTH + if config.USER_FRIENDLY_PASSWORDS: return UserFriendly_MakeRandomPassword(length) return Secure_MakeRandomPassword(length) @@ -356,9 +357,9 @@ def GetRandomSeed(): def set_global_password(pw, siteadmin=True): if siteadmin: - filename = mm_cfg.SITE_PW_FILE + filename = config.SITE_PW_FILE else: - filename = mm_cfg.LISTCREATOR_PW_FILE + filename = config.LISTCREATOR_PW_FILE # rw-r----- omask = os.umask(026) try: @@ -371,9 +372,9 @@ def set_global_password(pw, siteadmin=True): def get_global_password(siteadmin=True): if siteadmin: - filename = mm_cfg.SITE_PW_FILE + filename = config.SITE_PW_FILE else: - filename = mm_cfg.LISTCREATOR_PW_FILE + filename = config.LISTCREATOR_PW_FILE try: fp = open(filename) challenge = fp.read()[:-1] # strip off trailing nl @@ -486,14 +487,14 @@ def findtext(templatefile, dict=None, raw=False, lang=None, mlist=None): languages.append(lang) if mlist is not None: languages.append(mlist.preferred_language) - languages.append(mm_cfg.DEFAULT_SERVER_LANGUAGE) + languages.append(config.DEFAULT_SERVER_LANGUAGE) # Calculate the locations to scan searchdirs = [] if mlist is not None: searchdirs.append(mlist.fullpath()) - searchdirs.append(os.path.join(mm_cfg.TEMPLATE_DIR, mlist.host_name)) - searchdirs.append(os.path.join(mm_cfg.TEMPLATE_DIR, 'site')) - searchdirs.append(mm_cfg.TEMPLATE_DIR) + searchdirs.append(os.path.join(config.TEMPLATE_DIR, mlist.host_name)) + searchdirs.append(os.path.join(config.TEMPLATE_DIR, 'site')) + searchdirs.append(config.TEMPLATE_DIR) # Start scanning quickexit = 'quickexit' fp = None @@ -514,7 +515,7 @@ def findtext(templatefile, dict=None, raw=False, lang=None, mlist=None): # Try one last time with the distro English template, which, unless # you've got a really broken installation, must be there. try: - filename = os.path.join(mm_cfg.TEMPLATE_DIR, 'en', templatefile) + filename = os.path.join(config.TEMPLATE_DIR, 'en', templatefile) fp = open(filename) except IOError, e: if e.errno <> errno.ENOENT: raise @@ -573,7 +574,7 @@ def is_administrivia(msg): break if line.strip(): linecnt += 1 - if linecnt > mm_cfg.DEFAULT_MAIL_COMMANDS_MAX_LINES: + if linecnt > config.DEFAULT_MAIL_COMMANDS_MAX_LINES: return False lines.append(line) bodytext = NL.join(lines) @@ -652,14 +653,14 @@ def reap(kids, func=None, once=False): def GetLanguageDescr(lang): - return mm_cfg.LC_DESCRIPTIONS[lang][0] + return config.LC_DESCRIPTIONS[lang][0] def GetCharSet(lang): - return mm_cfg.LC_DESCRIPTIONS[lang][1] + return config.LC_DESCRIPTIONS[lang][1] def IsLanguage(lang): - return mm_cfg.LC_DESCRIPTIONS.has_key(lang) + return config.LC_DESCRIPTIONS.has_key(lang) @@ -669,23 +670,23 @@ def get_domain(): # Strip off the port if there is one if port and host.endswith(':' + port): host = host[:-len(port)-1] - if mm_cfg.VIRTUAL_HOST_OVERVIEW and host: + if config.VIRTUAL_HOST_OVERVIEW and host: return host.lower() else: # See the note in Defaults.py concerning DEFAULT_URL # vs. DEFAULT_URL_HOST. - hostname = ((mm_cfg.DEFAULT_URL - and urlparse.urlparse(mm_cfg.DEFAULT_URL)[1]) - or mm_cfg.DEFAULT_URL_HOST) + hostname = ((config.DEFAULT_URL + and urlparse.urlparse(config.DEFAULT_URL)[1]) + or config.DEFAULT_URL_HOST) return hostname.lower() def get_site_email(hostname=None, extra=None): if hostname is None: - hostname = mm_cfg.VIRTUAL_HOSTS.get(get_domain(), get_domain()) + hostname = config.VIRTUAL_HOSTS.get(get_domain(), get_domain()) if extra is None: - return '%s@%s' % (mm_cfg.MAILMAN_SITE_LIST, hostname) - return '%s-%s@%s' % (mm_cfg.MAILMAN_SITE_LIST, extra, hostname) + return '%s@%s' % (config.MAILMAN_SITE_LIST, hostname) + return '%s-%s@%s' % (config.MAILMAN_SITE_LIST, extra, hostname) diff --git a/Mailman/Version.py b/Mailman/Version.py index 6c9aad05e..91fb82e65 100644 --- a/Mailman/Version.py +++ b/Mailman/Version.py @@ -46,3 +46,7 @@ PENDING_FILE_SCHEMA_VERSION = 2 # version number for the lists/<listname>/request.db file schema REQUESTS_FILE_SCHEMA_VERSION = 1 + +# Printable version string used by command line scripts +MAILMAN_VERSION = 'GNU Mailman ' + VERSION + diff --git a/Mailman/bin/list_lists.py b/Mailman/bin/list_lists.py index a3493c3b8..0feac2845 100644 --- a/Mailman/bin/list_lists.py +++ b/Mailman/bin/list_lists.py @@ -18,9 +18,11 @@ import sys import optparse +from Mailman import Defaults from Mailman import MailList from Mailman import Utils -from Mailman import mm_cfg +from Mailman import Version +from Mailman.configuration import config from Mailman.i18n import _ __i18n_templates__ = True @@ -28,7 +30,7 @@ __i18n_templates__ = True def parseargs(): - parser = optparse.OptionParser(version=mm_cfg.MAILMAN_VERSION, + parser = optparse.OptionParser(version=Version.MAILMAN_VERSION, usage=_("""\ %prog [options] @@ -46,6 +48,8 @@ Displays only the list name, with no description.""")) help=_("""\ List only those mailing lists that are homed to the given virtual domain. This only works if the VIRTUAL_HOST_OVERVIEW variable is set.""")) + parser.add_option('-C', '--config', + help=_('Alternative configuration file to use')) opts, args = parser.parse_args() if args: parser.print_help() @@ -57,6 +61,7 @@ This only works if the VIRTUAL_HOST_OVERVIEW variable is set.""")) def main(): parser, opts, args = parseargs() + config.load(opts.config) names = Utils.list_names() names.sort() @@ -67,7 +72,7 @@ def main(): mlist = MailList.MailList(n, lock=False) if opts.advertised and not mlist.advertised: continue - if opts.vhost and mm_cfg.VIRTUAL_HOST_OVERVIEW and \ + if opts.vhost and config.VIRTUAL_HOST_OVERVIEW and \ opts.vhost.find(mlist.web_page_url) == -1 and \ mlist.web_page_url.find(opts.vhost) == -1: continue diff --git a/Mailman/bin/mailmanctl.py b/Mailman/bin/mailmanctl.py new file mode 100644 index 000000000..a91b5651b --- /dev/null +++ b/Mailman/bin/mailmanctl.py @@ -0,0 +1,521 @@ +# Copyright (C) 2001-2006 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +import os +import grp +import pwd +import sys +import errno +import signal +import socket +import logging +import optparse + +from Mailman import Defaults +from Mailman import Errors +from Mailman import LockFile +from Mailman import Utils +from Mailman import Version +from Mailman import loginit +from Mailman.MailList import MailList +from Mailman.configuration import config +from Mailman.i18n import _ + +__i18n_templates__ = True + +COMMASPACE = ', ' +DOT = '.' + +# Since we wake up once per day and refresh the lock, the LOCK_LIFETIME +# needn't be (much) longer than SNOOZE. We pad it 6 hours just to be safe. +LOCK_LIFETIME = Defaults.days(1) + Defaults.hours(6) +SNOOZE = Defaults.days(1) +MAX_RESTARTS = 10 + + + +def parseargs(): + parser = optparse.OptionParser(version=Version.MAILMAN_VERSION, + usage=_("""\ +Primary start-up and shutdown script for Mailman's qrunner daemon. + +This script starts, stops, and restarts the main Mailman queue runners, making +sure that the various long-running qrunners are still alive and kicking. It +does this by forking and exec'ing the qrunners and waiting on their pids. +When it detects a subprocess has exited, it may restart it. + +The qrunners respond to SIGINT, SIGTERM, and SIGHUP. SIGINT and SIGTERM both +cause the qrunners to exit cleanly, but the master will only restart qrunners +that have exited due to a SIGINT. SIGHUP causes the master and the qrunners +to close their log files, and reopen then upon the next printed message. + +The master also responds to SIGINT, SIGTERM, and SIGHUP, which it simply +passes on to the qrunners (note that the master will close and reopen its own +log files on receipt of a SIGHUP). The master also leaves its own process id +in the file data/master-qrunner.pid but you normally don't need to use this +pid directly. The `start', `stop', `restart', and `reopen' commands handle +everything for you. + +Commands: + + start - Start the master daemon and all qrunners. Prints a message and + exits if the master daemon is already running. + + stop - Stops the master daemon and all qrunners. After stopping, no + more messages will be processed. + + restart - Restarts the qrunners, but not the master process. Use this + whenever you upgrade or update Mailman so that the qrunners will + use the newly installed code. + + reopen - This will close all log files, causing them to be re-opened the + next time a message is written to them + +Usage: %prog [options] [ start | stop | restart | reopen ]""")) + parser.add_option('-n', '--no-restart', + dest='restart', default=True, action='store_false', + help=_("""\ +Don't restart the qrunners when they exit because of an error or a SIGINT. +They are never restarted if they exit in response to a SIGTERM. Use this only +for debugging. Only useful if the `start' command is given.""")) + parser.add_option('-u', '--run-as-user', + dest='checkprivs', default=True, action='store_false', + help=_("""\ +Normally, this script will refuse to run if the user id and group id are not +set to the `mailman' user and group (as defined when you configured Mailman). +If run as root, this script will change to this user and group before the +check is made. + +This can be inconvenient for testing and debugging purposes, so the -u flag +means that the step that sets and checks the uid/gid is skipped, and the +program is run as the current user and group. This flag is not recommended +for normal production environments. + +Note though, that if you run with -u and are not in the mailman group, you may +have permission problems, such as begin unable to delete a list's archives +through the web. Tough luck!""")) + parser.add_option('-s', '--stale-lock-cleanup', + dest='force', default=False, action='store_true', + help=_("""\ +If mailmanctl finds an existing master lock, it will normally exit with an +error message. With this option, mailmanctl will perform an extra level of +checking. If a process matching the host/pid described in the lock file is +running, mailmanctl will still exit, but if no matching process is found, +mailmanctl will remove the apparently stale lock and make another attempt to +claim the master lock.""")) + parser.add_option('-q', '--quiet', + default=False, action='store_true', + help=_("""\ +Don't print status messages. Error messages are still printed to standard +error.""")) + parser.add_option('-C', '--config', + help=_('Alternative configuration file to use')) + opts, args = parser.parse_args() + if not args: + parser.print_help() + print >> sys.stderr, _('No command given.') + sys.exit(1) + if len(args) > 1: + parse.print_help() + commands = COMMASPACE.join(args) + print >> sys.stderr, _('Bad command: $commands') + sys.exit(1) + return parser, opts, args + + + +def kill_watcher(sig): + try: + fp = open(config.PIDFILE) + pidstr = fp.read() + fp.close() + pid = int(pidstr.strip()) + except (IOError, ValueError), e: + # For i18n convenience + pidfile = config.PIDFILE + print >> sys.stderr, _('PID unreadable in: $pidfile') + print >> sys.stderr, e + print >> sys.stderr, _('Is qrunner even running?') + return + try: + os.kill(pid, sig) + except OSError, e: + if e.errno <> errno.ESRCH: raise + print >> sys.stderr, _('No child with pid: $pid') + print >> sys.stderr, e + print >> sys.stderr, _('Stale pid file removed.') + os.unlink(config.PIDFILE) + + + +def get_lock_data(): + # Return the hostname, pid, and tempfile + fp = open(config.LOCKFILE) + try: + filename = os.path.split(fp.read().strip())[1] + finally: + fp.close() + parts = filename.split('.') + hostname = DOT.join(parts[1:-1]) + pid = int(parts[-1]) + return hostname, int(pid), filename + + +def qrunner_state(): + # 1 if proc exists on host (but is it qrunner? ;) + # 0 if host matches but no proc + # hostname if hostname doesn't match + hostname, pid, tempfile = get_lock_data() + if hostname <> socket.gethostname(): + return hostname + # Find out if the process exists by calling kill with a signal 0. + try: + os.kill(pid, 0) + except OSError, e: + if e.errno <> errno.ESRCH: + raise + return 0 + return 1 + + +def acquire_lock_1(force): + # Be sure we can acquire the master qrunner lock. If not, it means some + # other master qrunner daemon is already going. + lock = LockFile.LockFile(config.LOCK_FILE, LOCK_LIFETIME) + try: + lock.lock(0.1) + return lock + except LockFile.TimeOutError: + if not force: + raise + # Force removal of lock first + lock._disown() + hostname, pid, tempfile = get_lock_data() + os.unlink(config.LOCKFILE) + os.unlink(os.path.join(config.LOCK_DIR, tempfile)) + return acquire_lock_1(force=False) + + +def acquire_lock(force): + try: + lock = acquire_lock_1(force) + return lock + except LockFile.TimeOutError: + status = qrunner_state() + if status == 1: + # host matches and proc exists + print >> sys.stderr, _("""\ +The master qrunner lock could not be acquired because it appears as if another +master qrunner is already running. +""") + elif status == 0: + # host matches but no proc + print >> sys.stderr, _("""\ +The master qrunner lock could not be acquired. It appears as though there is +a stale master qrunner lock. Try re-running mailmanctl with the -s flag. +""") + else: + # host doesn't even match + print >> sys.stderr, _("""\ +The master qrunner lock could not be acquired, because it appears as if some +process on some other host may have acquired it. We can't test for stale +locks across host boundaries, so you'll have to do this manually. Or, if you +know the lock is stale, re-run mailmanctl with the -s flag. + +Lock file: $config.LOCKFILE +Lock host: $status + +Exiting.""") + + + +def start_runner(qrname, slice, count): + pid = os.fork() + if pid: + # parent + return pid + # child + # + # Craft the command line arguments for the exec() call. + rswitch = '--runner=%s:%d:%d' % (qrname, slice, count) + exe = os.path.join(config.BIN_DIR, 'qrunner') + # config.PYTHON, which is the absolute path to the Python interpreter, + # must be given as argv[0] due to Python's library search algorithm. + os.execl(config.PYTHON, config.PYTHON, exe, rswitch, '-s') + # Should never get here + raise RuntimeError, 'os.execl() failed' + + +def start_all_runners(): + kids = {} + for qrname, count in config.QRUNNERS: + for slice in range(count): + # queue runner name, slice, numslices, restart count + info = (qrname, slice, count, 0) + pid = start_runner(qrname, slice, count) + kids[pid] = info + return kids + + + +def check_for_site_list(): + sitelistname = config.MAILMAN_SITE_LIST + try: + sitelist = MailList(sitelistname, lock=False) + except Errors.MMUnknownListError: + print >> sys.stderr, _('Site list is missing: $sitelistname') + elog.error('Site list is missing: %s', config.MAILMAN_SITE_LIST) + sys.exit(1) + + +def check_privs(): + # If we're running as root (uid == 0), coerce the uid and gid to that + # which Mailman was configured for, and refuse to run if we didn't coerce + # the uid/gid. + gid = grp.getgrnam(config.MAILMAN_GROUP)[2] + uid = pwd.getpwnam(config.MAILMAN_USER)[2] + myuid = os.getuid() + if myuid == 0: + # Set the process's supplimental groups. + groups = [x[2] for x in grp.getgrall() if config.MAILMAN_USER in x[3]] + groups.append(gid) + os.setgroups(groups) + os.setgid(gid) + os.setuid(uid) + elif myuid <> uid: + name = config.MAILMAN_USER + usage(1, _( + 'Run this program as root or as the $name user, or use -u.')) + + + +def main(): + global elog, qlog + + parser, opts, args = parseargs() + config.load(opts.config) + + loginit.initialize() + elog = logging.getLogger('mailman.error') + qlog = logging.getLogger('mailman.qrunner') + + if opts.checkprivs: + check_privs() + else: + print _('Warning! You may encounter permission problems.') + + # Handle the commands + command = args[0].lower() + if command == 'stop': + # Sent the master qrunner process a SIGINT, which is equivalent to + # giving cron/qrunner a ctrl-c or KeyboardInterrupt. This will + # effectively shut everything down. + if not opts.quiet: + print _("Shutting down Mailman's master qrunner") + kill_watcher(signal.SIGTERM) + elif command == 'restart': + # Sent the master qrunner process a SIGHUP. This will cause the + # master qrunner to kill and restart all the worker qrunners, and to + # close and re-open its log files. + if not opts.quiet: + print _("Restarting Mailman's master qrunner") + kill_watcher(signal.SIGINT) + elif command == 'reopen': + if not opts.quiet: + print _('Re-opening all log files') + kill_watcher(signal.SIGHUP) + elif command == 'start': + # First, complain loudly if there's no site list. + check_for_site_list() + # Here's the scoop on the processes we're about to create. We'll need + # one for each qrunner, and one for a master child process watcher / + # lock refresher process. + # + # The child watcher process simply waits on the pids of the children + # qrunners. Unless explicitly disabled by a mailmanctl switch (or the + # children are killed with SIGTERM instead of SIGINT), the watcher + # will automatically restart any child process that exits. This + # allows us to be more robust, and also to implement restart by simply + # SIGINT'ing the qrunner children, and letting the watcher restart + # them. + # + # Under normal operation, we have a child per queue. This lets us get + # the most out of the available resources, since a qrunner with no + # files in its queue directory is pretty cheap, but having a separate + # runner process per queue allows for a very responsive system. Some + # people want a more traditional (i.e. MM2.0.x) cron-invoked qrunner. + # No problem, but using mailmanctl isn't the answer. So while + # mailmanctl hard codes some things, others, such as the number of + # qrunners per queue, are configurable. + # + # First, acquire the master mailmanctl lock + lock = acquire_lock(opts.force) + if not lock: + return + # Daemon process startup according to Stevens, Advanced Programming in + # the UNIX Environment, Chapter 13. + pid = os.fork() + if pid: + # parent + if not opts.quiet: + print _("Starting Mailman's master qrunner.") + # Give up the lock "ownership". This just means the foreground + # process won't close/unlock the lock when it finalizes this lock + # instance. We'll let the mater watcher subproc own the lock. + lock._transfer_to(pid) + return + # child + lock._take_possession() + # First, save our pid in a file for "mailmanctl stop" rendezvous. We + # want the perms on the .pid file to be rw-rw---- + omask = os.umask(6) + try: + fp = open(config.PIDFILE, 'w') + print >> fp, os.getpid() + fp.close() + finally: + os.umask(omask) + # Create a new session and become the session leader, but since we + # won't be opening any terminal devices, don't do the ultra-paranoid + # suggestion of doing a second fork after the setsid() call. + os.setsid() + # Instead of cd'ing to root, cd to the Mailman installation home + os.chdir(config.PREFIX) + # Set our file mode creation umask + os.umask(007) + # I don't think we have any unneeded file descriptors. + # + # Now start all the qrunners. This returns a dictionary where the + # keys are qrunner pids and the values are tuples of the following + # form: (qrname, slice, count). This does its own fork and exec, and + # sets up its own signal handlers. + kids = start_all_runners() + # Set up a SIGALRM handler to refresh the lock once per day. The lock + # lifetime is 1day+6hours so this should be plenty. + def sigalrm_handler(signum, frame): + lock.refresh() + signal.alarm(Defaults.days(1)) + signal.signal(signal.SIGALRM, sigalrm_handler) + signal.alarm(Defaults.days(1)) + # Set up a SIGHUP handler so that if we get one, we'll pass it along + # to all the qrunner children. This will tell them to close and + # reopen their log files + def sighup_handler(signum, frame): + loginit.reopen() + for pid in kids.keys(): + os.kill(pid, signal.SIGHUP) + # And just to tweak things... + qlog.info('Master watcher caught SIGHUP. Re-opening log files.') + signal.signal(signal.SIGHUP, sighup_handler) + # We also need to install a SIGTERM handler because that's what init + # will kill this process with when changing run levels. + def sigterm_handler(signum, frame): + for pid in kids.keys(): + try: + os.kill(pid, signal.SIGTERM) + except OSError, e: + if e.errno <> errno.ESRCH: raise + qlog.info('Master watcher caught SIGTERM. Exiting.') + signal.signal(signal.SIGTERM, sigterm_handler) + # Finally, we need a SIGINT handler which will cause the sub-qrunners + # to exit, but the master will restart SIGINT'd sub-processes unless + # the -n flag was given. + def sigint_handler(signum, frame): + for pid in kids.keys(): + os.kill(pid, signal.SIGINT) + qlog.info('Master watcher caught SIGINT. Restarting.') + signal.signal(signal.SIGINT, sigint_handler) + # Now we're ready to simply do our wait/restart loop. This is the + # master qrunner watcher. + try: + while True: + try: + pid, status = os.wait() + except OSError, e: + # No children? We're done + if e.errno == errno.ECHILD: + break + # If the system call got interrupted, just restart it. + elif e.errno <> errno.EINTR: + raise + continue + killsig = exitstatus = None + if os.WIFSIGNALED(status): + killsig = os.WTERMSIG(status) + if os.WIFEXITED(status): + exitstatus = os.WEXITSTATUS(status) + # We'll restart the process unless we were given the + # "no-restart" switch, or if the process was SIGTERM'd or + # exitted with a SIGTERM exit status. This lets us better + # handle runaway restarts (say, if the subproc had a syntax + # error!) + restarting = '' + if opts.restart: + if ((exitstatus == None and killsig <> signal.SIGTERM) or + (killsig == None and exitstatus <> signal.SIGTERM)): + # Then + restarting = '[restarting]' + qrname, slice, count, restarts = kids[pid] + del kids[pid] + qlog.info("""\ +Master qrunner detected subprocess exit +(pid: %d, sig: %s, sts: %s, class: %s, slice: %d/%d) %s""", + pid, killsig, exitstatus, qrname, + slice+1, count, restarting) + # See if we've reached the maximum number of allowable restarts + if exitstatus <> signal.SIGINT: + restarts += 1 + if restarts > MAX_RESTARTS: + qlog.info("""\ +Qrunner %s reached maximum restart limit of %d, not restarting.""", + qrname, MAX_RESTARTS) + restarting = '' + # Now perhaps restart the process unless it exited with a + # SIGTERM or we aren't restarting. + if restarting: + newpid = start_runner(qrname, slice, count) + kids[newpid] = (qrname, slice, count, restarts) + finally: + # Should we leave the main loop for any reason, we want to be sure + # all of our children are exited cleanly. Send SIGTERMs to all + # the child processes and wait for them all to exit. + for pid in kids.keys(): + try: + os.kill(pid, signal.SIGTERM) + except OSError, e: + if e.errno == errno.ESRCH: + # The child has already exited + qlog.info('ESRCH on pid: %d', pid) + del kids[pid] + # Wait for all the children to go away + while True: + try: + pid, status = os.wait() + except OSError, e: + if e.errno == errno.ECHILD: + break + elif e.errno <> errno.EINTR: + raise + continue + # Finally, give up the lock + lock.unlock(unconditionally=True) + os._exit(0) + + + +if __name__ == '__main__': + main() diff --git a/Mailman/bin/newlist.py b/Mailman/bin/newlist.py index 26c7cd42b..0f62f5373 100644 --- a/Mailman/bin/newlist.py +++ b/Mailman/bin/newlist.py @@ -25,8 +25,9 @@ from Mailman import Errors from Mailman import MailList from Mailman import Message from Mailman import Utils +from Mailman import Version from Mailman import i18n -from Mailman import mm_cfg +from Mailman.configuration import config _ = i18n._ @@ -35,7 +36,7 @@ __i18n_templates__ = True def parseargs(): - parser = optparse.OptionParser(version=mm_cfg.MAILMAN_VERSION, + parser = optparse.OptionParser(version=Version.MAILMAN_VERSION, usage=_("""\ %%prog [options] [listname [listadmin-addr [admin-password]]] @@ -87,7 +88,6 @@ defined in your Defaults.py file or overridden by settings in mm_cfg.py). Note that listnames are forced to lowercase.""")) parser.add_option('-l', '--language', type='string', action='store', - default=mm_cfg.DEFAULT_SERVER_LANGUAGE, help=_("""\ Make the list's preferred language LANGUAGE, which must be a two letter language code.""")) @@ -110,18 +110,27 @@ This option suppresses the prompt prior to administrator notification but still sends the notification. It can be used to make newlist totally non-interactive but still send the notification, assuming listname, listadmin-addr and admin-password are all specified on the command line.""")) + parser.add_option('-C', '--config', + help=_('Alternative configuration file to use')) opts, args = parser.parse_args() - # Is the language known? - if opts.language not in mm_cfg.LC_DESCRIPTIONS: - parser.print_help() - print >> sys.stderr, _('Unknown language: $opts.language') - sys.exit(1) + # Can't verify opts.language here because the configuration isn't loaded + # yet. return parser, opts, args def main(): parser, opts, args = parseargs() + config.load(opts.config) + + # Set up some defaults we couldn't set up in parseargs() + if opts.language is None: + opts.language = config.DEFAULT_SERVER_LANGUAGE + # Is the language known? + if opts.language not in config.LC_DESCRIPTIONS: + parser.print_help() + print >> sys.stderr, _('Unknown language: $opts.language') + sys.exit(1) # Handle variable number of positional arguments if args: @@ -134,12 +143,12 @@ def main(): # Note that --urlhost and --emailhost have precedence listname, domain = listname.split('@', 1) urlhost = opts.urlhost or domain - emailhost = opts.emailhost or mm_cfg.VIRTUAL_HOSTS.get(domain, domain) + emailhost = opts.emailhost or config.VIRTUAL_HOSTS.get(domain, domain) - urlhost = opts.urlhost or mm_cfg.DEFAULT_URL_HOST + urlhost = opts.urlhost or config.DEFAULT_URL_HOST host_name = (opts.emailhost or - mm_cfg.VIRTUAL_HOSTS.get(urlhost, mm_cfg.DEFAULT_EMAIL_HOST)) - web_page_url = mm_cfg.DEFAULT_URL_PATTERN % urlhost + config.VIRTUAL_HOSTS.get(urlhost, config.DEFAULT_EMAIL_HOST)) + web_page_url = config.DEFAULT_URL_PATTERN % urlhost if Utils.list_exists(listname): parser.print_help() @@ -154,7 +163,12 @@ def main(): if args: listpasswd = args.pop(0) else: - listpasswd = getpass.getpass(_('Initial $listname password: ')) + while True: + listpasswd = getpass.getpass(_('Initial $listname password: ')) + confirm = getpass.getpass(_('Confirm $listname password: ')) + if listpasswd == confirm: + break + print _('Passwords did not match, try again (Ctrl-C to quit)') # List passwords cannot be empty listpasswd = listpasswd.strip() @@ -198,8 +212,8 @@ def main(): mlist.Unlock() # Now do the MTA-specific list creation tasks - if mm_cfg.MTA: - modname = 'Mailman.MTA.' + mm_cfg.MTA + if config.MTA: + modname = 'Mailman.MTA.' + config.MTA __import__(modname) sys.modules[modname].create(mlist) diff --git a/Mailman/bin/qrunner.py b/Mailman/bin/qrunner.py new file mode 100644 index 000000000..1fab69d98 --- /dev/null +++ b/Mailman/bin/qrunner.py @@ -0,0 +1,258 @@ +# Copyright (C) 2001-2006 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +import sys +import signal +import logging +import optparse + +from Mailman import Version +from Mailman import loginit +from Mailman.configuration import config +from Mailman.i18n import _ + +__i18n_templates__ = True + +COMMASPACE = ', ' +log = None + + + +def r_callback(option, opt, value, parser): + dest = getattr(parser.values, option.dest) + parts = value.split(':') + if len(parts) == 1: + runner = parts[0] + rslice = rrange = 1 + elif len(parts) == 3: + runner = parts[0] + try: + rslice = int(parts[1]) + rrange = int(parts[2]) + except ValueError: + parser.print_help() + print >> sys.stderr, _('Bad runner specification: $value') + sys.exit(1) + else: + parser.print_help() + print >> sys.stderr, _('Bad runner specification: $value') + sys.exit(1) + if runner == 'All': + for runnername, slices in config.QRUNNERS: + dest.append((runnername, rslice, rrange)) + elif not runner.endswith('Runner'): + runner += 'Runner' + dest.append((runner, rslice, rrange)) + + + +def parseargs(): + parser = optparse.OptionParser(version=Version.MAILMAN_VERSION, + usage=_("""\ +Run one or more qrunners, once or repeatedly. + +Each named runner class is run in round-robin fashion. In other words, the +first named runner is run to consume all the files currently in its +directory. When that qrunner is done, the next one is run to consume all the +files in /its/ directory, and so on. The number of total iterations can be +given on the command line. + +Usage: %prog [options] + +-r is required unless -l or -h is given, and its argument must be one of the +names displayed by the -l switch. + +Normally, this script should be started from mailmanctl. Running it +separately or with -o is generally useful only for debugging. +""")) + parser.add_option('-r', '--runner', + metavar='runner[:slice:range]', dest='runners', + type='string', default=[], + action='callback', callback=r_callback, + help=_("""\ +Run the named qrunner, which must be one of the strings returned by the -l +option. Optional slice:range if given, is used to assign multiple qrunner +processes to a queue. range is the total number of qrunners for this queue +while slice is the number of this qrunner from [0..range). + +When using the slice:range form, you must ensure that each qrunner for the +queue is given the same range value. If slice:runner is not given, then 1:1 +is used. + +Multiple -r options may be given, in which case each qrunner will run once in +round-robin fashion. The special runner `All' is shorthand for a qrunner for +each listed by the -l option.""")) + parser.add_option('-o', '--once', + default=False, action='store_true', help=_("""\ +Run each named qrunner exactly once through its main loop. Otherwise, each +qrunner runs indefinitely, until the process receives a SIGTERM or SIGINT.""")) + parser.add_option('-l', '--list', + default=False, action='store_true', + help=_('List the available qrunner names and exit.')) + parser.add_option('-v', '--verbose', + default=0, action='count', help=_("""\ +Display more debugging information to the logs/qrunner log file.""")) + parser.add_option('-s', '--subproc', + default=False, action='store_true', help=_("""\ +This should only be used when running qrunner as a subprocess of the +mailmanctl startup script. It changes some of the exit-on-error behavior to +work better with that framework.""")) + parser.add_option('-C', '--config', + help=_('Alternative configuration file to use')) + opts, args = parser.parse_args() + if args: + parser.print_help() + print >> sys.stderr, _('Unexpected arguments') + sys.exit(1) + if not opts.runners and not opts.list: + parser.print_help() + print >> sys.stderr, _('No runner name given.') + sys.exit(1) + return parser, opts, args + + + +def make_qrunner(name, slice, range, once=False): + modulename = 'Mailman.Queue.' + name + try: + __import__(modulename) + except ImportError, e: + if opts.subproc: + # Exit with SIGTERM exit code so mailmanctl won't try to restart us + print >> sys.stderr, _('Cannot import runner module: $modulename') + print >> sys.stderr, e + sys.exit(signal.SIGTERM) + else: + print >> sys.stderr, e + sys.exit(1) + qrclass = getattr(sys.modules[modulename], name) + if once: + # Subclass to hack in the setting of the stop flag in _doperiodic() + class Once(qrclass): + def _doperiodic(self): + self.stop() + qrunner = Once(slice, range) + else: + qrunner = qrclass(slice, range) + return qrunner + + + +def set_signals(loop): + # Set up the SIGTERM handler for stopping the loop + def sigterm_handler(signum, frame): + # Exit the qrunner cleanly + loop.stop() + loop.status = signal.SIGTERM + log.info('%s qrunner caught SIGTERM. Stopping.', loop.name()) + signal.signal(signal.SIGTERM, sigterm_handler) + # Set up the SIGINT handler for stopping the loop. For us, SIGINT is + # the same as SIGTERM, but our parent treats the exit statuses + # differently (it restarts a SIGINT but not a SIGTERM). + def sigint_handler(signum, frame): + # Exit the qrunner cleanly + loop.stop() + loop.status = signal.SIGINT + log.info('%s qrunner caught SIGINT. Stopping.', loop.name()) + signal.signal(signal.SIGINT, sigint_handler) + # SIGHUP just tells us to rotate our log files. + def sighup_handler(signum, frame): + loginit.reopen() + log.info('%s qrunner caught SIGHUP. Reopening logs.', loop.name()) + signal.signal(signal.SIGHUP, sighup_handler) + + + +def main(): + global log, opts + + parser, opts, args = parseargs() + config.load(opts.config) + + # If we're not running as a subprocess of mailmanctl, then we'll log to + # stderr in addition to logging to the log files. We do this by passing a + # value of True to propagate, which allows the 'mailman' root logger to + # see the log messages. + loginit.initialize(propagate=not opts.subproc) + log = logging.getLogger('mailman.qrunner') + + if opts.list: + for runnername, slices in config.QRUNNERS: + if runnername.endswith('Runner'): + name = runnername[:-len('Runner')] + else: + name = runnername + print _('$name runs the $runnername qrunner') + print _('All runs all the above qrunners') + sys.exit(0) + + # Fast track for one infinite runner + if len(opts.runners) == 1 and not opts.once: + qrunner = make_qrunner(*opts.runners[0]) + class Loop: + status = 0 + def __init__(self, qrunner): + self._qrunner = qrunner + def name(self): + return self._qrunner.__class__.__name__ + def stop(self): + self._qrunner.stop() + loop = Loop(qrunner) + set_signals(loop) + # Now start up the main loop + log.info('%s qrunner started.', loop.name()) + qrunner.run() + log.info('%s qrunner exiting.', loop.name()) + else: + # Anything else we have to handle a bit more specially + qrunners = [] + for runner, rslice, rrange in opts.runners: + qrunner = make_qrunner(runner, rslice, rrange, once=True) + qrunners.append(qrunner) + # This class is used to manage the main loop + class Loop: + status = 0 + def __init__(self): + self._isdone = False + def name(self): + return 'Main loop' + def stop(self): + self._isdone = True + def isdone(self): + return self._isdone + loop = Loop() + set_signals(loop) + log.info('Main qrunner loop started.') + while not loop.isdone(): + for qrunner in qrunners: + # In case the SIGTERM came in the middle of this iteration + if loop.isdone(): + break + if opts.verbose: + log.info('Now doing a %s qrunner iteration', + qrunner.__class__.__bases__[0].__name__) + qrunner.run() + if opts.once: + break + log.info('Main qrunner loop exiting.') + # All done + sys.exit(loop.status) + + + +if __name__ == '__main__': + main() diff --git a/Mailman/bin/rmlist.py b/Mailman/bin/rmlist.py index aef927c88..9655327f0 100644 --- a/Mailman/bin/rmlist.py +++ b/Mailman/bin/rmlist.py @@ -22,7 +22,8 @@ import optparse from Mailman import MailList from Mailman import Utils -from Mailman import mm_cfg +from Mailman import Version +from Mailman.configuration import config from Mailman.i18n import _ __i18n_templates__ = True @@ -44,7 +45,7 @@ def remove_it(listname, filename, msg): def parseargs(): - parser = optparse.OptionParser(version=mm_cfg.MAILMAN_VERSION, + parser = optparse.OptionParser(version=Version.MAILMAN_VERSION, usage=_("""\ %prog [options] listname @@ -57,6 +58,8 @@ archives are not removed, which is very handy for retiring old lists. default=False, action='store_true', help=_("""\ Remove the list's archives too, or if the list has already been deleted, remove any residual archives.""")) + parser.add_option('-C', '--config', + help=_('Alternative configuration file to use')) opts, args = parser.parse_args() if not args: parser.print_help() @@ -72,6 +75,8 @@ remove any residual archives.""")) def main(): parser, opts, args = parseargs() + config.load(opts.config) + listname = args[0].lower().strip() if not Utils.list_exists(listname): if not opts.archives: @@ -89,18 +94,18 @@ def main(): if Utils.list_exists(listname): mlist = MailList.MailList(listname, lock=False) # Do the MTA-specific list deletion tasks - if mm_cfg.MTA: - modname = 'Mailman.MTA.' + mm_cfg.MTA + if config.MTA: + modname = 'Mailman.MTA.' + config.MTA __import__(modname) sys.modules[modname].remove(mlist) removeables.append((os.path.join('lists', listname), _('list info'))) # Remove any stale locks associated with the list - for filename in os.listdir(mm_cfg.LOCK_DIR): + for filename in os.listdir(config.LOCK_DIR): fn_listname = filename.split('.')[0] if fn_listname == listname: - removeables.append((os.path.join(mm_cfg.LOCK_DIR, filename), + removeables.append((os.path.join(config.LOCK_DIR, filename), _('stale lock file'))) if opts.archives: @@ -116,7 +121,7 @@ def main(): ]) for dirtmpl, msg in removeables: - path = os.path.join(mm_cfg.VAR_PREFIX, dirtmpl) + path = os.path.join(config.VAR_PREFIX, dirtmpl) remove_it(listname, path, msg) diff --git a/Mailman/bin/update.py b/Mailman/bin/update.py new file mode 100644 index 000000000..ea74abfd6 --- /dev/null +++ b/Mailman/bin/update.py @@ -0,0 +1,767 @@ +# Copyright (C) 1998-2006 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +import os +import md5 +import sys +import time +import email +import errno +import shutil +import cPickle +import marshal +import optparse + +from Mailman import MailList +from Mailman import Message +from Mailman import Pending +from Mailman import Utils +from Mailman import Version +from Mailman.LockFile import TimeOutError +from Mailman.MemberAdaptor import BYBOUNCE, ENABLED +from Mailman.OldStyleMemberships import OldStyleMemberships +from Mailman.Queue.Switchboard import Switchboard +from Mailman.configuration import config +from Mailman.i18n import _ + +__i18n_templates__ = True + +FRESH = 0 +NOTFRESH = -1 + + + +def parseargs(): + parser = optparse.OptionParser(version=Version.MAILMAN_VERSION, + usage=_("""\ +Perform all necessary upgrades. + +%prog [options]""")) + parser.add_option('-f', '--force', + default=False, action='store_true', help=_("""\ +Force running the upgrade procedures. Normally, if the version number of the +installed Mailman matches the current version number (or a 'downgrade' is +detected), nothing will be done.""")) + parser.add_option('-C', '--config', + help=_('Alternative configuration file to use')) + opts, args = parser.parse_args() + if args: + parser.print_help() + print >> sys.stderr, _('Unexpected arguments') + sys.exit(1) + return parser, opts, args + + + +def calcversions(): + # Returns a tuple of (lastversion, thisversion). If the last version + # could not be determined, lastversion will be FRESH or NOTFRESH, + # depending on whether this installation appears to be fresh or not. The + # determining factor is whether there are files in the $var_prefix/logs + # subdir or not. The version numbers are HEX_VERSIONs. + # + # See if we stored the last updated version + lastversion = None + thisversion = config.HEX_VERSION + try: + fp = open(os.path.join(config.DATA_DIR, 'last_mailman_version')) + data = fp.read() + fp.close() + lastversion = int(data, 16) + except (IOError, ValueError): + pass + # + # try to figure out if this is a fresh install + if lastversion is None: + lastversion = FRESH + try: + if os.listdir(config.LOG_DIR): + lastversion = NOTFRESH + except OSError: + pass + return (lastversion, thisversion) + + + +def makeabs(relpath): + return os.path.join(config.PREFIX, relpath) + + +def make_varabs(relpath): + return os.path.join(config.VAR_PREFIX, relpath) + + + +def move_language_templates(mlist): + listname = mlist.internal_name() + print _('Fixing language templates: $listname') + # Mailman 2.1 has a new cascading search for its templates, defined and + # described in Utils.py:maketext(). Putting templates in the top level + # templates/ subdir or the lists/<listname> subdir is deprecated and no + # longer searched.. + # + # What this means is that most templates can live in the global templates/ + # subdirectory, and only needs to be copied into the list-, vhost-, or + # site-specific language directories when needed. + # + # Also, by default all standard (i.e. English) templates must now live in + # the templates/en directory. This update cleans up all the templates, + # deleting more-specific duplicates (as calculated by md5 checksums) in + # favor of more-global locations. + # + # First, get rid of any lists/<list> template or lists/<list>/en template + # that is identical to the global templates/* default. + for gtemplate in os.listdir(os.path.join(config.TEMPLATE_DIR, 'en')): + # BAW: get rid of old templates, e.g. admlogin.txt and + # handle_opts.html + try: + fp = open(os.path.join(config.TEMPLATE_DIR, gtemplate)) + except IOError, e: + if e.errno <> errno.ENOENT: + raise + # No global template + continue + gcksum = md5.new(fp.read()).digest() + fp.close() + # Match against the lists/<list>/* template + try: + fp = open(os.path.join(mlist.fullpath(), gtemplate)) + except IOError, e: + if e.errno <> errno.ENOENT: + raise + else: + tcksum = md5.new(fp.read()).digest() + fp.close() + if gcksum == tcksum: + os.unlink(os.path.join(mlist.fullpath(), gtemplate)) + # Match against the lists/<list>/*.prev template + try: + fp = open(os.path.join(mlist.fullpath(), gtemplate + '.prev')) + except IOError, e: + if e.errno <> errno.ENOENT: + raise + else: + tcksum = md5.new(fp.read()).digest() + fp.close() + if gcksum == tcksum: + os.unlink(os.path.join(mlist.fullpath(), gtemplate + '.prev')) + # Match against the lists/<list>/en/* templates + try: + fp = open(os.path.join(mlist.fullpath(), 'en', gtemplate)) + except IOError, e: + if e.errno <> errno.ENOENT: + raise + else: + tcksum = md5.new(fp.read()).digest() + fp.close() + if gcksum == tcksum: + os.unlink(os.path.join(mlist.fullpath(), 'en', gtemplate)) + # Match against the templates/* template + try: + fp = open(os.path.join(config.TEMPLATE_DIR, gtemplate)) + except IOError, e: + if e.errno <> errno.ENOENT: + raise + else: + tcksum = md5.new(fp.read()).digest() + fp.close() + if gcksum == tcksum: + os.unlink(os.path.join(config.TEMPLATE_DIR, gtemplate)) + # Match against the templates/*.prev template + try: + fp = open(os.path.join(config.TEMPLATE_DIR, gtemplate + '.prev')) + except IOError, e: + if e.errno <> errno.ENOENT: + raise + else: + tcksum = md5.new(fp.read()).digest() + fp.close() + if gcksum == tcksum: + os.unlink(os.path.join(config.TEMPLATE_DIR, + gtemplate + '.prev')) + + + +def dolist(listname): + mlist = MailList.MailList(listname, lock=False) + try: + mlist.Lock(0.5) + except TimeOutError: + print >> sys.stderr, _( + 'WARNING: could not acquire lock for list: $listname') + return 1 + # Sanity check the invariant that every BYBOUNCE disabled member must have + # bounce information. Some earlier betas broke this. BAW: we're + # submerging below the MemberAdaptor interface, so skip this if we're not + # using OldStyleMemberships. + if isinstance(mlist._memberadaptor, OldStyleMemberships): + noinfo = {} + for addr, (reason, when) in mlist.delivery_status.items(): + if reason == BYBOUNCE and not mlist.bounce_info.has_key(addr): + noinfo[addr] = reason, when + # What to do about these folks with a BYBOUNCE delivery status and no + # bounce info? This number should be very small, and I think it's + # fine to simple re-enable them and let the bounce machinery + # re-disable them if necessary. + n = len(noinfo) + if n > 0: + print _( + 'Resetting $n BYBOUNCEs disabled addrs with no bounce info') + for addr in noinfo.keys(): + mlist.setDeliveryStatus(addr, ENABLED) + + # Update the held requests database + print _("""Updating the held requests database.""") + mlist._UpdateRecords() + + mbox_dir = make_varabs('archives/private/%s.mbox' % (listname)) + mbox_file = make_varabs('archives/private/%s.mbox/%s' % (listname, + listname)) + o_pub_mbox_file = make_varabs('archives/public/%s' % (listname)) + o_pri_mbox_file = make_varabs('archives/private/%s' % (listname)) + html_dir = o_pri_mbox_file + o_html_dir = makeabs('public_html/archives/%s' % (listname)) + # Make the mbox directory if it's not there. + if not os.path.exists(mbox_dir): + ou = os.umask(0) + os.mkdir(mbox_dir, 02775) + os.umask(ou) + else: + # This shouldn't happen, but hey, just in case + if not os.path.isdir(mbox_dir): + print _("""\ +For some reason, $mbox_dir exists as a file. This won't work with b6, so I'm +renaming it to ${mbox_dir}.tmp and proceeding.""") + os.rename(mbox_dir, "%s.tmp" % (mbox_dir)) + ou = os.umask(0) + os.mkdir(mbox_dir, 02775) + os.umask(ou) + # Move any existing mboxes around, but watch out for both a public and a + # private one existing + if os.path.isfile(o_pri_mbox_file) and os.path.isfile(o_pub_mbox_file): + if mlist.archive_private: + print _("""\ + +$listname has both public and private mbox archives. Since this list +currently uses private archiving, I'm installing the private mbox archive -- +$o_pri_mbox_file -- as the active archive, and renaming + $o_pub_mbox_file +to + ${o_pub_mbox_file}.preb6 + +You can integrate that into the archives if you want by using the 'arch' +script. +""") % (mlist._internal_name, o_pri_mbox_file, o_pub_mbox_file, + o_pub_mbox_file) + os.rename(o_pub_mbox_file, "%s.preb6" % (o_pub_mbox_file)) + else: + print _("""\ +$mlist._internal_name has both public and private mbox archives. Since this +list currently uses public archiving, I'm installing the public mbox file +archive file ($o_pub_mbox_file) as the active one, and renaming +$o_pri_mbox_file to ${o_pri_mbox_file}.preb6 + +You can integrate that into the archives if you want by using the 'arch' +script. +""") + os.rename(o_pri_mbox_file, "%s.preb6" % (o_pri_mbox_file)) + # Move private archive mbox there if it's around + # and take into account all sorts of absurdities + print _('- updating old private mbox file') + if os.path.exists(o_pri_mbox_file): + if os.path.isfile(o_pri_mbox_file): + os.rename(o_pri_mbox_file, mbox_file) + elif not os.path.isdir(o_pri_mbox_file): + newname = "%s.mm_install-dunno_what_this_was_but_its_in_the_way" \ + % o_pri_mbox_file + os.rename(o_pri_mbox_file, newname) + print _("""\ + unknown file in the way, moving + $o_pri_mbox_file + to + $newname""") + else: + # directory + print _("""\ + looks like you have a really recent CVS installation... + you're either one brave soul, or you already ran me""") + # Move public archive mbox there if it's around + # and take into account all sorts of absurdities. + print _('- updating old public mbox file') + if os.path.exists(o_pub_mbox_file): + if os.path.isfile(o_pub_mbox_file): + os.rename(o_pub_mbox_file, mbox_file) + elif not os.path.isdir(o_pub_mbox_file): + newname = "%s.mm_install-dunno_what_this_was_but_its_in_the_way" \ + % o_pub_mbox_file + os.rename(o_pub_mbox_file, newname) + print _("""\ + unknown file in the way, moving + $o_pub_mbox_file + to + $newname""") + else: # directory + print _("""\ + looks like you have a really recent CVS installation... + you're either one brave soul, or you already ran me""") + # Move the html archives there + if os.path.isdir(o_html_dir): + os.rename(o_html_dir, html_dir) + # chmod the html archives + os.chmod(html_dir, 02775) + # BAW: Is this still necessary?! + mlist.Save() + # Check to see if pre-b4 list-specific templates are around + # and move them to the new place if there's not already + # a new one there + tmpl_dir = os.path.join(config.PREFIX, "templates") + list_dir = os.path.join(config.PREFIX, "lists") + b4_tmpl_dir = os.path.join(tmpl_dir, mlist._internal_name) + new_tmpl_dir = os.path.join(list_dir, mlist._internal_name) + if os.path.exists(b4_tmpl_dir): + print _("""\ +- This list looks like it might have <= b4 list templates around""") + for f in os.listdir(b4_tmpl_dir): + o_tmpl = os.path.join(b4_tmpl_dir, f) + n_tmpl = os.path.join(new_tmpl_dir, f) + if os.path.exists(o_tmpl): + if not os.path.exists(n_tmpl): + os.rename(o_tmpl, n_tmpl) + print _('- moved $o_tmpl to $n_tmpl') + else: + print _("""\ +- both $o_tmpl and $n_tmpl exist, leaving untouched""") + else: + print _("""\ +- $o_tmpl doesn't exist, leaving untouched""") + # Move all the templates to the en language subdirectory as required for + # Mailman 2.1 + move_language_templates(mlist) + # Avoid eating filehandles with the list lockfiles + mlist.Unlock() + return 0 + + + +def archive_path_fixer(unused_arg, dir, files): + # Passed to os.path.walk to fix the perms on old html archives. + for f in files: + abs = os.path.join(dir, f) + if os.path.isdir(abs): + if f == "database": + os.chmod(abs, 02770) + else: + os.chmod(abs, 02775) + elif os.path.isfile(abs): + os.chmod(abs, 0664) + + +def remove_old_sources(module): + # Also removes old directories. + src = '%s/%s' % (config.PREFIX, module) + pyc = src + "c" + if os.path.isdir(src): + print _('removing directory $src and everything underneath') + shutil.rmtree(src) + elif os.path.exists(src): + print _('removing $src') + try: + os.unlink(src) + except os.error, rest: + print _("Warning: couldn't remove $src -- $rest") + if module.endswith('.py') and os.path.exists(pyc): + try: + os.unlink(pyc) + except OSError, rest: + print _("couldn't remove old file $pyc -- $rest") + + + +def update_qfiles(): + print _('updating old qfiles') + prefix = `time.time()` + '+' + # Be sure the qfiles/in directory exists (we don't really need the + # switchboard object, but it's convenient for creating the directory). + sb = Switchboard(config.INQUEUE_DIR) + for filename in os.listdir(config.QUEUE_DIR): + # Updating means just moving the .db and .msg files to qfiles/in where + # it should be dequeued, converted, and processed normally. + if os.path.splitext(filename) == '.msg': + oldmsgfile = os.path.join(config.QUEUE_DIR, filename) + newmsgfile = os.path.join(config.INQUEUE_DIR, prefix + filename) + os.rename(oldmsgfile, newmsgfile) + elif os.path.splitext(filename) == '.db': + olddbfile = os.path.join(config.QUEUE_DIR, filename) + newdbfile = os.path.join(config.INQUEUE_DIR, prefix + filename) + os.rename(olddbfile, newdbfile) + # Now update for the Mailman 2.1.5 qfile format. For every filebase in + # the qfiles/* directories that has both a .pck and a .db file, pull the + # data out and re-queue them. + for dirname in os.listdir(config.QUEUE_DIR): + dirpath = os.path.join(config.QUEUE_DIR, dirname) + if dirpath == config.BADQUEUE_DIR: + # The files in qfiles/bad can't possibly be pickles + continue + sb = Switchboard(dirpath) + try: + for filename in os.listdir(dirpath): + filepath = os.path.join(dirpath, filename) + filebase, ext = os.path.splitext(filepath) + # Handle the .db metadata files as part of the handling of the + # .pck or .msg message files. + if ext not in ('.pck', '.msg'): + continue + msg, data = dequeue(filebase) + if msg is not None and data is not None: + sb.enqueue(msg, data) + except EnvironmentError, e: + if e.errno <> errno.ENOTDIR: + raise + print _('Warning! Not a directory: $dirpath') + + + +# Implementations taken from the pre-2.1.5 Switchboard +def ext_read(filename): + fp = open(filename) + d = marshal.load(fp) + # Update from version 2 files + if d.get('version', 0) == 2: + del d['filebase'] + # Do the reverse conversion (repr -> float) + for attr in ['received_time']: + try: + sval = d[attr] + except KeyError: + pass + else: + # Do a safe eval by setting up a restricted execution + # environment. This may not be strictly necessary since we + # know they are floats, but it can't hurt. + d[attr] = eval(sval, {'__builtins__': {}}) + fp.close() + return d + + +def dequeue(filebase): + # Calculate the .db and .msg filenames from the given filebase. + msgfile = os.path.join(filebase + '.msg') + pckfile = os.path.join(filebase + '.pck') + dbfile = os.path.join(filebase + '.db') + # Now we are going to read the message and metadata for the given + # filebase. We want to read things in this order: first, the metadata + # file to find out whether the message is stored as a pickle or as + # plain text. Second, the actual message file. However, we want to + # first unlink the message file and then the .db file, because the + # qrunner only cues off of the .db file + msg = None + try: + data = ext_read(dbfile) + os.unlink(dbfile) + except EnvironmentError, e: + if e.errno <> errno.ENOENT: + raise + data = {} + # Between 2.1b4 and 2.1b5, the `rejection-notice' key in the metadata + # was renamed to `rejection_notice', since dashes in the keys are not + # supported in METAFMT_ASCII. + if data.has_key('rejection-notice'): + data['rejection_notice'] = data['rejection-notice'] + del data['rejection-notice'] + msgfp = None + try: + try: + msgfp = open(pckfile) + msg = cPickle.load(msgfp) + os.unlink(pckfile) + except EnvironmentError, e: + if e.errno <> errno.ENOENT: raise + msgfp = None + try: + msgfp = open(msgfile) + msg = email.message_from_file(msgfp, Message.Message) + os.unlink(msgfile) + except EnvironmentError, e: + if e.errno <> errno.ENOENT: raise + except (email.Errors.MessageParseError, ValueError), e: + # This message was unparsable, most likely because its + # MIME encapsulation was broken. For now, there's not + # much we can do about it. + print _('message is unparsable: $filebase') + msgfp.close() + msgfp = None + if config.QRUNNER_SAVE_BAD_MESSAGES: + # Cheapo way to ensure the directory exists w/ the + # proper permissions. + sb = Switchboard(config.BADQUEUE_DIR) + os.rename(msgfile, os.path.join( + config.BADQUEUE_DIR, filebase + '.txt')) + else: + os.unlink(msgfile) + msg = data = None + except EOFError: + # For some reason the pckfile was empty. Just delete it. + print _('Warning! Deleting empty .pck file: $pckfile') + os.unlink(pckfile) + finally: + if msgfp: + msgfp.close() + return msg, data + + + +def update_pending(): + file20 = os.path.join(config.DATA_DIR, 'pending_subscriptions.db') + file214 = os.path.join(config.DATA_DIR, 'pending.pck') + db = None + # Try to load the Mailman 2.0 file + try: + fp = open(file20) + except IOError, e: + if e.errno <> errno.ENOENT: + raise + else: + print _('Updating Mailman 2.0 pending_subscriptions.db database') + db = marshal.load(fp) + # Convert to the pre-Mailman 2.1.5 format + db = Pending._update(db) + if db is None: + # Try to load the Mailman 2.1.x where x < 5, file + try: + fp = open(file214) + except IOError, e: + if e.errno <> errno.ENOENT: + raise + else: + print _('Updating Mailman 2.1.4 pending.pck database') + db = cPickle.load(fp) + if db is None: + print _('Nothing to do.') + return + # Now upgrade the database to the 2.1.5 format. Each list now has its own + # pending.pck file, but only the RE_ENABLE operation actually recorded the + # listname in the request. For the SUBSCRIPTION, UNSUBSCRIPTION, and + # CHANGE_OF_ADDRESS operations, we know the address of the person making + # the request so we can repend this request just for the lists the person + # is a member of. For the HELD_MESSAGE operation, we can check the list's + # requests.pck file for correlation. Evictions will take care of any + # misdirected pendings. + reenables_by_list = {} + addrops_by_address = {} + holds_by_id = {} + subs_by_address = {} + for key, val in db.items(): + if key in ('evictions', 'version'): + continue + try: + op = val[0] + data = val[1:] + except (IndexError, ValueError): + print _('Ignoring bad pended data: $key: $val') + continue + if op in (Pending.UNSUBSCRIPTION, Pending.CHANGE_OF_ADDRESS): + # data[0] is the address being unsubscribed + addrops_by_address.setdefault(data[0], []).append((key, val)) + elif op == Pending.SUBSCRIPTION: + # data[0] is a UserDesc object + addr = data[0].address + subs_by_address.setdefault(addr, []).append((key, val)) + elif op == Pending.RE_ENABLE: + # data[0] is the mailing list's internal name + reenables_by_list.setdefault(data[0], []).append((key, val)) + elif op == Pending.HELD_MESSAGE: + # data[0] is the hold id. There better only be one entry per id + id = data[0] + if holds_by_id.has_key(id): + print _('WARNING: Ignoring duplicate pending ID: $id.') + else: + holds_by_id[id] = (key, val) + # Now we have to lock every list and re-pend all the appropriate + # requests. Note that this will reset all the expiration dates, but that + # should be fine. + for listname in Utils.list_names(): + mlist = MailList.MailList(listname) + # This is not the most efficient way to do this because it loads and + # saves the pending.pck file each time. :( + try: + for cookie, data in reenables_by_list.get(listname, []): + mlist.pend_repend(cookie, data) + for id, (cookie, data) in holds_by_id.items(): + try: + rec = mlist.GetRecord(id) + except KeyError: + # Not for this list + pass + else: + mlist.pend_repend(cookie, data) + del holds_by_id[id] + for addr, recs in subs_by_address.items(): + # We shouldn't have a subscription confirmation if the address + # is already a member of the mailing list. + if mlist.isMember(addr): + continue + for cookie, data in recs: + mlist.pend_repend(cookie, data) + for addr, recs in addrops_by_address.items(): + # We shouldn't have unsubscriptions or change of address + # requests for addresses which aren't members of the list. + if not mlist.isMember(addr): + continue + for cookie, data in recs: + mlist.pend_repend(cookie, data) + mlist.Save() + finally: + mlist.Unlock() + try: + os.unlink(file20) + except OSError, e: + if e.errno <> errno.ENOENT: + raise + try: + os.unlink(file214) + except OSError, e: + if e.errno <> errno.ENOENT: + raise + + + +def main(): + parser, opts, args = parseargs() + config.load(opts.config) + + # calculate the versions + lastversion, thisversion = calcversions() + hexlversion = hex(lastversion) + hextversion = hex(thisversion) + if lastversion == thisversion and not opts.force: + # nothing to do + print _('No updates are necessary.') + sys.exit(0) + if lastversion > thisversion and not opts.force: + print _("""\ +Downgrade detected, from version $hexlversion to version $hextversion +This is probably not safe. +Exiting.""") + sys.exit(1) + print _('Upgrading from version $hexlversion to $hextversion') + errors = 0 + # get rid of old stuff + print _('getting rid of old source files') + for mod in ('Mailman/Archiver.py', 'Mailman/HyperArch.py', + 'Mailman/HyperDatabase.py', 'Mailman/pipermail.py', + 'Mailman/smtplib.py', 'Mailman/Cookie.py', + 'bin/update_to_10b6', 'scripts/mailcmd', + 'scripts/mailowner', 'mail/wrapper', 'Mailman/pythonlib', + 'cgi-bin/archives', 'Mailman/MailCommandHandler'): + remove_old_sources(mod) + listnames = Utils.list_names() + if not listnames: + print _('no lists == nothing to do, exiting') + return + # + # for people with web archiving, make sure the directories + # in the archiving are set with proper perms for b6. + # + if os.path.isdir("%s/public_html/archives" % config.PREFIX): + print _("""\ +fixing all the perms on your old html archives to work with b6 +If your archives are big, this could take a minute or two...""") + os.path.walk("%s/public_html/archives" % config.PREFIX, + archive_path_fixer, "") + print _('done') + for listname in listnames: + print _('Updating mailing list: $listname') + errors = errors + dolist(listname) + print + print _('Updating Usenet watermarks') + wmfile = os.path.join(config.DATA_DIR, 'gate_watermarks') + try: + fp = open(wmfile) + except IOError: + print _('- nothing to update here') + else: + d = marshal.load(fp) + fp.close() + for listname in d.keys(): + if listname not in listnames: + # this list no longer exists + continue + mlist = MailList.MailList(listname, lock=0) + try: + mlist.Lock(0.5) + except TimeOutError: + print >> sys.stderr, _( + 'WARNING: could not acquire lock for list: $listname') + errors = errors + 1 + else: + # Pre 1.0b7 stored 0 in the gate_watermarks file to indicate + # that no gating had been done yet. Without coercing this to + # None, the list could now suddenly get flooded. + mlist.usenet_watermark = d[listname] or None + mlist.Save() + mlist.Unlock() + os.unlink(wmfile) + print _('- usenet watermarks updated and gate_watermarks removed') + # In Mailman 2.1, the pending database format and file name changed, but + # in Mailman 2.1.5 it changed again. This should update all existing + # files to the 2.1.5 format. + update_pending() + # In Mailman 2.1, the qfiles directory has a different structure and a + # different content. Also, in Mailman 2.1.5 we collapsed the message + # files from separate .msg (pickled Message objects) and .db (marshalled + # dictionaries) to a shared .pck file containing two pickles. + update_qfiles() + # This warning was necessary for the upgrade from 1.0b9 to 1.0b10. + # There's no good way of figuring this out for releases prior to 2.0beta2 + # :( + if lastversion == NOTFRESH: + print _(""" + +NOTE NOTE NOTE NOTE NOTE + + You are upgrading an existing Mailman installation, but I can't tell what + version you were previously running. + + If you are upgrading from Mailman 1.0b9 or earlier you will need to + manually update your mailing lists. For each mailing list you need to + copy the file templates/options.html lists/<listname>/options.html. + + However, if you have edited this file via the Web interface, you will have + to merge your changes into this file, otherwise you will lose your + changes. + +NOTE NOTE NOTE NOTE NOTE + +""") + if not errors: + # Record the version we just upgraded to + fp = open(os.path.join(config.DATA_DIR, 'last_mailman_version'), 'w') + fp.write(hex(config.HEX_VERSION) + '\n') + fp.close() + else: + lockdir = config.LOCK_DIR + print _('''\ + +ERROR: + +The locks for some lists could not be acquired. This means that either +Mailman was still active when you upgraded, or there were stale locks in the +$lockdir directory. + +You must put Mailman into a quiescent state and remove all stale locks, then +re-run "make update" manually. See the INSTALL and UPGRADE files for details. +''') diff --git a/Mailman/configuration.py b/Mailman/configuration.py new file mode 100644 index 000000000..cf3af03f3 --- /dev/null +++ b/Mailman/configuration.py @@ -0,0 +1,103 @@ +# Copyright (C) 2006 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +"""Configuration file loading and management.""" + +import os +import errno + +from Mailman import Defaults + +_missing = object() + + + +class Configuration(object): + def load(self, filename=None): + # Load the configuration from the named file, or if not given, search + # in VAR_PREFIX for an etc/mailman.cfg file. If that file is missing, + # use Mailman/mm_cfg.py for backward compatibility. + # + # Whatever you find, create a namespace and execfile that file in it. + # The values in that namespace are exposed as attributes on this + # Configuration instance. + if filename is None: + filename = os.path.join(Defaults.VAR_PREFIX, 'etc', 'mailman.cfg') + # Set up the execfile namespace + ns = Defaults.__dict__.copy() + # Prune a few things + del ns['__file__'] + del ns['__name__'] + del ns['__doc__'] + # Attempt our first choice + path = os.path.abspath(os.path.expanduser(filename)) + try: + execfile(path, ns, ns) + except EnvironmentError, e: + if e.errno <> errno.ENOENT: + raise + # The file didn't exist, so try mm_cfg.py + from Mailman import mm_cfg + ns = mm_cfg.__dict__.copy() + # Pull out the defaults + PREFIX = ns['PREFIX'] + VAR_PREFIX = ns['VAR_PREFIX'] + EXEC_PREFIX = ns['EXEC_PREFIX'] + # Now that we've loaded all the configuration files we're going to + # load, set up some useful directories. + self.LIST_DATA_DIR = os.path.join(VAR_PREFIX, 'lists') + self.LOG_DIR = os.path.join(VAR_PREFIX, 'logs') + self.LOCK_DIR = lockdir = os.path.join(VAR_PREFIX, 'locks') + self.DATA_DIR = datadir = os.path.join(VAR_PREFIX, 'data') + self.ETC_DIR = etcdir = os.path.join(VAR_PREFIX, 'etc') + self.SPAM_DIR = os.path.join(VAR_PREFIX, 'spam') + self.WRAPPER_DIR = os.path.join(EXEC_PREFIX, 'mail') + self.BIN_DIR = os.path.join(PREFIX, 'bin') + self.SCRIPTS_DIR = os.path.join(PREFIX, 'scripts') + self.TEMPLATE_DIR = os.path.join(PREFIX, 'templates') + self.MESSAGES_DIR = os.path.join(PREFIX, 'messages') + self.PUBLIC_ARCHIVE_FILE_DIR = os.path.join(VAR_PREFIX, + 'archives', 'public') + self.PRIVATE_ARCHIVE_FILE_DIR = os.path.join(VAR_PREFIX, + 'archives', 'private') + # Directories used by the qrunner subsystem + self.QUEUE_DIR = qdir = os.path.join(VAR_PREFIX, 'qfiles') + self.INQUEUE_DIR = os.path.join(qdir, 'in') + self.OUTQUEUE_DIR = os.path.join(qdir, 'out') + self.CMDQUEUE_DIR = os.path.join(qdir, 'commands') + self.BOUNCEQUEUE_DIR = os.path.join(qdir, 'bounces') + self.NEWSQUEUE_DIR = os.path.join(qdir, 'news') + self.ARCHQUEUE_DIR = os.path.join(qdir, 'archive') + self.SHUNTQUEUE_DIR = os.path.join(qdir, 'shunt') + self.VIRGINQUEUE_DIR = os.path.join(qdir, 'virgin') + self.BADQUEUE_DIR = os.path.join(qdir, 'bad') + self.RETRYQUEUE_DIR = os.path.join(qdir, 'retry') + self.MAILDIR_DIR = os.path.join(qdir, 'maildir') + # Other useful files + self.PIDFILE = os.path.join(datadir, + 'master-qrunner.pid') + self.SITE_PW_FILE = os.path.join(datadir, 'adm.pw') + self.LISTCREATOR_PW_FILE = os.path.join(datadir, 'creator.pw') + self.CONFIG_FILE = os.path.join(etcdir, 'mailman.cfg') + self.LOCK_FILE = os.path.join(lockdir, 'master-qrunner') + # Now update our dict so attribute syntax just works + self.__dict__.update(ns) + + + +config = Configuration() + diff --git a/Mailman/i18n.py b/Mailman/i18n.py index 7d36e22d3..69f9801d1 100644 --- a/Mailman/i18n.py +++ b/Mailman/i18n.py @@ -20,8 +20,8 @@ import time import string import gettext -from Mailman import mm_cfg from Mailman.SafeDict import SafeDict +from Mailman.configuration import config _translation = None _missing = object() @@ -47,7 +47,7 @@ def set_language(language=None): if language is not None: language = [language] try: - _translation = gettext.translation('mailman', mm_cfg.MESSAGES_DIR, + _translation = gettext.translation('mailman', config.MESSAGES_DIR, language) except IOError: # The selected language was not installed in messages, so fall back to @@ -67,7 +67,7 @@ def set_translation(translation): # Set up the global translation based on environment variables. Mostly used # for command line scripts. if _translation is None: - set_language() + _translation = gettext.NullTranslations() diff --git a/Mailman/loginit.py b/Mailman/loginit.py index bcb016f08..fac01c0fb 100644 --- a/Mailman/loginit.py +++ b/Mailman/loginit.py @@ -24,7 +24,7 @@ import os import codecs import logging -from Mailman import mm_cfg +from Mailman.configuration import config FMT = '%(asctime)s (%(process)d) %(message)s' DATEFMT = '%b %d %H:%M:%S %Y' @@ -116,7 +116,7 @@ def initialize(propagate=False): # Propagation to the root logger is how we handle logging to stderr # when the qrunners are not run as a subprocess of mailmanctl. log.propagate = propagate - handler = ReopenableFileHandler(os.path.join(mm_cfg.LOG_DIR, logger)) + handler = ReopenableFileHandler(os.path.join(config.LOG_DIR, logger)) _handlers.append(handler) handler.setFormatter(formatter) log.addHandler(handler) diff --git a/Mailman/testing/emailbase.py b/Mailman/testing/emailbase.py index 115ea6839..6c290c8e5 100644 --- a/Mailman/testing/emailbase.py +++ b/Mailman/testing/emailbase.py @@ -17,14 +17,18 @@ """Base class for tests that email things.""" +import os import smtpd import socket import asyncore +import tempfile import subprocess from Mailman import mm_cfg from Mailman.testing.base import TestBase +TESTPORT = 10825 + MSGTEXT = None @@ -49,15 +53,23 @@ class SinkServer(smtpd.SMTPServer): class EmailBase(TestBase): def setUp(self): TestBase.setUp(self) - # Find an unused non-root requiring port to listen on - oldport = mm_cfg.SMTPPORT - mm_cfg.SMTPPORT = port = 10825 + # Find an unused non-root requiring port to listen on. Set up a + # configuration file that causes the underlying outgoing runner to use + # the same port, then start Mailman. + fd, self._configfile = tempfile.mkstemp(suffix='.cfg') + print 'config file:', self._configfile + fp = os.fdopen(fd, 'w') + print >> fp, 'SMTPPORT =', TESTPORT + fp.close() # Second argument is ignored. - self._server = SinkServer(('localhost', port), None) + self._server = SinkServer(('localhost', TESTPORT), None) + os.system('bin/mailmanctl start') def tearDown(self): + os.system('bin/mailmanctl stop') self._server.close() TestBase.tearDown(self) + os.remove(self._configfile) def _readmsg(self): global MSGTEXT diff --git a/Makefile.in b/Makefile.in index aa49d9310..9c5fe7807 100644 --- a/Makefile.in +++ b/Makefile.in @@ -48,7 +48,7 @@ VAR_DIRS= \ archives/private archives/public ARCH_INDEP_DIRS= \ - bin templates scripts cron pythonlib \ + bin etc templates scripts cron pythonlib \ Mailman Mailman/bin Mailman/Cgi Mailman/Archiver \ Mailman/Handlers Mailman/Queue Mailman/Bouncers \ Mailman/MTA Mailman/Gui Mailman/Commands messages icons \ diff --git a/bin/Makefile.in b/bin/Makefile.in index 40fba3ed4..37084d1f7 100644 --- a/bin/Makefile.in +++ b/bin/Makefile.in @@ -45,17 +45,18 @@ SCRIPTSDIR= $(prefix)/bin SHELL= /bin/sh SCRIPTS= mmshell \ - remove_members clone_member update \ + remove_members clone_member \ sync_members check_db withlist check_perms \ config_list dumpdb cleanarch \ - list_admins genaliases mailmanctl qrunner \ + list_admins genaliases \ fix_url.py convert.py transcheck \ msgfmt.py discard \ reset_pw.py templ2pot.py po2templ.py LN_SCRIPTS= add_members arch change_pw find_member inject list_lists \ - list_members list_owners mmsitepass newlist rmlist \ - show_qfiles show_mm_cfg testall unshunt version + list_members list_owners mailmanctl mmsitepass newlist \ + qrunner rmlist show_qfiles show_mm_cfg testall unshunt \ + update version BUILDDIR= ../build/bin diff --git a/bin/mailmanctl b/bin/mailmanctl index 36c3ab545..e69de29bb 100644 --- a/bin/mailmanctl +++ b/bin/mailmanctl @@ -1,545 +0,0 @@ -#! @PYTHON@ - -# Copyright (C) 2001-2006 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""Primary start-up and shutdown script for Mailman's qrunner daemon. - -This script starts, stops, and restarts the main Mailman queue runners, making -sure that the various long-running qrunners are still alive and kicking. It -does this by forking and exec'ing the qrunners and waiting on their pids. -When it detects a subprocess has exited, it may restart it. - -The qrunners respond to SIGINT, SIGTERM, and SIGHUP. SIGINT and SIGTERM both -cause the qrunners to exit cleanly, but the master will only restart qrunners -that have exited due to a SIGINT. SIGHUP causes the master and the qrunners -to close their log files, and reopen then upon the next printed message. - -The master also responds to SIGINT, SIGTERM, and SIGHUP, which it simply -passes on to the qrunners (note that the master will close and reopen its own -log files on receipt of a SIGHUP). The master also leaves its own process id -in the file data/master-qrunner.pid but you normally don't need to use this -pid directly. The `start', `stop', `restart', and `reopen' commands handle -everything for you. - -Usage: %(PROGRAM)s [options] [ start | stop | restart | reopen ] - -Options: - - -n/--no-restart - Don't restart the qrunners when they exit because of an error or a - SIGINT. They are never restarted if they exit in response to a - SIGTERM. Use this only for debugging. Only useful if the `start' - command is given. - - -u/--run-as-user - Normally, this script will refuse to run if the user id and group id - are not set to the `mailman' user and group (as defined when you - configured Mailman). If run as root, this script will change to this - user and group before the check is made. - - This can be inconvenient for testing and debugging purposes, so the -u - flag means that the step that sets and checks the uid/gid is skipped, - and the program is run as the current user and group. This flag is - not recommended for normal production environments. - - Note though, that if you run with -u and are not in the mailman group, - you may have permission problems, such as begin unable to delete a - list's archives through the web. Tough luck! - - -s/--stale-lock-cleanup - If mailmanctl finds an existing master lock, it will normally exit - with an error message. With this option, mailmanctl will perform an - extra level of checking. If a process matching the host/pid described - in the lock file is running, mailmanctl will still exit, but if no - matching process is found, mailmanctl will remove the apparently stale - lock and make another attempt to claim the master lock. - - -q/--quiet - Don't print status messages. Error messages are still printed to - standard error. - - -h/--help - Print this message and exit. - -Commands: - - start - Start the master daemon and all qrunners. Prints a message and - exits if the master daemon is already running. - - stop - Stops the master daemon and all qrunners. After stopping, no - more messages will be processed. - - restart - Restarts the qrunners, but not the master process. Use this - whenever you upgrade or update Mailman so that the qrunners will - use the newly installed code. - - reopen - This will close all log files, causing them to be re-opened the - next time a message is written to them -""" - -import os -import grp -import pwd -import sys -import errno -import getopt -import signal -import socket -import logging - -import paths -from Mailman import Errors -from Mailman import LockFile -from Mailman import Utils -from Mailman import loginit -from Mailman import mm_cfg -from Mailman.MailList import MailList -from Mailman.i18n import _ - -PROGRAM = sys.argv[0] -COMMASPACE = ', ' -DOT = '.' - -# Locking contantsa -LOCKFILE = os.path.join(mm_cfg.LOCK_DIR, 'master-qrunner') -# Since we wake up once per day and refresh the lock, the LOCK_LIFETIME -# needn't be (much) longer than SNOOZE. We pad it 6 hours just to be safe. -LOCK_LIFETIME = mm_cfg.days(1) + mm_cfg.hours(6) -SNOOZE = mm_cfg.days(1) -MAX_RESTARTS = 10 - -loginit.initialize() -elog = logging.getLogger('mailman.error') -qlog = logging.getLogger('mailman.qrunner') - - - -def usage(code, msg=''): - if code: - fd = sys.stderr - else: - fd = sys.stdout - print >> fd, _(__doc__) - if msg: - print >> fd, msg - sys.exit(code) - - - -def kill_watcher(sig): - try: - fp = open(mm_cfg.PIDFILE) - pidstr = fp.read() - fp.close() - pid = int(pidstr.strip()) - except (IOError, ValueError), e: - # For i18n convenience - pidfile = mm_cfg.PIDFILE - print >> sys.stderr, _('PID unreadable in: %(pidfile)s') - print >> sys.stderr, e - print >> sys.stderr, _('Is qrunner even running?') - return - try: - os.kill(pid, sig) - except OSError, e: - if e.errno <> errno.ESRCH: raise - print >> sys.stderr, _('No child with pid: %(pid)s') - print >> sys.stderr, e - print >> sys.stderr, _('Stale pid file removed.') - os.unlink(mm_cfg.PIDFILE) - - - -def get_lock_data(): - # Return the hostname, pid, and tempfile - fp = open(LOCKFILE) - try: - filename = os.path.split(fp.read().strip())[1] - finally: - fp.close() - parts = filename.split('.') - hostname = DOT.join(parts[1:-1]) - pid = int(parts[-1]) - return hostname, int(pid), filename - - -def qrunner_state(): - # 1 if proc exists on host (but is it qrunner? ;) - # 0 if host matches but no proc - # hostname if hostname doesn't match - hostname, pid, tempfile = get_lock_data() - if hostname <> socket.gethostname(): - return hostname - # Find out if the process exists by calling kill with a signal 0. - try: - os.kill(pid, 0) - except OSError, e: - if e.errno <> errno.ESRCH: - raise - return 0 - return 1 - - -def acquire_lock_1(force): - # Be sure we can acquire the master qrunner lock. If not, it means some - # other master qrunner daemon is already going. - lock = LockFile.LockFile(LOCKFILE, LOCK_LIFETIME) - try: - lock.lock(0.1) - return lock - except LockFile.TimeOutError: - if not force: - raise - # Force removal of lock first - lock._disown() - hostname, pid, tempfile = get_lock_data() - os.unlink(LOCKFILE) - os.unlink(os.path.join(mm_cfg.LOCK_DIR, tempfile)) - return acquire_lock_1(force=0) - - -def acquire_lock(force): - try: - lock = acquire_lock_1(force) - return lock - except LockFile.TimeOutError: - status = qrunner_state() - if status == 1: - # host matches and proc exists - print >> sys.stderr, _("""\ -The master qrunner lock could not be acquired because it appears as if another -master qrunner is already running. -""") - elif status == 0: - # host matches but no proc - print >> sys.stderr, _("""\ -The master qrunner lock could not be acquired. It appears as though there is -a stale master qrunner lock. Try re-running mailmanctl with the -s flag. -""") - else: - # host doesn't even match - print >> sys.stderr, _("""\ -The master qrunner lock could not be acquired, because it appears as if some -process on some other host may have acquired it. We can't test for stale -locks across host boundaries, so you'll have to do this manually. Or, if you -know the lock is stale, re-run mailmanctl with the -s flag. - -Lock file: %(LOCKFILE)s -Lock host: %(status)s - -Exiting.""") - - - -def start_runner(qrname, slice, count): - pid = os.fork() - if pid: - # parent - return pid - # child - # - # Craft the command line arguments for the exec() call. - rswitch = '--runner=%s:%d:%d' % (qrname, slice, count) - exe = os.path.join(mm_cfg.BIN_DIR, 'qrunner') - # mm_cfg.PYTHON, which is the absolute path to the Python interpreter, - # must be given as argv[0] due to Python's library search algorithm. - os.execl(mm_cfg.PYTHON, mm_cfg.PYTHON, exe, rswitch, '-s') - # Should never get here - raise RuntimeError, 'os.execl() failed' - - -def start_all_runners(): - kids = {} - for qrname, count in mm_cfg.QRUNNERS: - for slice in range(count): - # queue runner name, slice, numslices, restart count - info = (qrname, slice, count, 0) - pid = start_runner(qrname, slice, count) - kids[pid] = info - return kids - - - -def check_for_site_list(): - sitelistname = mm_cfg.MAILMAN_SITE_LIST - try: - sitelist = MailList(sitelistname, lock=False) - except Errors.MMUnknownListError: - print >> sys.stderr, _('Site list is missing: %(sitelistname)s') - elog.error('Site list is missing: %s', mm_cfg.MAILMAN_SITE_LIST) - sys.exit(1) - - -def check_privs(): - # If we're running as root (uid == 0), coerce the uid and gid to that - # which Mailman was configured for, and refuse to run if we didn't coerce - # the uid/gid. - gid = grp.getgrnam(mm_cfg.MAILMAN_GROUP)[2] - uid = pwd.getpwnam(mm_cfg.MAILMAN_USER)[2] - myuid = os.getuid() - if myuid == 0: - # Set the process's supplimental groups. - groups = [x[2] for x in grp.getgrall() if mm_cfg.MAILMAN_USER in x[3]] - groups.append(gid) - os.setgroups(groups) - os.setgid(gid) - os.setuid(uid) - elif myuid <> uid: - name = mm_cfg.MAILMAN_USER - usage(1, _( - 'Run this program as root or as the %(name)s user, or use -u.')) - - - -def main(): - global quiet - try: - opts, args = getopt.getopt(sys.argv[1:], 'hnusq', - ['help', 'no-start', 'run-as-user', - 'stale-lock-cleanup', 'quiet']) - except getopt.error, msg: - usage(1, msg) - - restart = 1 - checkprivs = 1 - force = 0 - quiet = 0 - for opt, arg in opts: - if opt in ('-h', '--help'): - usage(0) - elif opt in ('-n', '--no-restart'): - restart = 0 - elif opt in ('-u', '--run-as-user'): - checkprivs = 0 - elif opt in ('-s', '--stale-lock-cleanup'): - force = 1 - elif opt in ('-q', '--quiet'): - quiet = 1 - - if len(args) < 1: - usage(1, _('No command given.')) - elif len(args) > 1: - command = COMMASPACE.join(args) - usage(1, _('Bad command: %(command)s')) - - if checkprivs: - check_privs() - else: - print _('Warning! You may encounter permission problems.') - - # Handle the commands - command = args[0].lower() - if command == 'stop': - # Sent the master qrunner process a SIGINT, which is equivalent to - # giving cron/qrunner a ctrl-c or KeyboardInterrupt. This will - # effectively shut everything down. - if not quiet: - print _("Shutting down Mailman's master qrunner") - kill_watcher(signal.SIGTERM) - elif command == 'restart': - # Sent the master qrunner process a SIGHUP. This will cause the - # master qrunner to kill and restart all the worker qrunners, and to - # close and re-open its log files. - if not quiet: - print _("Restarting Mailman's master qrunner") - kill_watcher(signal.SIGINT) - elif command == 'reopen': - if not quiet: - print _('Re-opening all log files') - kill_watcher(signal.SIGHUP) - elif command == 'start': - # First, complain loudly if there's no site list. - check_for_site_list() - # Here's the scoop on the processes we're about to create. We'll need - # one for each qrunner, and one for a master child process watcher / - # lock refresher process. - # - # The child watcher process simply waits on the pids of the children - # qrunners. Unless explicitly disabled by a mailmanctl switch (or the - # children are killed with SIGTERM instead of SIGINT), the watcher - # will automatically restart any child process that exits. This - # allows us to be more robust, and also to implement restart by simply - # SIGINT'ing the qrunner children, and letting the watcher restart - # them. - # - # Under normal operation, we have a child per queue. This lets us get - # the most out of the available resources, since a qrunner with no - # files in its queue directory is pretty cheap, but having a separate - # runner process per queue allows for a very responsive system. Some - # people want a more traditional (i.e. MM2.0.x) cron-invoked qrunner. - # No problem, but using mailmanctl isn't the answer. So while - # mailmanctl hard codes some things, others, such as the number of - # qrunners per queue, is configurable in mm_cfg.py. - # - # First, acquire the master mailmanctl lock - lock = acquire_lock(force) - if not lock: - return - # Daemon process startup according to Stevens, Advanced Programming in - # the UNIX Environment, Chapter 13. - pid = os.fork() - if pid: - # parent - if not quiet: - print _("Starting Mailman's master qrunner.") - # Give up the lock "ownership". This just means the foreground - # process won't close/unlock the lock when it finalizes this lock - # instance. We'll let the mater watcher subproc own the lock. - lock._transfer_to(pid) - return - # child - lock._take_possession() - # First, save our pid in a file for "mailmanctl stop" rendezvous. We - # want the perms on the .pid file to be rw-rw---- - omask = os.umask(6) - try: - fp = open(mm_cfg.PIDFILE, 'w') - print >> fp, os.getpid() - fp.close() - finally: - os.umask(omask) - # Create a new session and become the session leader, but since we - # won't be opening any terminal devices, don't do the ultra-paranoid - # suggestion of doing a second fork after the setsid() call. - os.setsid() - # Instead of cd'ing to root, cd to the Mailman installation home - os.chdir(mm_cfg.PREFIX) - # Set our file mode creation umask - os.umask(007) - # I don't think we have any unneeded file descriptors. - # - # Now start all the qrunners. This returns a dictionary where the - # keys are qrunner pids and the values are tuples of the following - # form: (qrname, slice, count). This does its own fork and exec, and - # sets up its own signal handlers. - kids = start_all_runners() - # Set up a SIGALRM handler to refresh the lock once per day. The lock - # lifetime is 1day+6hours so this should be plenty. - def sigalrm_handler(signum, frame, lock=lock): - lock.refresh() - signal.alarm(mm_cfg.days(1)) - signal.signal(signal.SIGALRM, sigalrm_handler) - signal.alarm(mm_cfg.days(1)) - # Set up a SIGHUP handler so that if we get one, we'll pass it along - # to all the qrunner children. This will tell them to close and - # reopen their log files - def sighup_handler(signum, frame, kids=kids): - loginit.reopen() - for pid in kids.keys(): - os.kill(pid, signal.SIGHUP) - # And just to tweak things... - qlog.info('Master watcher caught SIGHUP. Re-opening log files.') - signal.signal(signal.SIGHUP, sighup_handler) - # We also need to install a SIGTERM handler because that's what init - # will kill this process with when changing run levels. - def sigterm_handler(signum, frame, kids=kids): - for pid in kids.keys(): - try: - os.kill(pid, signal.SIGTERM) - except OSError, e: - if e.errno <> errno.ESRCH: raise - qlog.info('Master watcher caught SIGTERM. Exiting.') - signal.signal(signal.SIGTERM, sigterm_handler) - # Finally, we need a SIGINT handler which will cause the sub-qrunners - # to exit, but the master will restart SIGINT'd sub-processes unless - # the -n flag was given. - def sigint_handler(signum, frame, kids=kids): - for pid in kids.keys(): - os.kill(pid, signal.SIGINT) - qlog.info('Master watcher caught SIGINT. Restarting.') - signal.signal(signal.SIGINT, sigint_handler) - # Now we're ready to simply do our wait/restart loop. This is the - # master qrunner watcher. - try: - while True: - try: - pid, status = os.wait() - except OSError, e: - # No children? We're done - if e.errno == errno.ECHILD: - break - # If the system call got interrupted, just restart it. - elif e.errno <> errno.EINTR: - raise - continue - killsig = exitstatus = None - if os.WIFSIGNALED(status): - killsig = os.WTERMSIG(status) - if os.WIFEXITED(status): - exitstatus = os.WEXITSTATUS(status) - # We'll restart the process unless we were given the - # "no-restart" switch, or if the process was SIGTERM'd or - # exitted with a SIGTERM exit status. This lets us better - # handle runaway restarts (say, if the subproc had a syntax - # error!) - restarting = '' - if restart: - if (exitstatus == None and killsig <> signal.SIGTERM) or \ - (killsig == None and exitstatus <> signal.SIGTERM): - # Then - restarting = '[restarting]' - qrname, slice, count, restarts = kids[pid] - del kids[pid] - qlog.info("""\ -Master qrunner detected subprocess exit -(pid: %d, sig: %s, sts: %s, class: %s, slice: %d/%d) %s""", - pid, killsig, exitstatus, qrname, - slice+1, count, restarting) - # See if we've reached the maximum number of allowable restarts - if exitstatus <> signal.SIGINT: - restarts += 1 - if restarts > MAX_RESTARTS: - qlog.info("""\ -Qrunner %s reached maximum restart limit of %d, not restarting.""", - qrname, MAX_RESTARTS) - restarting = '' - # Now perhaps restart the process unless it exited with a - # SIGTERM or we aren't restarting. - if restarting: - newpid = start_runner(qrname, slice, count) - kids[newpid] = (qrname, slice, count, restarts) - finally: - # Should we leave the main loop for any reason, we want to be sure - # all of our children are exited cleanly. Send SIGTERMs to all - # the child processes and wait for them all to exit. - for pid in kids.keys(): - try: - os.kill(pid, signal.SIGTERM) - except OSError, e: - if e.errno == errno.ESRCH: - # The child has already exited - qlog.info('ESRCH on pid: %d', pid) - del kids[pid] - # Wait for all the children to go away - while True: - try: - pid, status = os.wait() - except OSError, e: - if e.errno == errno.ECHILD: - break - elif e.errno <> errno.EINTR: - raise - continue - # Finally, give up the lock - lock.unlock(unconditionally=True) - os._exit(0) - - - -if __name__ == '__main__': - main() diff --git a/bin/qrunner b/bin/qrunner index 42248e472..e69de29bb 100644 --- a/bin/qrunner +++ b/bin/qrunner @@ -1,278 +0,0 @@ -#! @PYTHON@ - -# Copyright (C) 2001-2006 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""Run one or more qrunners, once or repeatedly. - -Each named runner class is run in round-robin fashion. In other words, the -first named runner is run to consume all the files currently in its -directory. When that qrunner is done, the next one is run to consume all the -files in /its/ directory, and so on. The number of total iterations can be -given on the command line. - -Usage: %(PROGRAM)s [options] - -Options: - - -r runner[:slice:range] - --runner=runner[:slice:range] - Run the named qrunner, which must be one of the strings returned by - the -l option. Optional slice:range if given, is used to assign - multiple qrunner processes to a queue. range is the total number of - qrunners for this queue while slice is the number of this qrunner from - [0..range). - - If using the slice:range form, you better make sure that each qrunner - for the queue is given the same range value. If slice:runner is not - given, then 1:1 is used. - - Multiple -r options may be given, in which case each qrunner will run - once in round-robin fashion. The special runner `All' is shorthand - for a qrunner for each listed by the -l option. - - --once - -o - Run each named qrunner exactly once through its main loop. Otherwise, - each qrunner runs indefinitely, until the process receives a SIGTERM - or SIGINT. - - -l/--list - Shows the available qrunner names and exit. - - -v/--verbose - Spit out more debugging information to the logs/qrunner log file. - - -s/--subproc - This should only be used when running qrunner as a subprocess of the - mailmanctl startup script. It changes some of the exit-on-error - behavior to work better with that framework. - - -h/--help - Print this message and exit. - -runner is required unless -l or -h is given, and it must be one of the names -displayed by the -l switch. - -Note also that this script should be started up from mailmanctl as a normal -operation. It is only useful for debugging if it is run separately. -""" - -import sys -import getopt -import signal -import logging - -import paths -from Mailman import mm_cfg -from Mailman import loginit -from Mailman.i18n import _ - -PROGRAM = sys.argv[0] -COMMASPACE = ', ' - -# Flag which says whether we're running under mailmanctl or not. -AS_SUBPROC = 0 - -log = logging.getLogger('mailman.qrunner') - - - -def usage(code, msg=''): - if code: - fd = sys.stderr - else: - fd = sys.stdout - print >> fd, _(__doc__) - if msg: - print >> fd, msg - sys.exit(code) - - - -def make_qrunner(name, slice, range, once=0): - modulename = 'Mailman.Queue.' + name - try: - __import__(modulename) - except ImportError, e: - if AS_SUBPROC: - # Exit with SIGTERM exit code so mailmanctl won't try to restart us - print >> sys.stderr, 'Cannot import runner module', modulename - print >> sys.stderr, e - sys.exit(signal.SIGTERM) - else: - usage(1, e) - qrclass = getattr(sys.modules[modulename], name) - if once: - # Subclass to hack in the setting of the stop flag in _doperiodic() - class Once(qrclass): - def _doperiodic(self): - self.stop() - qrunner = Once(slice, range) - else: - qrunner = qrclass(slice, range) - return qrunner - - - -def set_signals(loop): - # Set up the SIGTERM handler for stopping the loop - def sigterm_handler(signum, frame, loop=loop): - # Exit the qrunner cleanly - loop.stop() - loop.status = signal.SIGTERM - log.info('%s qrunner caught SIGTERM. Stopping.', loop.name()) - signal.signal(signal.SIGTERM, sigterm_handler) - # Set up the SIGINT handler for stopping the loop. For us, SIGINT is - # the same as SIGTERM, but our parent treats the exit statuses - # differently (it restarts a SIGINT but not a SIGTERM). - def sigint_handler(signum, frame, loop=loop): - # Exit the qrunner cleanly - loop.stop() - loop.status = signal.SIGINT - log.info('%s qrunner caught SIGINT. Stopping.', loop.name()) - signal.signal(signal.SIGINT, sigint_handler) - # SIGHUP just tells us to rotate our log files. - def sighup_handler(signum, frame, loop=loop): - loginit.reopen() - log.info('%s qrunner caught SIGHUP. Reopening logs.', loop.name()) - signal.signal(signal.SIGHUP, sighup_handler) - - - -def main(): - global AS_SUBPROC - try: - opts, args = getopt.getopt( - sys.argv[1:], 'hlor:vs', - ['help', 'list', 'once', 'runner=', 'verbose', 'subproc']) - except getopt.error, msg: - usage(1, msg) - - once = 0 - runners = [] - verbose = 0 - for opt, arg in opts: - if opt in ('-h', '--help'): - usage(0) - elif opt in ('-l', '--list'): - for runnername, slices in mm_cfg.QRUNNERS: - if runnername.endswith('Runner'): - name = runnername[:-len('Runner')] - else: - name = runnername - print _('%(name)s runs the %(runnername)s qrunner') - print _('All runs all the above qrunners') - sys.exit(0) - elif opt in ('-o', '--once'): - once = 1 - elif opt in ('-r', '--runner'): - runnerspec = arg - parts = runnerspec.split(':') - if len(parts) == 1: - runner = parts[0] - slice = 1 - range = 1 - elif len(parts) == 3: - runner = parts[0] - try: - slice = int(parts[1]) - range = int(parts[2]) - except ValueError: - usage(1, 'Bad runner specification: %(runnerspec)s') - else: - usage(1, 'Bad runner specification: %(runnerspec)s') - if runner == 'All': - for runnername, slices in mm_cfg.QRUNNERS: - runners.append((runnername, slice, range)) - else: - if runner.endswith('Runner'): - runners.append((runner, slice, range)) - else: - runners.append((runner + 'Runner', slice, range)) - elif opt in ('-s', '--subproc'): - AS_SUBPROC = 1 - elif opt in ('-v', '--verbose'): - verbose = 1 - - if len(args) <> 0: - usage(1) - if len(runners) == 0: - usage(1, _('No runner name given.')) - - # If we're not running as a subprocess of mailmanctl, then we'll log to - # stderr in addition to logging to the log files. We do this by passing a - # value of True to propagate, which allows the 'mailman' root logger to - # see the log messages. - loginit.initialize(propagate=not AS_SUBPROC) - - # Fast track for one infinite runner - if len(runners) == 1 and not once: - qrunner = make_qrunner(*runners[0]) - class Loop: - status = 0 - def __init__(self, qrunner): - self.__qrunner = qrunner - def name(self): - return self.__qrunner.__class__.__name__ - def stop(self): - self.__qrunner.stop() - loop = Loop(qrunner) - set_signals(loop) - # Now start up the main loop - log.info('%s qrunner started.', loop.name()) - qrunner.run() - log.info('%s qrunner exiting.', loop.name()) - else: - # Anything else we have to handle a bit more specially - qrunners = [] - for runner, slice, range in runners: - qrunner = make_qrunner(runner, slice, range, 1) - qrunners.append(qrunner) - # This class is used to manage the main loop - class Loop: - status = 0 - def __init__(self): - self.__isdone = 0 - def name(self): - return 'Main loop' - def stop(self): - self.__isdone = 1 - def isdone(self): - return self.__isdone - loop = Loop() - set_signals(loop) - log.info('Main qrunner loop started.') - while not loop.isdone(): - for qrunner in qrunners: - # In case the SIGTERM came in the middle of this iteration - if loop.isdone(): - break - if verbose: - log.info('Now doing a %s qrunner iteration', - qrunner.__class__.__bases__[0].__name__) - qrunner.run() - if once: - break - log.info('Main qrunner loop exiting.') - # All done - sys.exit(loop.status) - - - -if __name__ == '__main__': - main() diff --git a/bin/update b/bin/update index 23429508e..e69de29bb 100755 --- a/bin/update +++ b/bin/update @@ -1,807 +0,0 @@ -#! @PYTHON@ -# -# Copyright (C) 1998-2005 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -"""Perform all necessary upgrades. - -Usage: %(PROGRAM)s [options] - -Options: - -f/--force - Force running the upgrade procedures. Normally, if the version number - of the installed Mailman matches the current version number (or a - `downgrade' is detected), nothing will be done. - - -h/--help - Print this text and exit. - -Use this script to help you update to the latest release of Mailman from -some previous version. It knows about versions back to 1.0b4 (?). -""" - -import os -import md5 -import sys -import time -import errno -import getopt -import shutil -import cPickle -import marshal - -import paths -import email - -from Mailman import mm_cfg -from Mailman import Utils -from Mailman import MailList -from Mailman import Message -from Mailman import Pending -from Mailman.LockFile import TimeOutError -from Mailman.i18n import _ -from Mailman.Queue.Switchboard import Switchboard -from Mailman.OldStyleMemberships import OldStyleMemberships -from Mailman.MemberAdaptor import BYBOUNCE, ENABLED - -FRESH = 0 -NOTFRESH = -1 - -LMVFILE = os.path.join(mm_cfg.DATA_DIR, 'last_mailman_version') -PROGRAM = sys.argv[0] - - - -def calcversions(): - # Returns a tuple of (lastversion, thisversion). If the last version - # could not be determined, lastversion will be FRESH or NOTFRESH, - # depending on whether this installation appears to be fresh or not. The - # determining factor is whether there are files in the $var_prefix/logs - # subdir or not. The version numbers are HEX_VERSIONs. - # - # See if we stored the last updated version - lastversion = None - thisversion = mm_cfg.HEX_VERSION - try: - fp = open(LMVFILE) - data = fp.read() - fp.close() - lastversion = int(data, 16) - except (IOError, ValueError): - pass - # - # try to figure out if this is a fresh install - if lastversion is None: - lastversion = FRESH - try: - if os.listdir(mm_cfg.LOG_DIR): - lastversion = NOTFRESH - except OSError: - pass - return (lastversion, thisversion) - - - -def makeabs(relpath): - return os.path.join(mm_cfg.PREFIX, relpath) - -def make_varabs(relpath): - return os.path.join(mm_cfg.VAR_PREFIX, relpath) - - -def move_language_templates(mlist): - listname = mlist.internal_name() - print _('Fixing language templates: %(listname)s') - # Mailman 2.1 has a new cascading search for its templates, defined and - # described in Utils.py:maketext(). Putting templates in the top level - # templates/ subdir or the lists/<listname> subdir is deprecated and no - # longer searched.. - # - # What this means is that most templates can live in the global templates/ - # subdirectory, and only needs to be copied into the list-, vhost-, or - # site-specific language directories when needed. - # - # Also, by default all standard (i.e. English) templates must now live in - # the templates/en directory. This update cleans up all the templates, - # deleting more-specific duplicates (as calculated by md5 checksums) in - # favor of more-global locations. - # - # First, get rid of any lists/<list> template or lists/<list>/en template - # that is identical to the global templates/* default. - for gtemplate in os.listdir(os.path.join(mm_cfg.TEMPLATE_DIR, 'en')): - # BAW: get rid of old templates, e.g. admlogin.txt and - # handle_opts.html - try: - fp = open(os.path.join(mm_cfg.TEMPLATE_DIR, gtemplate)) - except IOError, e: - if e.errno <> errno.ENOENT: raise - # No global template - continue - - gcksum = md5.new(fp.read()).digest() - fp.close() - # Match against the lists/<list>/* template - try: - fp = open(os.path.join(mlist.fullpath(), gtemplate)) - except IOError, e: - if e.errno <> errno.ENOENT: raise - else: - tcksum = md5.new(fp.read()).digest() - fp.close() - if gcksum == tcksum: - os.unlink(os.path.join(mlist.fullpath(), gtemplate)) - # Match against the lists/<list>/*.prev template - try: - fp = open(os.path.join(mlist.fullpath(), gtemplate + '.prev')) - except IOError, e: - if e.errno <> errno.ENOENT: raise - else: - tcksum = md5.new(fp.read()).digest() - fp.close() - if gcksum == tcksum: - os.unlink(os.path.join(mlist.fullpath(), gtemplate + '.prev')) - # Match against the lists/<list>/en/* templates - try: - fp = open(os.path.join(mlist.fullpath(), 'en', gtemplate)) - except IOError, e: - if e.errno <> errno.ENOENT: raise - else: - tcksum = md5.new(fp.read()).digest() - fp.close() - if gcksum == tcksum: - os.unlink(os.path.join(mlist.fullpath(), 'en', gtemplate)) - # Match against the templates/* template - try: - fp = open(os.path.join(mm_cfg.TEMPLATE_DIR, gtemplate)) - except IOError, e: - if e.errno <> errno.ENOENT: raise - else: - tcksum = md5.new(fp.read()).digest() - fp.close() - if gcksum == tcksum: - os.unlink(os.path.join(mm_cfg.TEMPLATE_DIR, gtemplate)) - # Match against the templates/*.prev template - try: - fp = open(os.path.join(mm_cfg.TEMPLATE_DIR, gtemplate + '.prev')) - except IOError, e: - if e.errno <> errno.ENOENT: raise - else: - tcksum = md5.new(fp.read()).digest() - fp.close() - if gcksum == tcksum: - os.unlink(os.path.join(mm_cfg.TEMPLATE_DIR, - gtemplate + '.prev')) - - - -def dolist(listname): - errors = 0 - mlist = MailList.MailList(listname, lock=0) - try: - mlist.Lock(0.5) - except TimeOutError: - print >> sys.stderr, _('WARNING: could not acquire lock for list: ' - '%(listname)s') - return 1 - - # Sanity check the invariant that every BYBOUNCE disabled member must have - # bounce information. Some earlier betas broke this. BAW: we're - # submerging below the MemberAdaptor interface, so skip this if we're not - # using OldStyleMemberships. - if isinstance(mlist._memberadaptor, OldStyleMemberships): - noinfo = {} - for addr, (reason, when) in mlist.delivery_status.items(): - if reason == BYBOUNCE and not mlist.bounce_info.has_key(addr): - noinfo[addr] = reason, when - # What to do about these folks with a BYBOUNCE delivery status and no - # bounce info? This number should be very small, and I think it's - # fine to simple re-enable them and let the bounce machinery - # re-disable them if necessary. - n = len(noinfo) - if n > 0: - print _( - 'Resetting %(n)s BYBOUNCEs disabled addrs with no bounce info') - for addr in noinfo.keys(): - mlist.setDeliveryStatus(addr, ENABLED) - - # Update the held requests database - print _("""Updating the held requests database.""") - mlist._UpdateRecords() - - mbox_dir = make_varabs('archives/private/%s.mbox' % (listname)) - mbox_file = make_varabs('archives/private/%s.mbox/%s' % (listname, - listname)) - - o_pub_mbox_file = make_varabs('archives/public/%s' % (listname)) - o_pri_mbox_file = make_varabs('archives/private/%s' % (listname)) - - html_dir = o_pri_mbox_file - o_html_dir = makeabs('public_html/archives/%s' % (listname)) - # - # make the mbox directory if it's not there. - # - if not os.path.exists(mbox_dir): - ou = os.umask(0) - os.mkdir(mbox_dir, 02775) - os.umask(ou) - else: - # this shouldn't happen, but hey, just in case - if not os.path.isdir(mbox_dir): - print _("""\ -For some reason, %(mbox_dir)s exists as a file. This won't work with -b6, so I'm renaming it to %(mbox_dir)s.tmp and proceeding.""") - os.rename(mbox_dir, "%s.tmp" % (mbox_dir)) - ou = os.umask(0) - os.mkdir(mbox_dir, 02775) - os.umask(ou) - - # Move any existing mboxes around, but watch out for both a public and a - # private one existing - if os.path.isfile(o_pri_mbox_file) and os.path.isfile(o_pub_mbox_file): - if mlist.archive_private: - print _("""\ - -%(listname)s has both public and private mbox archives. Since this list -currently uses private archiving, I'm installing the private mbox archive --- %(o_pri_mbox_file)s -- as the active archive, and renaming - %(o_pub_mbox_file)s -to - %(o_pub_mbox_file)s.preb6 - -You can integrate that into the archives if you want by using the 'arch' -script. -""") % (mlist._internal_name, o_pri_mbox_file, o_pub_mbox_file, - o_pub_mbox_file) - os.rename(o_pub_mbox_file, "%s.preb6" % (o_pub_mbox_file)) - else: - print _("""\ -%s has both public and private mbox archives. Since this list -currently uses public archiving, I'm installing the public mbox file -archive file (%s) as the active one, and renaming - %s - to - %s.preb6 - -You can integrate that into the archives if you want by using the 'arch' -script. -""") % (mlist._internal_name, o_pub_mbox_file, o_pri_mbox_file, - o_pri_mbox_file) - os.rename(o_pri_mbox_file, "%s.preb6" % (o_pri_mbox_file)) - # - # move private archive mbox there if it's around - # and take into account all sorts of absurdities - # - print _('- updating old private mbox file') - if os.path.exists(o_pri_mbox_file): - if os.path.isfile(o_pri_mbox_file): - os.rename(o_pri_mbox_file, mbox_file) - elif not os.path.isdir(o_pri_mbox_file): - newname = "%s.mm_install-dunno_what_this_was_but_its_in_the_way" \ - % o_pri_mbox_file - os.rename(o_pri_mbox_file, newname) - print _("""\ - unknown file in the way, moving - %(o_pri_mbox_file)s - to - %(newname)s""") - else: - # directory - print _("""\ - looks like you have a really recent CVS installation... - you're either one brave soul, or you already ran me""") - - - # - # move public archive mbox there if it's around - # and take into account all sorts of absurdities. - # - print _('- updating old public mbox file') - if os.path.exists(o_pub_mbox_file): - if os.path.isfile(o_pub_mbox_file): - os.rename(o_pub_mbox_file, mbox_file) - elif not os.path.isdir(o_pub_mbox_file): - newname = "%s.mm_install-dunno_what_this_was_but_its_in_the_way" \ - % o_pub_mbox_file - os.rename(o_pub_mbox_file, newname) - print _("""\ - unknown file in the way, moving - %(o_pub_mbox_file)s - to - %(newname)s""") - else: # directory - print _("""\ - looks like you have a really recent CVS installation... - you're either one brave soul, or you already ran me""") - - # - # move the html archives there - # - if os.path.isdir(o_html_dir): - os.rename(o_html_dir, html_dir) - # - # chmod the html archives - # - os.chmod(html_dir, 02775) - # BAW: Is this still necessary?! - mlist.Save() - # - # check to see if pre-b4 list-specific templates are around - # and move them to the new place if there's not already - # a new one there - # - tmpl_dir = os.path.join(mm_cfg.PREFIX, "templates") - list_dir = os.path.join(mm_cfg.PREFIX, "lists") - b4_tmpl_dir = os.path.join(tmpl_dir, mlist._internal_name) - new_tmpl_dir = os.path.join(list_dir, mlist._internal_name) - if os.path.exists(b4_tmpl_dir): - print _("""\ -- This list looks like it might have <= b4 list templates around""") - for f in os.listdir(b4_tmpl_dir): - o_tmpl = os.path.join(b4_tmpl_dir, f) - n_tmpl = os.path.join(new_tmpl_dir, f) - if os.path.exists(o_tmpl): - if not os.path.exists(n_tmpl): - os.rename(o_tmpl, n_tmpl) - print _('- moved %(o_tmpl)s to %(n_tmpl)s') - else: - print _("""\ -- both %(o_tmpl)s and %(n_tmpl)s exist, leaving untouched""") - else: - print _("""\ -- %(o_tmpl)s doesn't exist, leaving untouched""") - # - # Move all the templates to the en language subdirectory as required for - # Mailman 2.1 - # - move_language_templates(mlist) - # Avoid eating filehandles with the list lockfiles - mlist.Unlock() - return 0 - - - -def archive_path_fixer(unused_arg, dir, files): - # Passed to os.path.walk to fix the perms on old html archives. - for f in files: - abs = os.path.join(dir, f) - if os.path.isdir(abs): - if f == "database": - os.chmod(abs, 02770) - else: - os.chmod(abs, 02775) - elif os.path.isfile(abs): - os.chmod(abs, 0664) - -def remove_old_sources(module): - # Also removes old directories. - src = '%s/%s' % (mm_cfg.PREFIX, module) - pyc = src + "c" - if os.path.isdir(src): - print _('removing directory %(src)s and everything underneath') - shutil.rmtree(src) - elif os.path.exists(src): - print _('removing %(src)s') - try: - os.unlink(src) - except os.error, rest: - print _("Warning: couldn't remove %(src)s -- %(rest)s") - if module.endswith('.py') and os.path.exists(pyc): - try: - os.unlink(pyc) - except os.error, rest: - print _("couldn't remove old file %(pyc)s -- %(rest)s") - - -def update_qfiles(): - print _('updating old qfiles') - prefix = `time.time()` + '+' - # Be sure the qfiles/in directory exists (we don't really need the - # switchboard object, but it's convenient for creating the directory). - sb = Switchboard(mm_cfg.INQUEUE_DIR) - for filename in os.listdir(mm_cfg.QUEUE_DIR): - # Updating means just moving the .db and .msg files to qfiles/in where - # it should be dequeued, converted, and processed normally. - if filename.endswith('.msg'): - oldmsgfile = os.path.join(mm_cfg.QUEUE_DIR, filename) - newmsgfile = os.path.join(mm_cfg.INQUEUE_DIR, prefix + filename) - os.rename(oldmsgfile, newmsgfile) - elif filename.endswith('.db'): - olddbfile = os.path.join(mm_cfg.QUEUE_DIR, filename) - newdbfile = os.path.join(mm_cfg.INQUEUE_DIR, prefix + filename) - os.rename(olddbfile, newdbfile) - # Now update for the Mailman 2.1.5 qfile format. For every filebase in - # the qfiles/* directories that has both a .pck and a .db file, pull the - # data out and re-queue them. - for dirname in os.listdir(mm_cfg.QUEUE_DIR): - dirpath = os.path.join(mm_cfg.QUEUE_DIR, dirname) - if dirpath == mm_cfg.BADQUEUE_DIR: - # The files in qfiles/bad can't possibly be pickles - continue - sb = Switchboard(dirpath) - try: - for filename in os.listdir(dirpath): - filepath = os.path.join(dirpath, filename) - filebase, ext = os.path.splitext(filepath) - # Handle the .db metadata files as part of the handling of the - # .pck or .msg message files. - if ext not in ('.pck', '.msg'): - continue - msg, data = dequeue(filebase) - if msg is not None and data is not None: - sb.enqueue(msg, data) - except EnvironmentError, e: - if e.errno <> errno.ENOTDIR: - raise - print _('Warning! Not a directory: %(dirpath)s') - - - -# Implementations taken from the pre-2.1.5 Switchboard -def ext_read(filename): - fp = open(filename) - d = marshal.load(fp) - # Update from version 2 files - if d.get('version', 0) == 2: - del d['filebase'] - # Do the reverse conversion (repr -> float) - for attr in ['received_time']: - try: - sval = d[attr] - except KeyError: - pass - else: - # Do a safe eval by setting up a restricted execution - # environment. This may not be strictly necessary since we - # know they are floats, but it can't hurt. - d[attr] = eval(sval, {'__builtins__': {}}) - fp.close() - return d - - -def dequeue(filebase): - # Calculate the .db and .msg filenames from the given filebase. - msgfile = os.path.join(filebase + '.msg') - pckfile = os.path.join(filebase + '.pck') - dbfile = os.path.join(filebase + '.db') - # Now we are going to read the message and metadata for the given - # filebase. We want to read things in this order: first, the metadata - # file to find out whether the message is stored as a pickle or as - # plain text. Second, the actual message file. However, we want to - # first unlink the message file and then the .db file, because the - # qrunner only cues off of the .db file - msg = None - try: - data = ext_read(dbfile) - os.unlink(dbfile) - except EnvironmentError, e: - if e.errno <> errno.ENOENT: raise - data = {} - # Between 2.1b4 and 2.1b5, the `rejection-notice' key in the metadata - # was renamed to `rejection_notice', since dashes in the keys are not - # supported in METAFMT_ASCII. - if data.has_key('rejection-notice'): - data['rejection_notice'] = data['rejection-notice'] - del data['rejection-notice'] - msgfp = None - try: - try: - msgfp = open(pckfile) - msg = cPickle.load(msgfp) - os.unlink(pckfile) - except EnvironmentError, e: - if e.errno <> errno.ENOENT: raise - msgfp = None - try: - msgfp = open(msgfile) - msg = email.message_from_file(msgfp, Message.Message) - os.unlink(msgfile) - except EnvironmentError, e: - if e.errno <> errno.ENOENT: raise - except (email.Errors.MessageParseError, ValueError), e: - # This message was unparsable, most likely because its - # MIME encapsulation was broken. For now, there's not - # much we can do about it. - print _('message is unparsable: %(filebase)s') - msgfp.close() - msgfp = None - if mm_cfg.QRUNNER_SAVE_BAD_MESSAGES: - # Cheapo way to ensure the directory exists w/ the - # proper permissions. - sb = Switchboard(mm_cfg.BADQUEUE_DIR) - os.rename(msgfile, os.path.join( - mm_cfg.BADQUEUE_DIR, filebase + '.txt')) - else: - os.unlink(msgfile) - msg = data = None - except EOFError: - # For some reason the pckfile was empty. Just delete it. - print _('Warning! Deleting empty .pck file: %(pckfile)s') - os.unlink(pckfile) - finally: - if msgfp: - msgfp.close() - return msg, data - - - -def update_pending(): - file20 = os.path.join(mm_cfg.DATA_DIR, 'pending_subscriptions.db') - file214 = os.path.join(mm_cfg.DATA_DIR, 'pending.pck') - db = None - # Try to load the Mailman 2.0 file - try: - fp = open(file20) - except IOError, e: - if e.errno <> errno.ENOENT: raise - else: - print _('Updating Mailman 2.0 pending_subscriptions.db database') - db = marshal.load(fp) - # Convert to the pre-Mailman 2.1.5 format - db = Pending._update(db) - if db is None: - # Try to load the Mailman 2.1.x where x < 5, file - try: - fp = open(file214) - except IOError, e: - if e.errno <> errno.ENOENT: raise - else: - print _('Updating Mailman 2.1.4 pending.pck database') - db = cPickle.load(fp) - if db is None: - print _('Nothing to do.') - return - # Now upgrade the database to the 2.1.5 format. Each list now has its own - # pending.pck file, but only the RE_ENABLE operation actually recorded the - # listname in the request. For the SUBSCRIPTION, UNSUBSCRIPTION, and - # CHANGE_OF_ADDRESS operations, we know the address of the person making - # the request so we can repend this request just for the lists the person - # is a member of. For the HELD_MESSAGE operation, we can check the list's - # requests.pck file for correlation. Evictions will take care of any - # misdirected pendings. - reenables_by_list = {} - addrops_by_address = {} - holds_by_id = {} - subs_by_address = {} - for key, val in db.items(): - if key in ('evictions', 'version'): - continue - try: - op = val[0] - data = val[1:] - except (IndexError, ValueError): - print _('Ignoring bad pended data: %(key)s: %(val)s') - continue - if op in (Pending.UNSUBSCRIPTION, Pending.CHANGE_OF_ADDRESS): - # data[0] is the address being unsubscribed - addrops_by_address.setdefault(data[0], []).append((key, val)) - elif op == Pending.SUBSCRIPTION: - # data[0] is a UserDesc object - addr = data[0].address - subs_by_address.setdefault(addr, []).append((key, val)) - elif op == Pending.RE_ENABLE: - # data[0] is the mailing list's internal name - reenables_by_list.setdefault(data[0], []).append((key, val)) - elif op == Pending.HELD_MESSAGE: - # data[0] is the hold id. There better only be one entry per id - id = data[0] - if holds_by_id.has_key(id): - print _('WARNING: Ignoring duplicate pending ID: %(id)s.') - else: - holds_by_id[id] = (key, val) - # Now we have to lock every list and re-pend all the appropriate - # requests. Note that this will reset all the expiration dates, but that - # should be fine. - for listname in Utils.list_names(): - mlist = MailList.MailList(listname) - # This is not the most efficient way to do this because it loads and - # saves the pending.pck file each time. :( - try: - for cookie, data in reenables_by_list.get(listname, []): - mlist.pend_repend(cookie, data) - for id, (cookie, data) in holds_by_id.items(): - try: - rec = mlist.GetRecord(id) - except KeyError: - # Not for this list - pass - else: - mlist.pend_repend(cookie, data) - del holds_by_id[id] - for addr, recs in subs_by_address.items(): - # We shouldn't have a subscription confirmation if the address - # is already a member of the mailing list. - if mlist.isMember(addr): - continue - for cookie, data in recs: - mlist.pend_repend(cookie, data) - for addr, recs in addrops_by_address.items(): - # We shouldn't have unsubscriptions or change of address - # requests for addresses which aren't members of the list. - if not mlist.isMember(addr): - continue - for cookie, data in recs: - mlist.pend_repend(cookie, data) - mlist.Save() - finally: - mlist.Unlock() - try: - os.unlink(file20) - except OSError, e: - if e.errno <> errno.ENOENT: raise - try: - os.unlink(file214) - except OSError, e: - if e.errno <> errno.ENOENT: raise - - - -def main(): - errors = 0 - # get rid of old stuff - print _('getting rid of old source files') - for mod in ('Mailman/Archiver.py', 'Mailman/HyperArch.py', - 'Mailman/HyperDatabase.py', 'Mailman/pipermail.py', - 'Mailman/smtplib.py', 'Mailman/Cookie.py', - 'bin/update_to_10b6', 'scripts/mailcmd', - 'scripts/mailowner', 'mail/wrapper', 'Mailman/pythonlib', - 'cgi-bin/archives', 'Mailman/MailCommandHandler'): - remove_old_sources(mod) - listnames = Utils.list_names() - if not listnames: - print _('no lists == nothing to do, exiting') - return - # - # for people with web archiving, make sure the directories - # in the archiving are set with proper perms for b6. - # - if os.path.isdir("%s/public_html/archives" % mm_cfg.PREFIX): - print _("""\ -fixing all the perms on your old html archives to work with b6 -If your archives are big, this could take a minute or two...""") - os.path.walk("%s/public_html/archives" % mm_cfg.PREFIX, - archive_path_fixer, "") - print _('done') - for listname in listnames: - print _('Updating mailing list: %(listname)s') - errors = errors + dolist(listname) - print - print _('Updating Usenet watermarks') - wmfile = os.path.join(mm_cfg.DATA_DIR, 'gate_watermarks') - try: - fp = open(wmfile) - except IOError: - print _('- nothing to update here') - else: - d = marshal.load(fp) - fp.close() - for listname in d.keys(): - if listname not in listnames: - # this list no longer exists - continue - mlist = MailList.MailList(listname, lock=0) - try: - mlist.Lock(0.5) - except TimeOutError: - print >> sys.stderr, _( - 'WARNING: could not acquire lock for list: %(listname)s') - errors = errors + 1 - else: - # Pre 1.0b7 stored 0 in the gate_watermarks file to indicate - # that no gating had been done yet. Without coercing this to - # None, the list could now suddenly get flooded. - mlist.usenet_watermark = d[listname] or None - mlist.Save() - mlist.Unlock() - os.unlink(wmfile) - print _('- usenet watermarks updated and gate_watermarks removed') - # In Mailman 2.1, the pending database format and file name changed, but - # in Mailman 2.1.5 it changed again. This should update all existing - # files to the 2.1.5 format. - update_pending() - # In Mailman 2.1, the qfiles directory has a different structure and a - # different content. Also, in Mailman 2.1.5 we collapsed the message - # files from separate .msg (pickled Message objects) and .db (marshalled - # dictionaries) to a shared .pck file containing two pickles. - update_qfiles() - # This warning was necessary for the upgrade from 1.0b9 to 1.0b10. - # There's no good way of figuring this out for releases prior to 2.0beta2 - # :( - if lastversion == NOTFRESH: - print _(""" - -NOTE NOTE NOTE NOTE NOTE - - You are upgrading an existing Mailman installation, but I can't tell what - version you were previously running. - - If you are upgrading from Mailman 1.0b9 or earlier you will need to - manually update your mailing lists. For each mailing list you need to - copy the file templates/options.html lists/<listname>/options.html. - - However, if you have edited this file via the Web interface, you will have - to merge your changes into this file, otherwise you will lose your - changes. - -NOTE NOTE NOTE NOTE NOTE - -""") - return errors - - - -def usage(code, msg=''): - if code: - fd = sys.stderr - else: - fd = sys.stdout - print >> fd, _(__doc__) % globals() - if msg: - print >> sys.stderr, msg - sys.exit(code) - - - -if __name__ == '__main__': - try: - opts, args = getopt.getopt(sys.argv[1:], 'hf', - ['help', 'force']) - except getopt.error, msg: - usage(1, msg) - - if args: - usage(1, 'Unexpected arguments: %s' % args) - - force = 0 - for opt, arg in opts: - if opt in ('-h', '--help'): - usage(0) - elif opt in ('-f', '--force'): - force = 1 - - # calculate the versions - lastversion, thisversion = calcversions() - hexlversion = hex(lastversion) - hextversion = hex(thisversion) - if lastversion == thisversion and not force: - # nothing to do - print _('No updates are necessary.') - sys.exit(0) - if lastversion > thisversion and not force: - print _("""\ -Downgrade detected, from version %(hexlversion)s to version %(hextversion)s -This is probably not safe. -Exiting.""") - sys.exit(1) - print _('Upgrading from version %(hexlversion)s to %(hextversion)s') - errors = main() - if not errors: - # Record the version we just upgraded to - fp = open(LMVFILE, 'w') - fp.write(hex(mm_cfg.HEX_VERSION) + '\n') - fp.close() - else: - lockdir = mm_cfg.LOCK_DIR - print _('''\ - -ERROR: - -The locks for some lists could not be acquired. This means that either -Mailman was still active when you upgraded, or there were stale locks in the -%(lockdir)s directory. - -You must put Mailman into a quiescent state and remove all stale locks, then -re-run "make update" manually. See the INSTALL and UPGRADE files for details. -''') @@ -1,5 +1,5 @@ #! /bin/sh -# From configure.in Revision: 7897 . +# From configure.in Revision: 7899 . # Guess values for system-dependent variables and create Makefiles. # Generated by GNU Autoconf 2.59 for GNU Mailman 2.2.0a0. # @@ -4295,16 +4295,13 @@ build/bin/dumpdb:bin/dumpdb \ build/bin/fix_url.py:bin/fix_url.py \ build/bin/genaliases:bin/genaliases \ build/bin/list_admins:bin/list_admins \ -build/bin/mailmanctl:bin/mailmanctl \ build/bin/mmshell:bin/mmshell \ build/bin/msgfmt.py:bin/msgfmt.py \ build/bin/pygettext.py:bin/pygettext.py \ -build/bin/qrunner:bin/qrunner \ build/bin/remove_members:bin/remove_members \ build/bin/reset_pw.py:bin/reset_pw.py \ build/bin/sync_members:bin/sync_members \ build/bin/transcheck:bin/transcheck \ -build/bin/update:bin/update \ build/bin/withlist:bin/withlist \ build/bin/templ2pot.py:bin/templ2pot.py \ build/bin/po2templ.py:bin/po2templ.py \ diff --git a/configure.in b/configure.in index 3cd34d5aa..4cfeba5d3 100644 --- a/configure.in +++ b/configure.in @@ -15,7 +15,7 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. dnl Process this file with autoconf to produce a configure script. -AC_REVISION($Revision: 7899 $) +AC_REVISION($Revision: 7926 $) AC_PREREQ(2.0) AC_INIT([GNU Mailman], [2.2.0a0]) @@ -602,16 +602,13 @@ bin/dumpdb \ bin/fix_url.py \ bin/genaliases \ bin/list_admins \ -bin/mailmanctl \ bin/mmshell \ bin/msgfmt.py \ bin/pygettext.py \ -bin/qrunner \ bin/remove_members \ bin/reset_pw.py \ bin/sync_members \ bin/transcheck \ -bin/update \ bin/withlist \ bin/templ2pot.py \ bin/po2templ.py \ diff --git a/misc/Makefile.in b/misc/Makefile.in index 20d90d640..6c1a3a55e 100644 --- a/misc/Makefile.in +++ b/misc/Makefile.in @@ -42,6 +42,7 @@ OPT= @OPT@ CFLAGS= $(OPT) $(DEFS) PACKAGEDIR= $(prefix)/Mailman DATADIR= $(var_prefix)/data +ETCDIR= $(var_prefix)/etc ICONDIR= $(prefix)/icons SCRIPTSDIR= $(prefix)/scripts @@ -83,6 +84,7 @@ install-other: done $(INSTALL) -m $(EXEMODE) mailman $(DESTDIR)$(SCRIPTSDIR) $(INSTALL) -m $(FILEMODE) sitelist.cfg $(DESTDIR)$(DATADIR) + $(INSTALL) -m $(FILEMODE) mailman.cfg.sample $(DESTDIR)$(ETCDIR) install-packages: for p in $(PACKAGES); \ diff --git a/misc/mailman.cfg.sample b/misc/mailman.cfg.sample new file mode 100644 index 000000000..4cc4ec280 --- /dev/null +++ b/misc/mailman.cfg.sample @@ -0,0 +1,66 @@ +# -*- python -*- + +# Copyright (C) 2006 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +"""This module contains your site-specific settings. + +Use this to override the default settings in Mailman/Defaults.py. You only +need to include those settings that you want to change, and unlike the old +mm_cfg.py file, you do /not/ need to import Defaults. Its variables will +automatically be available in this module's namespace. + +You should consult Defaults.py though for a complete listing of configuration +variables that you can change. + +To use this, copy this file to $VAR_PREFIX/etc/mailman.cfg + +Mailman's installation procedure will never overwrite mailman.cfg. +""" +# -*- python -*- + +# Copyright (C) 2006 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +"""This module contains your site-specific settings. + +Use this to override the default settings in Mailman/Defaults.py. You only +need to include those settings that you want to change, and unlike the old +mm_cfg.py file, you do /not/ need to import Defaults. Its variables will +automatically be available in this module's namespace. + +You should consult Defaults.py though for a complete listing of configuration +variables that you can change. + +To use this, copy this file to $VAR_PREFIX/etc/mailman.cfg + +Mailman's installation procedure will never overwrite mailman.cfg. +""" |