diff options
| author | bwarsaw | 2006-07-08 17:37:55 +0000 |
|---|---|---|
| committer | bwarsaw | 2006-07-08 17:37:55 +0000 |
| commit | cbef3114de3e80b9436d909b11568858e3a1cf42 (patch) | |
| tree | f567fe3fbc331fe399b92e93f80068e8995a7821 | |
| parent | 60b723291e592ff7925e1b15b79161d1cdac5938 (diff) | |
| download | mailman-cbef3114de3e80b9436d909b11568858e3a1cf42.tar.gz mailman-cbef3114de3e80b9436d909b11568858e3a1cf42.tar.zst mailman-cbef3114de3e80b9436d909b11568858e3a1cf42.zip | |
40 files changed, 2029 insertions, 1943 deletions
diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in index 27eb73ee5..d8248a3cf 100644 --- a/Mailman/Defaults.py.in +++ b/Mailman/Defaults.py.in @@ -1264,44 +1264,10 @@ AuthListAdmin = 3 # List Administrator (total control over list) AuthListModerator = 4 # List Moderator (can only handle held requests) AuthSiteAdmin = 5 # Site Administrator (total control over everything) -# Useful directories -LIST_DATA_DIR = os.path.join(VAR_PREFIX, 'lists') -LOG_DIR = os.path.join(VAR_PREFIX, 'logs') -LOCK_DIR = os.path.join(VAR_PREFIX, 'locks') -DATA_DIR = os.path.join(VAR_PREFIX, 'data') -SPAM_DIR = os.path.join(VAR_PREFIX, 'spam') -WRAPPER_DIR = os.path.join(EXEC_PREFIX, 'mail') -BIN_DIR = os.path.join(PREFIX, 'bin') -SCRIPTS_DIR = os.path.join(PREFIX, 'scripts') -TEMPLATE_DIR = os.path.join(PREFIX, 'templates') -MESSAGES_DIR = os.path.join(PREFIX, 'messages') -PUBLIC_ARCHIVE_FILE_DIR = os.path.join(VAR_PREFIX, 'archives', 'public') -PRIVATE_ARCHIVE_FILE_DIR = os.path.join(VAR_PREFIX, 'archives', 'private') - -# Directories used by the qrunner subsystem -QUEUE_DIR = os.path.join(VAR_PREFIX, 'qfiles') -INQUEUE_DIR = os.path.join(QUEUE_DIR, 'in') -OUTQUEUE_DIR = os.path.join(QUEUE_DIR, 'out') -CMDQUEUE_DIR = os.path.join(QUEUE_DIR, 'commands') -BOUNCEQUEUE_DIR = os.path.join(QUEUE_DIR, 'bounces') -NEWSQUEUE_DIR = os.path.join(QUEUE_DIR, 'news') -ARCHQUEUE_DIR = os.path.join(QUEUE_DIR, 'archive') -SHUNTQUEUE_DIR = os.path.join(QUEUE_DIR, 'shunt') -VIRGINQUEUE_DIR = os.path.join(QUEUE_DIR, 'virgin') -BADQUEUE_DIR = os.path.join(QUEUE_DIR, 'bad') -RETRYQUEUE_DIR = os.path.join(QUEUE_DIR, 'retry') -MAILDIR_DIR = os.path.join(QUEUE_DIR, 'maildir') - -# Other useful files -PIDFILE = os.path.join(DATA_DIR, 'master-qrunner.pid') -SITE_PW_FILE = os.path.join(DATA_DIR, 'adm.pw') -LISTCREATOR_PW_FILE = os.path.join(DATA_DIR, 'creator.pw') - # Import a bunch of version numbers from Version import * -MAILMAN_VERSION = 'GNU Mailman ' + VERSION # Vgg: Language descriptions and charsets dictionary, any new supported # language must have a corresponding entry here. Key is the name of the diff --git a/Mailman/Handlers/SMTPDirect.py b/Mailman/Handlers/SMTPDirect.py index e4fa6ec3b..60fb52359 100644 --- a/Mailman/Handlers/SMTPDirect.py +++ b/Mailman/Handlers/SMTPDirect.py @@ -38,19 +38,19 @@ from email.Header import Header from email.Utils import formataddr from Mailman import Errors -from Mailman import mm_cfg from Mailman import Utils from Mailman.Handlers import Decorate from Mailman.SafeDict import MsgSafeDict +from Mailman.configuration import config DOT = '.' log = logging.getLogger('mailman.smtp') flog = logging.getLogger('mailman.smtp-failure') -every_log = logging.getLogger('mailman.' + mm_cfg.SMTP_LOG_EVERY_MESSAGE[0]) -success_log = logging.getLogger('mailman.' + mm_cfg.SMTP_LOG_SUCCESS[0]) -refused_log = logging.getLogger('mailman.' + mm_cfg.SMTP_LOG_REFUSED[0]) -failure_log = logging.getLogger('mailman.' + mm_cfg.SMTP_LOG_EACH_FAILURE[0]) +every_log = logging.getLogger('mailman.' + config.SMTP_LOG_EVERY_MESSAGE[0]) +success_log = logging.getLogger('mailman.' + config.SMTP_LOG_SUCCESS[0]) +refused_log = logging.getLogger('mailman.' + config.SMTP_LOG_REFUSED[0]) +failure_log = logging.getLogger('mailman.' + config.SMTP_LOG_EACH_FAILURE[0]) @@ -61,8 +61,8 @@ class Connection: def __connect(self): self.__conn = smtplib.SMTP() - self.__conn.connect(mm_cfg.SMTPHOST, mm_cfg.SMTPPORT) - self.__numsessions = mm_cfg.SMTP_MAX_SESSIONS_PER_CONNECTION + self.__conn.connect(config.SMTPHOST, config.SMTPPORT) + self.__numsessions = config.SMTP_MAX_SESSIONS_PER_CONNECTION def sendmail(self, envsender, recips, msgtext): if self.__conn is None: @@ -118,10 +118,10 @@ def process(mlist, msg, msgdata): chunks = [[recip] for recip in recips] msgdata['personalize'] = 1 deliveryfunc = verpdeliver - elif mm_cfg.SMTP_MAX_RCPTS <= 0: + elif config.SMTP_MAX_RCPTS <= 0: chunks = [recips] else: - chunks = chunkify(recips, mm_cfg.SMTP_MAX_RCPTS) + chunks = chunkify(recips, config.SMTP_MAX_RCPTS) # See if this is an unshunted message for which some were undelivered if msgdata.has_key('undelivered'): chunks = msgdata['undelivered'] @@ -181,12 +181,12 @@ def process(mlist, msg, msgdata): # We have to use the copy() method because extended call syntax requires a # concrete dictionary object; it does not allow a generic mapping (XXX is # this still true in Python 2.3?). - if mm_cfg.SMTP_LOG_EVERY_MESSAGE: - every_log.info('%s', mm_cfg.SMTP_LOG_EVERY_MESSAGE[1] % d) + if config.SMTP_LOG_EVERY_MESSAGE: + every_log.info('%s', config.SMTP_LOG_EVERY_MESSAGE[1] % d) if refused: - if mm_cfg.SMTP_LOG_REFUSED: - refused_log.info('%s', mm_cfg.SMTP_LOG_REFUSED[1] % d) + if config.SMTP_LOG_REFUSED: + refused_log.info('%s', config.SMTP_LOG_REFUSED[1] % d) elif msgdata.get('tolist'): # Log the successful post, but only if it really was a post to the @@ -194,8 +194,8 @@ def process(mlist, msg, msgdata): # -request addrs should never get here. BAW: it may be useful to log # the other messages, but in that case, we should probably have a # separate configuration variable to control that. - if mm_cfg.SMTP_LOG_SUCCESS: - success_log.info('%s', mm_cfg.SMTP_LOG_SUCCESS[1] % d) + if config.SMTP_LOG_SUCCESS: + success_log.info('%s', config.SMTP_LOG_SUCCESS[1] % d) # Process any failed deliveries. tempfailures = [] @@ -217,11 +217,11 @@ def process(mlist, msg, msgdata): # Deal with persistent transient failures by queuing them up for # future delivery. TBD: this could generate lots of log entries! tempfailures.append(recip) - if mm_cfg.SMTP_LOG_EACH_FAILURE: + if config.SMTP_LOG_EACH_FAILURE: d.update({'recipient': recip, 'failcode' : code, 'failmsg' : smtpmsg}) - failure_log.info('%s', mm_cfg.SMTP_LOG_EACH_FAILURE[1] % d) + failure_log.info('%s', config.SMTP_LOG_EACH_FAILURE[1] % d) # Return the results if tempfailures or permfailures: raise Errors.SomeRecipientsFailed(tempfailures, permfailures) @@ -300,7 +300,7 @@ def verpdeliver(mlist, msg, msgdata, envsender, failures, conn): 'mailbox': rmailbox, 'host' : DOT.join(rdomain), } - envsender = '%s@%s' % ((mm_cfg.VERP_FORMAT % d), DOT.join(bdomain)) + envsender = '%s@%s' % ((config.VERP_FORMAT % d), DOT.join(bdomain)) if mlist.personalize == 2: # When fully personalizing, we want the To address to point to the # recipient, not to the mailing list diff --git a/Mailman/MTA/Utils.py b/Mailman/MTA/Utils.py index 14562de66..ef3df0cb8 100644 --- a/Mailman/MTA/Utils.py +++ b/Mailman/MTA/Utils.py @@ -1,25 +1,26 @@ -# Copyright (C) 2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2006 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. """Utilities for list creation/deletion hooks.""" import os import pwd -from Mailman import mm_cfg +from Mailman.configuration import config @@ -35,7 +36,7 @@ def getusername(): def _makealiases_mailprog(listname): - wrapper = os.path.join(mm_cfg.WRAPPER_DIR, 'mailman') + wrapper = os.path.join(config.WRAPPER_DIR, 'mailman') # Most of the list alias extensions are quite regular. I.e. if the # message is delivered to listname-foobar, it will be filtered to a # program called foobar. There are two exceptions: @@ -57,7 +58,7 @@ def _makealiases_mailprog(listname): def _makealiases_maildir(listname): - maildir = mm_cfg.MAILDIR_DIR + maildir = config.MAILDIR_DIR if not maildir.endswith('/'): maildir += '/' # Deliver everything using maildir style. This way there's no mail @@ -73,7 +74,9 @@ def _makealiases_maildir(listname): -if mm_cfg.USE_MAILDIR: +# XXX This won't work if Mailman.MTA.Utils is imported before the +# configuration is loaded. +if config.USE_MAILDIR: makealiases = _makealiases_maildir else: makealiases = _makealiases_mailprog diff --git a/Mailman/MailList.py b/Mailman/MailList.py index 04fb0f22c..ecf449246 100644 --- a/Mailman/MailList.py +++ b/Mailman/MailList.py @@ -45,8 +45,8 @@ from email.Utils import getaddresses, formataddr, parseaddr from Mailman import Errors from Mailman import LockFile from Mailman import Utils -from Mailman import mm_cfg from Mailman.UserDesc import UserDesc +from Mailman.configuration import config # Base classes from Mailman import Pending @@ -196,19 +196,19 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, return self.getListAddress('owner') def GetRequestEmail(self, cookie=''): - if mm_cfg.VERP_CONFIRMATIONS and cookie: + if config.VERP_CONFIRMATIONS and cookie: return self.GetConfirmEmail(cookie) else: return self.getListAddress('request') def GetConfirmEmail(self, cookie): - return mm_cfg.VERP_CONFIRM_FORMAT % { + return config.VERP_CONFIRM_FORMAT % { 'addr' : '%s-confirm' % self.internal_name(), 'cookie': cookie, } + '@' + self.host_name def GetConfirmJoinSubject(self, listname, cookie): - if mm_cfg.VERP_CONFIRMATIONS and cookie: + if config.VERP_CONFIRMATIONS and cookie: cset = i18n.get_translation().charset() or \ Utils.GetCharSet(self.preferred_language) subj = Header( @@ -219,7 +219,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, return 'confirm ' + cookie def GetConfirmLeaveSubject(self, listname, cookie): - if mm_cfg.VERP_CONFIRMATIONS and cookie: + if config.VERP_CONFIRMATIONS and cookie: cset = i18n.get_translation().charset() or \ Utils.GetCharSet(self.preferred_language) subj = Header( @@ -269,8 +269,8 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # need to reload, otherwise... we do. self.__timestamp = 0 self.__lock = LockFile.LockFile( - os.path.join(mm_cfg.LOCK_DIR, name or '<site>') + '.lock', - lifetime=mm_cfg.LIST_LOCK_LIFETIME) + os.path.join(config.LOCK_DIR, name or '<site>') + '.lock', + lifetime=config.LIST_LOCK_LIFETIME) self._internal_name = name if name: self._full_path = Site.get_listpath(name) @@ -299,7 +299,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # Must save this state, even though it isn't configurable self.volume = 1 self.members = {} # self.digest_members is initted in mm_digest - self.data_version = mm_cfg.DATA_FILE_VERSION + self.data_version = config.DATA_FILE_VERSION self.last_post_time = 0 self.post_id = 1. # A float so it never has a chance to overflow. @@ -307,76 +307,76 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, self.language = {} self.usernames = {} self.passwords = {} - self.new_member_options = mm_cfg.DEFAULT_NEW_MEMBER_OPTIONS + self.new_member_options = config.DEFAULT_NEW_MEMBER_OPTIONS # This stuff is configurable self.respond_to_post_requests = 1 - self.advertised = mm_cfg.DEFAULT_LIST_ADVERTISED - self.max_num_recipients = mm_cfg.DEFAULT_MAX_NUM_RECIPIENTS - self.max_message_size = mm_cfg.DEFAULT_MAX_MESSAGE_SIZE + self.advertised = config.DEFAULT_LIST_ADVERTISED + self.max_num_recipients = config.DEFAULT_MAX_NUM_RECIPIENTS + self.max_message_size = config.DEFAULT_MAX_MESSAGE_SIZE # See the note in Defaults.py concerning DEFAULT_HOST_NAME # vs. DEFAULT_EMAIL_HOST. - self.host_name = mm_cfg.DEFAULT_HOST_NAME or mm_cfg.DEFAULT_EMAIL_HOST + self.host_name = config.DEFAULT_HOST_NAME or config.DEFAULT_EMAIL_HOST self.web_page_url = ( - mm_cfg.DEFAULT_URL or - mm_cfg.DEFAULT_URL_PATTERN % mm_cfg.DEFAULT_URL_HOST) + config.DEFAULT_URL or + config.DEFAULT_URL_PATTERN % config.DEFAULT_URL_HOST) self.owner = [admin] self.moderator = [] - self.reply_goes_to_list = mm_cfg.DEFAULT_REPLY_GOES_TO_LIST + self.reply_goes_to_list = config.DEFAULT_REPLY_GOES_TO_LIST self.reply_to_address = '' - self.first_strip_reply_to = mm_cfg.DEFAULT_FIRST_STRIP_REPLY_TO - self.admin_immed_notify = mm_cfg.DEFAULT_ADMIN_IMMED_NOTIFY + self.first_strip_reply_to = config.DEFAULT_FIRST_STRIP_REPLY_TO + self.admin_immed_notify = config.DEFAULT_ADMIN_IMMED_NOTIFY self.admin_notify_mchanges = \ - mm_cfg.DEFAULT_ADMIN_NOTIFY_MCHANGES + config.DEFAULT_ADMIN_NOTIFY_MCHANGES self.require_explicit_destination = \ - mm_cfg.DEFAULT_REQUIRE_EXPLICIT_DESTINATION - self.acceptable_aliases = mm_cfg.DEFAULT_ACCEPTABLE_ALIASES - self.umbrella_list = mm_cfg.DEFAULT_UMBRELLA_LIST + config.DEFAULT_REQUIRE_EXPLICIT_DESTINATION + self.acceptable_aliases = config.DEFAULT_ACCEPTABLE_ALIASES + self.umbrella_list = config.DEFAULT_UMBRELLA_LIST self.umbrella_member_suffix = \ - mm_cfg.DEFAULT_UMBRELLA_MEMBER_ADMIN_SUFFIX - self.send_reminders = mm_cfg.DEFAULT_SEND_REMINDERS - self.send_welcome_msg = mm_cfg.DEFAULT_SEND_WELCOME_MSG - self.send_goodbye_msg = mm_cfg.DEFAULT_SEND_GOODBYE_MSG + config.DEFAULT_UMBRELLA_MEMBER_ADMIN_SUFFIX + self.send_reminders = config.DEFAULT_SEND_REMINDERS + self.send_welcome_msg = config.DEFAULT_SEND_WELCOME_MSG + self.send_goodbye_msg = config.DEFAULT_SEND_GOODBYE_MSG self.bounce_matching_headers = \ - mm_cfg.DEFAULT_BOUNCE_MATCHING_HEADERS + config.DEFAULT_BOUNCE_MATCHING_HEADERS self.header_filter_rules = [] - self.anonymous_list = mm_cfg.DEFAULT_ANONYMOUS_LIST + self.anonymous_list = config.DEFAULT_ANONYMOUS_LIST internalname = self.internal_name() self.real_name = internalname[0].upper() + internalname[1:] self.description = '' self.info = '' self.welcome_msg = '' self.goodbye_msg = '' - self.subscribe_policy = mm_cfg.DEFAULT_SUBSCRIBE_POLICY - self.subscribe_auto_approval = mm_cfg.DEFAULT_SUBSCRIBE_AUTO_APPROVAL - self.unsubscribe_policy = mm_cfg.DEFAULT_UNSUBSCRIBE_POLICY - self.private_roster = mm_cfg.DEFAULT_PRIVATE_ROSTER - self.obscure_addresses = mm_cfg.DEFAULT_OBSCURE_ADDRESSES - self.admin_member_chunksize = mm_cfg.DEFAULT_ADMIN_MEMBER_CHUNKSIZE - self.administrivia = mm_cfg.DEFAULT_ADMINISTRIVIA - self.preferred_language = mm_cfg.DEFAULT_SERVER_LANGUAGE + self.subscribe_policy = config.DEFAULT_SUBSCRIBE_POLICY + self.subscribe_auto_approval = config.DEFAULT_SUBSCRIBE_AUTO_APPROVAL + self.unsubscribe_policy = config.DEFAULT_UNSUBSCRIBE_POLICY + self.private_roster = config.DEFAULT_PRIVATE_ROSTER + self.obscure_addresses = config.DEFAULT_OBSCURE_ADDRESSES + self.admin_member_chunksize = config.DEFAULT_ADMIN_MEMBER_CHUNKSIZE + self.administrivia = config.DEFAULT_ADMINISTRIVIA + self.preferred_language = config.DEFAULT_SERVER_LANGUAGE self.available_languages = [] self.include_rfc2369_headers = 1 self.include_list_post_header = 1 - self.filter_mime_types = mm_cfg.DEFAULT_FILTER_MIME_TYPES - self.pass_mime_types = mm_cfg.DEFAULT_PASS_MIME_TYPES + self.filter_mime_types = config.DEFAULT_FILTER_MIME_TYPES + self.pass_mime_types = config.DEFAULT_PASS_MIME_TYPES self.filter_filename_extensions = \ - mm_cfg.DEFAULT_FILTER_FILENAME_EXTENSIONS - self.pass_filename_extensions = mm_cfg.DEFAULT_PASS_FILENAME_EXTENSIONS - self.filter_content = mm_cfg.DEFAULT_FILTER_CONTENT - self.collapse_alternatives = mm_cfg.DEFAULT_COLLAPSE_ALTERNATIVES + config.DEFAULT_FILTER_FILENAME_EXTENSIONS + self.pass_filename_extensions = config.DEFAULT_PASS_FILENAME_EXTENSIONS + self.filter_content = config.DEFAULT_FILTER_CONTENT + self.collapse_alternatives = config.DEFAULT_COLLAPSE_ALTERNATIVES self.convert_html_to_plaintext = \ - mm_cfg.DEFAULT_CONVERT_HTML_TO_PLAINTEXT - self.filter_action = mm_cfg.DEFAULT_FILTER_ACTION + config.DEFAULT_CONVERT_HTML_TO_PLAINTEXT + self.filter_action = config.DEFAULT_FILTER_ACTION # Analogs to these are initted in Digester.InitVars - self.nondigestable = mm_cfg.DEFAULT_NONDIGESTABLE + self.nondigestable = config.DEFAULT_NONDIGESTABLE self.personalize = 0 # New sender-centric moderation (privacy) options self.default_member_moderation = \ - mm_cfg.DEFAULT_DEFAULT_MEMBER_MODERATION + config.DEFAULT_DEFAULT_MEMBER_MODERATION # Emergency moderation bit self.emergency = 0 - # This really ought to default to mm_cfg.HOLD, but that doesn't work + # This really ought to default to config.HOLD, but that doesn't work # with the current GUI description model. So, 0==Hold, 1==Reject, # 2==Discard self.member_moderation_action = 0 @@ -385,8 +385,8 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, self.hold_these_nonmembers = [] self.reject_these_nonmembers = [] self.discard_these_nonmembers = [] - self.forward_auto_discards = mm_cfg.DEFAULT_FORWARD_AUTO_DISCARDS - self.generic_nonmember_action = mm_cfg.DEFAULT_GENERIC_NONMEMBER_ACTION + self.forward_auto_discards = config.DEFAULT_FORWARD_AUTO_DISCARDS + self.generic_nonmember_action = config.DEFAULT_GENERIC_NONMEMBER_ACTION self.nonmember_rejection_notice = '' # Ban lists self.ban_list = [] @@ -403,9 +403,9 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # These need to come near the bottom because they're dependent on # other settings. - self.subject_prefix = mm_cfg.DEFAULT_SUBJECT_PREFIX % self.__dict__ - self.msg_header = mm_cfg.DEFAULT_MSG_HEADER - self.msg_footer = mm_cfg.DEFAULT_MSG_FOOTER + self.subject_prefix = config.DEFAULT_SUBJECT_PREFIX % self.__dict__ + self.msg_header = config.DEFAULT_MSG_HEADER + self.msg_footer = config.DEFAULT_MSG_FOOTER # Set this to Never if the list's preferred language uses us-ascii, # otherwise set it to As Needed if Utils.GetCharSet(self.preferred_language) == 'us-ascii': @@ -413,9 +413,9 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, else: self.encode_ascii_prefixes = 2 # scrub regular delivery - self.scrub_nondigest = mm_cfg.DEFAULT_SCRUB_NONDIGEST + self.scrub_nondigest = config.DEFAULT_SCRUB_NONDIGEST # automatic discarding - self.max_days_to_hold = mm_cfg.DEFAULT_MAX_DAYS_TO_HOLD + self.max_days_to_hold = config.DEFAULT_MAX_DAYS_TO_HOLD # @@ -425,17 +425,17 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, class CategoryDict(UserDict): def __init__(self): UserDict.__init__(self) - self.keysinorder = mm_cfg.ADMIN_CATEGORIES[:] + self.keysinorder = config.ADMIN_CATEGORIES[:] def keys(self): return self.keysinorder def items(self): items = [] - for k in mm_cfg.ADMIN_CATEGORIES: + for k in config.ADMIN_CATEGORIES: items.append((k, self.data[k])) return items def values(self): values = [] - for k in mm_cfg.ADMIN_CATEGORIES: + for k in config.ADMIN_CATEGORIES: values.append(self.data[k]) return values @@ -476,7 +476,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # However, most scripts already catch MMBadEmailError as exceptions on # the admin's email address, so transform the exception. if emailhost is None: - emailhost = mm_cfg.DEFAULT_EMAIL_HOST + emailhost = config.DEFAULT_EMAIL_HOST postingaddr = '%s@%s' % (name, emailhost) try: Utils.ValidateEmail(postingaddr) @@ -515,7 +515,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # Use a binary format... it's more efficient. cPickle.dump(dict, fp, 1) fp.flush() - if mm_cfg.SYNC_AFTER_WRITE: + if config.SYNC_AFTER_WRITE: os.fsync(fp.fileno()) fp.close() except IOError, e: @@ -697,7 +697,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # def CheckVersion(self, stored_state): """Auto-update schema if necessary.""" - if self.data_version >= mm_cfg.DATA_FILE_VERSION: + if self.data_version >= config.DATA_FILE_VERSION: return # Initialize any new variables self.InitVars() @@ -712,7 +712,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, try: from versions import Update Update(self, stored_state) - self.data_version = mm_cfg.DATA_FILE_VERSION + self.data_version = config.DATA_FILE_VERSION self.Save() finally: if not waslocked: @@ -725,8 +725,8 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # URL is empty; substitute faulty value with (hopefully sane) # default. Note that DEFAULT_URL is obsolete. self.web_page_url = ( - mm_cfg.DEFAULT_URL or - mm_cfg.DEFAULT_URL_PATTERN % mm_cfg.DEFAULT_URL_HOST) + config.DEFAULT_URL or + config.DEFAULT_URL_PATTERN % config.DEFAULT_URL_HOST) if self.web_page_url and self.web_page_url[-1] <> '/': self.web_page_url = self.web_page_url + '/' # Legacy reply_to_address could be an illegal value. We now verify @@ -958,9 +958,9 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # Do the actual addition self.addNewMember(email, realname=name, digest=digest, password=password, language=lang) - self.setMemberOption(email, mm_cfg.DisableMime, + self.setMemberOption(email, config.DisableMime, 1 - self.mime_is_default_digest) - self.setMemberOption(email, mm_cfg.Moderate, + self.setMemberOption(email, config.Moderate, self.default_member_moderation) # Now send and log results if digest: @@ -1283,17 +1283,17 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, if approved is not None: # Does it match the list password? Note that we purposefully # do not allow the site password here. - if self.Authenticate([mm_cfg.AuthListAdmin, - mm_cfg.AuthListModerator], - approved) <> mm_cfg.UnAuthorized: - action = mm_cfg.APPROVE + if self.Authenticate([config.AuthListAdmin, + config.AuthListModerator], + approved) <> config.UnAuthorized: + action = config.APPROVE else: # The password didn't match. Re-pend the message and # inform the list moderators about the problem. self.pend_repend(cookie, rec) raise Errors.MMBadPasswordError else: - action = mm_cfg.DISCARD + action = config.DISCARD try: self.HandleRequest(id, action) except KeyError: @@ -1459,7 +1459,7 @@ bad regexp in bounce_matching_header line: %s lang = self.preferred_language i18n.set_language(lang) # No limit - if mm_cfg.MAX_AUTORESPONSES_PER_DAY == 0: + if config.MAX_AUTORESPONSES_PER_DAY == 0: return 1 today = time.localtime()[:3] info = self.hold_and_cmd_autoresponses.get(sender) @@ -1473,7 +1473,7 @@ bad regexp in bounce_matching_header line: %s # They've already hit the limit for today. vlog.info('-request/hold autoresponse discarded for: %s', sender) return 0 - if count >= mm_cfg.MAX_AUTORESPONSES_PER_DAY: + if count >= config.MAX_AUTORESPONSES_PER_DAY: vlog.info('-request/hold autoresponse limit hit for: %s', sender) self.hold_and_cmd_autoresponses[sender] = (today, -1) # Send this notification message instead @@ -1544,8 +1544,8 @@ bad regexp in bounce_matching_header line: %s # language support to the list, then the general admin page may have a # blank field where the list owner is supposed to chose the list's # preferred language. - if mm_cfg.DEFAULT_SERVER_LANGUAGE not in langs: - langs.append(mm_cfg.DEFAULT_SERVER_LANGUAGE) + if config.DEFAULT_SERVER_LANGUAGE not in langs: + langs.append(config.DEFAULT_SERVER_LANGUAGE) # When testing, it's possible we've disabled a language, so just # filter things out so we don't get tracebacks. - return [lang for lang in langs if mm_cfg.LC_DESCRIPTIONS.has_key(lang)] + return [lang for lang in langs if config.LC_DESCRIPTIONS.has_key(lang)] diff --git a/Mailman/Message.py b/Mailman/Message.py index 7216d7c9a..b39bcb3fe 100644 --- a/Mailman/Message.py +++ b/Mailman/Message.py @@ -29,8 +29,8 @@ import email.Utils from email.Charset import Charset from email.Header import Header -from Mailman import mm_cfg from Mailman import Utils +from Mailman.configuration import config COMMASPACE = ', ' @@ -113,7 +113,7 @@ class Message(email.Message.Message): This method differs from get_senders() in that it returns one and only one address, and uses a different search order. """ - senderfirst = mm_cfg.USE_ENVELOPE_SENDER + senderfirst = config.USE_ENVELOPE_SENDER if use_envelope is not None: senderfirst = use_envelope if senderfirst: @@ -165,7 +165,7 @@ class Message(email.Message.Message): names without the trailing colon. """ if headers is None: - headers = mm_cfg.SENDER_HEADERS + headers = config.SENDER_HEADERS pairs = [] for h in headers: if h is None: @@ -245,7 +245,7 @@ class UserNotification(Message): def _enqueue(self, mlist, **_kws): # Not imported at module scope to avoid import loop from Mailman.Queue.sbcache import get_switchboard - virginq = get_switchboard(mm_cfg.VIRGINQUEUE_DIR) + virginq = get_switchboard(config.VIRGINQUEUE_DIR) # The message metadata better have a `recip' attribute virginq.enqueue(self, listname = mlist.internal_name(), @@ -276,7 +276,7 @@ class OwnerNotification(UserNotification): def _enqueue(self, mlist, **_kws): # Not imported at module scope to avoid import loop from Mailman.Queue.sbcache import get_switchboard - virginq = get_switchboard(mm_cfg.VIRGINQUEUE_DIR) + virginq = get_switchboard(config.VIRGINQUEUE_DIR) # The message metadata better have a `recip' attribute virginq.enqueue(self, listname = mlist.internal_name(), diff --git a/Mailman/Queue/ArchRunner.py b/Mailman/Queue/ArchRunner.py index 627145372..f3c90f5c3 100644 --- a/Mailman/Queue/ArchRunner.py +++ b/Mailman/Queue/ArchRunner.py @@ -1,4 +1,4 @@ -# Copyright (C) 2000,2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 2000-2006 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -12,21 +12,22 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. """Archive queue runner.""" import time from email.Utils import parsedate_tz, mktime_tz, formatdate -from Mailman import mm_cfg from Mailman import LockFile from Mailman.Queue.Runner import Runner +from Mailman.configuration import config class ArchRunner(Runner): - QDIR = mm_cfg.ARCHQUEUE_DIR + QDIR = config.ARCHQUEUE_DIR def _dispose(self, mlist, msg, msgdata): # Support clobber_date, i.e. setting the date in the archive to the @@ -37,9 +38,9 @@ class ArchRunner(Runner): receivedtime = formatdate(msgdata['received_time']) if not originaldate: clobber = 1 - elif mm_cfg.ARCHIVER_CLOBBER_DATE_POLICY == 1: + elif config.ARCHIVER_CLOBBER_DATE_POLICY == 1: clobber = 1 - elif mm_cfg.ARCHIVER_CLOBBER_DATE_POLICY == 2: + elif config.ARCHIVER_CLOBBER_DATE_POLICY == 2: # what's the timestamp on the original message? tup = parsedate_tz(originaldate) now = time.time() @@ -47,7 +48,7 @@ class ArchRunner(Runner): if not tup: clobber = 1 elif abs(now - mktime_tz(tup)) > \ - mm_cfg.ARCHIVER_ALLOWABLE_SANE_DATE_SKEW: + config.ARCHIVER_ALLOWABLE_SANE_DATE_SKEW: clobber = 1 except (ValueError, OverflowError): # The likely cause of this is that the year in the Date: field @@ -65,7 +66,7 @@ class ArchRunner(Runner): msg['X-List-Received-Date'] = receivedtime # Now try to get the list lock try: - mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT) + mlist.Lock(timeout=config.LIST_LOCK_TIMEOUT) except LockFile.TimeOutError: # oh well, try again later return 1 diff --git a/Mailman/Queue/BounceRunner.py b/Mailman/Queue/BounceRunner.py index 0063635ac..93bee2062 100644 --- a/Mailman/Queue/BounceRunner.py +++ b/Mailman/Queue/BounceRunner.py @@ -28,13 +28,13 @@ from email.MIMEText import MIMEText from email.Utils import parseaddr from Mailman import LockFile -from Mailman import mm_cfg from Mailman import Utils from Mailman.Bouncers import BouncerAPI -from Mailman.i18n import _ from Mailman.Message import UserNotification from Mailman.Queue.Runner import Runner from Mailman.Queue.sbcache import get_switchboard +from Mailman.configuration import config +from Mailman.i18n import _ COMMASPACE = ', ' @@ -78,10 +78,10 @@ class BounceMixin: # their lists. So now we ignore site list bounces. Ce La Vie for # password reminder bounces. self._bounce_events_file = os.path.join( - mm_cfg.DATA_DIR, 'bounce-events-%05d.pck' % os.getpid()) + config.DATA_DIR, 'bounce-events-%05d.pck' % os.getpid()) self._bounce_events_fp = None self._bouncecnt = 0 - self._nextaction = time.time() + mm_cfg.REGISTER_BOUNCES_EVERY + self._nextaction = time.time() + config.REGISTER_BOUNCES_EVERY def _queue_bounces(self, listname, addrs, msg): today = time.localtime()[:3] @@ -137,7 +137,7 @@ class BounceMixin: if self._nextaction > now or self._bouncecnt == 0: return # Let's go ahead and register the bounces we've got stored up - self._nextaction = now + mm_cfg.REGISTER_BOUNCES_EVERY + self._nextaction = now + config.REGISTER_BOUNCES_EVERY self._register_bounces() def _probe_bounce(self, mlist, token): @@ -158,7 +158,7 @@ class BounceMixin: class BounceRunner(Runner, BounceMixin): - QDIR = mm_cfg.BOUNCEQUEUE_DIR + QDIR = config.BOUNCEQUEUE_DIR def __init__(self, slice=None, numslices=1): Runner.__init__(self, slice, numslices) @@ -167,7 +167,7 @@ class BounceRunner(Runner, BounceMixin): def _dispose(self, mlist, msg, msgdata): # Make sure we have the most up-to-date state mlist.Load() - outq = get_switchboard(mm_cfg.OUTQUEUE_DIR) + outq = get_switchboard(config.OUTQUEUE_DIR) # There are a few possibilities here: # # - the message could have been VERP'd in which case, we know exactly @@ -245,7 +245,7 @@ def verp_bounce(mlist, msg): to = parseaddr(field)[1] if not to: continue # empty header - mo = re.search(mm_cfg.VERP_REGEXP, to) + mo = re.search(config.VERP_REGEXP, to) if not mo: continue # no match of regexp try: @@ -255,7 +255,7 @@ def verp_bounce(mlist, msg): addr = '%s@%s' % mo.group('mailbox', 'host') except IndexError: elog.error("VERP_REGEXP doesn't yield the right match groups: %s", - mm_cfg.VERP_REGEXP) + config.VERP_REGEXP) return [] return [addr] @@ -276,7 +276,7 @@ def verp_probe(mlist, msg): to = parseaddr(field)[1] if not to: continue # empty header - mo = re.search(mm_cfg.VERP_PROBE_REGEXP, to) + mo = re.search(config.VERP_PROBE_REGEXP, to) if not mo: continue # no match of regexp try: @@ -290,7 +290,7 @@ def verp_probe(mlist, msg): except IndexError: elog.error( "VERP_PROBE_REGEXP doesn't yield the right match groups: %s", - mm_cfg.VERP_PROBE_REGEXP) + config.VERP_PROBE_REGEXP) return None diff --git a/Mailman/Queue/CommandRunner.py b/Mailman/Queue/CommandRunner.py index fe30a3cd3..2d1fc1031 100644 --- a/Mailman/Queue/CommandRunner.py +++ b/Mailman/Queue/CommandRunner.py @@ -33,11 +33,11 @@ from email.MIMEText import MIMEText from Mailman import LockFile from Mailman import Message -from Mailman import mm_cfg from Mailman import Utils from Mailman.Handlers import Replybot -from Mailman.i18n import _ from Mailman.Queue.Runner import Runner +from Mailman.configuration import config +from Mailman.i18n import _ NL = '\n' @@ -88,8 +88,8 @@ class Results: assert isinstance(body, basestring) lines = body.splitlines() # Use no more lines than specified - self.commands.extend(lines[:mm_cfg.DEFAULT_MAIL_COMMANDS_MAX_LINES]) - self.ignored.extend(lines[mm_cfg.DEFAULT_MAIL_COMMANDS_MAX_LINES:]) + self.commands.extend(lines[:config.DEFAULT_MAIL_COMMANDS_MAX_LINES]) + self.ignored.extend(lines[config.DEFAULT_MAIL_COMMANDS_MAX_LINES:]) def process(self): # Now, process each line until we find an error. The first @@ -192,7 +192,7 @@ To obtain instructions, send a message containing just the word "help". class CommandRunner(Runner): - QDIR = mm_cfg.CMDQUEUE_DIR + QDIR = config.CMDQUEUE_DIR def _dispose(self, mlist, msg, msgdata): # The policy here is similar to the Replybot policy. If a message has @@ -217,7 +217,7 @@ class CommandRunner(Runner): # locked. Still, it's more convenient to lock it here and now and # deal with lock failures in one place. try: - mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT) + mlist.Lock(timeout=config.LIST_LOCK_TIMEOUT) except LockFile.TimeOutError: # Oh well, try again later return True @@ -232,7 +232,7 @@ class CommandRunner(Runner): elif msgdata.get('toleave'): res.do_command('leave') elif msgdata.get('toconfirm'): - mo = re.match(mm_cfg.VERP_CONFIRM_REGEXP, msg.get('to', '')) + mo = re.match(config.VERP_CONFIRM_REGEXP, msg.get('to', '')) if mo: res.do_command('confirm', (mo.group('cookie'),)) res.send_response() diff --git a/Mailman/Queue/IncomingRunner.py b/Mailman/Queue/IncomingRunner.py index 19a315040..332c9f129 100644 --- a/Mailman/Queue/IncomingRunner.py +++ b/Mailman/Queue/IncomingRunner.py @@ -103,8 +103,8 @@ from cStringIO import StringIO from Mailman import Errors from Mailman import LockFile -from Mailman import mm_cfg from Mailman.Queue.Runner import Runner +from Mailman.configuration import config log = logging.getLogger('mailman.error') vlog = logging.getLogger('mailman.vette') @@ -112,12 +112,12 @@ vlog = logging.getLogger('mailman.vette') class IncomingRunner(Runner): - QDIR = mm_cfg.INQUEUE_DIR + QDIR = config.INQUEUE_DIR def _dispose(self, mlist, msg, msgdata): # Try to get the list lock. try: - mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT) + mlist.Lock(timeout=config.LIST_LOCK_TIMEOUT) except LockFile.TimeOutError: # Oh well, try again later return 1 @@ -146,7 +146,7 @@ class IncomingRunner(Runner): # flows through the pipeline will empty it out! return msgdata.get('pipeline', getattr(mlist, 'pipeline', - mm_cfg.GLOBAL_PIPELINE))[:] + config.GLOBAL_PIPELINE))[:] def _dopipeline(self, mlist, msg, msgdata, pipeline): while pipeline: diff --git a/Mailman/Queue/MaildirRunner.py b/Mailman/Queue/MaildirRunner.py index 97b71c7ef..9e5e08be5 100644 --- a/Mailman/Queue/MaildirRunner.py +++ b/Mailman/Queue/MaildirRunner.py @@ -57,11 +57,11 @@ import logging from email.Parser import Parser from email.Utils import parseaddr -from Mailman import mm_cfg from Mailman import Utils from Mailman.Message import Message from Mailman.Queue.Runner import Runner from Mailman.Queue.sbcache import get_switchboard +from Mailman.configuration import config # We only care about the listname and the subq as in listname@ or # listname-request@ @@ -88,8 +88,8 @@ class MaildirRunner(Runner): # Don't call the base class constructor, but build enough of the # underlying attributes to use the base class's implementation. self._stop = 0 - self._dir = os.path.join(mm_cfg.MAILDIR_DIR, 'new') - self._cur = os.path.join(mm_cfg.MAILDIR_DIR, 'cur') + self._dir = os.path.join(config.MAILDIR_DIR, 'new') + self._cur = os.path.join(config.MAILDIR_DIR, 'cur') self._parser = Parser(Message) def _oneloop(self): @@ -150,29 +150,29 @@ class MaildirRunner(Runner): msgdata = {'listname': listname} # -admin is deprecated if subq in ('bounces', 'admin'): - queue = get_switchboard(mm_cfg.BOUNCEQUEUE_DIR) + queue = get_switchboard(config.BOUNCEQUEUE_DIR) elif subq == 'confirm': msgdata['toconfirm'] = 1 - queue = get_switchboard(mm_cfg.CMDQUEUE_DIR) + queue = get_switchboard(config.CMDQUEUE_DIR) elif subq in ('join', 'subscribe'): msgdata['tojoin'] = 1 - queue = get_switchboard(mm_cfg.CMDQUEUE_DIR) + queue = get_switchboard(config.CMDQUEUE_DIR) elif subq in ('leave', 'unsubscribe'): msgdata['toleave'] = 1 - queue = get_switchboard(mm_cfg.CMDQUEUE_DIR) + queue = get_switchboard(config.CMDQUEUE_DIR) elif subq == 'owner': msgdata.update({ 'toowner': 1, 'envsender': Utils.get_site_email(extra='bounces'), - 'pipeline': mm_cfg.OWNER_PIPELINE, + 'pipeline': config.OWNER_PIPELINE, }) - queue = get_switchboard(mm_cfg.INQUEUE_DIR) + queue = get_switchboard(config.INQUEUE_DIR) elif subq is None: msgdata['tolist'] = 1 - queue = get_switchboard(mm_cfg.INQUEUE_DIR) + queue = get_switchboard(config.INQUEUE_DIR) elif subq == 'request': msgdata['torequest'] = 1 - queue = get_switchboard(mm_cfg.CMDQUEUE_DIR) + queue = get_switchboard(config.CMDQUEUE_DIR) else: log.error('Unknown sub-queue: %s', subq) os.rename(dstname, xdstname) diff --git a/Mailman/Queue/NewsRunner.py b/Mailman/Queue/NewsRunner.py index 7a71f5bf2..31e88da9a 100644 --- a/Mailman/Queue/NewsRunner.py +++ b/Mailman/Queue/NewsRunner.py @@ -12,7 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. """NNTP queue runner.""" @@ -27,9 +28,9 @@ from email.Utils import getaddresses COMMASPACE = ', ' -from Mailman import mm_cfg from Mailman import Utils from Mailman.Queue.Runner import Runner +from Mailman.configuration import config log = logging.getLogger('mailman.error') @@ -48,7 +49,7 @@ mcre = re.compile(r""" class NewsRunner(Runner): - QDIR = mm_cfg.NEWSQUEUE_DIR + QDIR = config.NEWSQUEUE_DIR def _dispose(self, mlist, msg, msgdata): # Make sure we have the most up-to-date state @@ -64,8 +65,8 @@ class NewsRunner(Runner): nntp_host, nntp_port = Utils.nntpsplit(mlist.nntp_host) conn = nntplib.NNTP(nntp_host, nntp_port, readermode=True, - user=mm_cfg.NNTP_USERNAME, - password=mm_cfg.NNTP_PASSWORD) + user=config.NNTP_USERNAME, + password=config.NNTP_PASSWORD) conn.post(fp) except nntplib.error_temp, e: log.error('(NNTPDirect) NNTP error for list "%s": %s', @@ -146,9 +147,9 @@ def prepare_message(mlist, msg, msgdata): # woon't completely sanitize the message, but it will eliminate the bulk # of the rejections based on message headers. The NNTP server may still # reject the message because of other problems. - for header in mm_cfg.NNTP_REMOVE_HEADERS: + for header in config.NNTP_REMOVE_HEADERS: del msg[header] - for header, rewrite in mm_cfg.NNTP_REWRITE_DUPLICATE_HEADERS: + for header, rewrite in config.NNTP_REWRITE_DUPLICATE_HEADERS: values = msg.get_all(header, []) if len(values) < 2: # We only care about duplicates diff --git a/Mailman/Queue/OutgoingRunner.py b/Mailman/Queue/OutgoingRunner.py index 3d4575ed2..351f24da6 100644 --- a/Mailman/Queue/OutgoingRunner.py +++ b/Mailman/Queue/OutgoingRunner.py @@ -12,7 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. """Outgoing queue runner.""" @@ -27,10 +28,10 @@ import logging from Mailman import Errors from Mailman import LockFile from Mailman import Message -from Mailman import mm_cfg from Mailman.Queue.BounceRunner import BounceMixin from Mailman.Queue.Runner import Runner from Mailman.Queue.Switchboard import Switchboard +from Mailman.configuration import config # This controls how often _doperiodic() will try to deal with deferred # permanent failures. It is a count of calls to _doperiodic() @@ -41,20 +42,20 @@ log = logging.getLogger('mailman.error') class OutgoingRunner(Runner, BounceMixin): - QDIR = mm_cfg.OUTQUEUE_DIR + QDIR = config.OUTQUEUE_DIR def __init__(self, slice=None, numslices=1): Runner.__init__(self, slice, numslices) BounceMixin.__init__(self) # We look this function up only at startup time - modname = 'Mailman.Handlers.' + mm_cfg.DELIVERY_MODULE + modname = 'Mailman.Handlers.' + config.DELIVERY_MODULE mod = __import__(modname) self._func = getattr(sys.modules[modname], 'process') # This prevents smtp server connection problems from filling up the # error log. It gets reset if the message was successfully sent, and # set if there was a socket.error. self.__logged = False - self.__retryq = Switchboard(mm_cfg.RETRYQUEUE_DIR) + self.__retryq = Switchboard(config.RETRYQUEUE_DIR) def _dispose(self, mlist, msg, msgdata): # See if we should retry delivery of this message again. @@ -75,13 +76,13 @@ class OutgoingRunner(Runner, BounceMixin): # There was a problem connecting to the SMTP server. Log this # once, but crank up our sleep time so we don't fill the error # log. - port = mm_cfg.SMTPPORT + port = config.SMTPPORT if port == 0: port = 'smtp' # Log this just once. if not self.__logged: log.error('Cannot connect to SMTP server %s on port %s', - mm_cfg.SMTPHOST, port) + config.SMTPHOST, port) self.__logged = True return True except Errors.SomeRecipientsFailed, e: @@ -115,7 +116,7 @@ class OutgoingRunner(Runner, BounceMixin): return False else: # Keep trying to delivery this message for a while - deliver_until = now + mm_cfg.DELIVERY_RETRY_PERIOD + deliver_until = now + config.DELIVERY_RETRY_PERIOD msgdata['last_recip_count'] = len(recips) msgdata['deliver_until'] = deliver_until msgdata['recips'] = recips diff --git a/Mailman/Queue/RetryRunner.py b/Mailman/Queue/RetryRunner.py index b6c0eac6b..da87982c1 100644 --- a/Mailman/Queue/RetryRunner.py +++ b/Mailman/Queue/RetryRunner.py @@ -12,23 +12,24 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. import time -from Mailman import mm_cfg from Mailman.Queue.Runner import Runner from Mailman.Queue.Switchboard import Switchboard +from Mailman.configuration import config class RetryRunner(Runner): - QDIR = mm_cfg.RETRYQUEUE_DIR - SLEEPTIME = mm_cfg.minutes(15) + QDIR = config.RETRYQUEUE_DIR + SLEEPTIME = config.minutes(15) def __init__(self, slice=None, numslices=1): Runner.__init__(self, slice, numslices) - self.__outq = Switchboard(mm_cfg.OUTQUEUE_DIR) + self.__outq = Switchboard(config.OUTQUEUE_DIR) def _dispose(self, mlist, msg, msgdata): # Move it to the out queue for another retry diff --git a/Mailman/Queue/Runner.py b/Mailman/Queue/Runner.py index 56baf431b..95cb3fd43 100644 --- a/Mailman/Queue/Runner.py +++ b/Mailman/Queue/Runner.py @@ -12,7 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. """Generic queue runner class.""" @@ -25,11 +26,11 @@ import email.Errors from cStringIO import StringIO from Mailman import Errors -from Mailman import i18n from Mailman import MailList -from Mailman import mm_cfg from Mailman import Utils +from Mailman import i18n from Mailman.Queue.Switchboard import Switchboard +from Mailman.configuration import config log = logging.getLogger('mailman.error') @@ -37,7 +38,7 @@ log = logging.getLogger('mailman.error') class Runner: QDIR = None - SLEEPTIME = mm_cfg.QRUNNER_SLEEP_TIME + SLEEPTIME = config.QRUNNER_SLEEP_TIME def __init__(self, slice=None, numslices=1): self._kids = {} @@ -45,7 +46,7 @@ class Runner: # we want to provide slice and numslice arguments. self._switchboard = Switchboard(self.QDIR, slice, numslices) # Create the shunt switchboard - self._shunt = Switchboard(mm_cfg.SHUNTQUEUE_DIR) + self._shunt = Switchboard(config.SHUNTQUEUE_DIR) self._stop = False def __repr__(self): @@ -132,7 +133,7 @@ class Runner: # Find out which mailing list this message is destined for. listname = msgdata.get('listname') if not listname: - listname = mm_cfg.MAILMAN_SITE_LIST + listname = config.MAILMAN_SITE_LIST mlist = self._open_list(listname) if not mlist: log.error('Dequeuing message destined for missing list: %s', @@ -153,7 +154,7 @@ class Runner: if mlist: lang = mlist.getMemberLanguage(sender) else: - lang = mm_cfg.DEFAULT_SERVER_LANGUAGE + lang = config.DEFAULT_SERVER_LANGUAGE i18n.set_language(lang) msgdata['lang'] = lang try: diff --git a/Mailman/Queue/Switchboard.py b/Mailman/Queue/Switchboard.py index 36b69283e..d9fc5dc27 100644 --- a/Mailman/Queue/Switchboard.py +++ b/Mailman/Queue/Switchboard.py @@ -12,7 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. """Reading and writing message objects and message metadata.""" @@ -41,8 +42,8 @@ import cPickle import marshal from Mailman import Message -from Mailman import mm_cfg from Mailman import Utils +from Mailman.configuration import config # 20 bytes of all bits set, maximum sha.digest() value shamax = 0xffffffffffffffffffffffffffffffffffffffffL @@ -104,7 +105,7 @@ class Switchboard: filename = os.path.join(self.__whichq, filebase + '.pck') tmpfile = filename + '.tmp' # Always add the metadata schema version number - data['version'] = mm_cfg.QFILE_SCHEMA_VERSION + data['version'] = config.QFILE_SCHEMA_VERSION # Filter out volatile entries for k in data.keys(): if k.startswith('_'): diff --git a/Mailman/Queue/VirginRunner.py b/Mailman/Queue/VirginRunner.py index 7d09ef8e9..bdb8a26bf 100644 --- a/Mailman/Queue/VirginRunner.py +++ b/Mailman/Queue/VirginRunner.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2006 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -12,7 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. """Virgin message queue runner. @@ -22,14 +23,14 @@ to go through some minimal processing before they can be sent out to the recipient. """ -from Mailman import mm_cfg -from Mailman.Queue.Runner import Runner from Mailman.Queue.IncomingRunner import IncomingRunner +from Mailman.Queue.Runner import Runner +from Mailman.configuration import config class VirginRunner(IncomingRunner): - QDIR = mm_cfg.VIRGINQUEUE_DIR + QDIR = config.VIRGINQUEUE_DIR def _dispose(self, mlist, msg, msgdata): # We need to fasttrack this message through any handlers that touch diff --git a/Mailman/Queue/__init__.py b/Mailman/Queue/__init__.py index 7fef22451..e69de29bb 100644 --- a/Mailman/Queue/__init__.py +++ b/Mailman/Queue/__init__.py @@ -1,15 +0,0 @@ -# Copyright (C) 2000,2001,2002 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. diff --git a/Mailman/Queue/sbcache.py b/Mailman/Queue/sbcache.py index dee1bca7f..6fe499d0d 100644 --- a/Mailman/Queue/sbcache.py +++ b/Mailman/Queue/sbcache.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2006 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -12,7 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. """A factory of Switchboards with caching.""" diff --git a/Mailman/Site.py b/Mailman/Site.py index 691d23870..72becb224 100644 --- a/Mailman/Site.py +++ b/Mailman/Site.py @@ -12,7 +12,8 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. """Provide some customization for site-wide behavior. @@ -23,7 +24,7 @@ implementation should work for standard Mailman. import os import errno -from Mailman import mm_cfg +from Mailman.configuration import config @@ -54,7 +55,7 @@ def get_listpath(listname, domain=None, create=0): should not attempt to create the path heirarchy (and in fact the absence of the path might be significant). """ - path = os.path.join(mm_cfg.LIST_DATA_DIR, listname) + path = os.path.join(config.LIST_DATA_DIR, listname) if create: _makedir(path) return path @@ -79,9 +80,9 @@ def get_archpath(listname, domain=None, create=False, public=False): is usually a symlink instead of a directory). """ if public: - subdir = mm_cfg.PUBLIC_ARCHIVE_FILE_DIR + subdir = config.PUBLIC_ARCHIVE_FILE_DIR else: - subdir = mm_cfg.PRIVATE_ARCHIVE_FILE_DIR + subdir = config.PRIVATE_ARCHIVE_FILE_DIR path = os.path.join(subdir, listname) if create: _makedir(path) @@ -101,7 +102,7 @@ def get_listnames(domain=None): from Mailman.Utils import list_exists # We don't currently support separate virtual domain directories got = [] - for fn in os.listdir(mm_cfg.LIST_DATA_DIR): + for fn in os.listdir(config.LIST_DATA_DIR): if list_exists(fn): got.append(fn) return got diff --git a/Mailman/Utils.py b/Mailman/Utils.py index 42bacc16a..5e594be8d 100644 --- a/Mailman/Utils.py +++ b/Mailman/Utils.py @@ -15,7 +15,6 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, # USA. - """Miscellaneous essential routines. This includes actual message transmission routines, address checking and @@ -42,8 +41,8 @@ from string import ascii_letters, digits, whitespace from Mailman import Errors from Mailman import Site -from Mailman import mm_cfg from Mailman.SafeDict import SafeDict +from Mailman.configuration import config EMPTYSTRING = '' UEMPTYSTRING = u'' @@ -228,7 +227,7 @@ def ScriptURL(target, web_page_url=None, absolute=False): absolute - a flag which if set, generates an absolute url """ if web_page_url is None: - web_page_url = mm_cfg.DEFAULT_URL_PATTERN % get_domain() + web_page_url = config.DEFAULT_URL_PATTERN % get_domain() if web_page_url[-1] <> '/': web_page_url = web_page_url + '/' fullpath = os.environ.get('REQUEST_URI') @@ -247,7 +246,7 @@ def ScriptURL(target, web_page_url=None, absolute=False): path = ('../' * count) + target else: path = web_page_url + target - return path + mm_cfg.CGIEXT + return path + config.CGIEXT @@ -334,8 +333,10 @@ def Secure_MakeRandomPassword(length): os.close(fd) -def MakeRandomPassword(length=mm_cfg.MEMBER_PASSWORD_LENGTH): - if mm_cfg.USER_FRIENDLY_PASSWORDS: +def MakeRandomPassword(length=None): + if length is None: + length = config.MEMBER_PASSWORD_LENGTH + if config.USER_FRIENDLY_PASSWORDS: return UserFriendly_MakeRandomPassword(length) return Secure_MakeRandomPassword(length) @@ -356,9 +357,9 @@ def GetRandomSeed(): def set_global_password(pw, siteadmin=True): if siteadmin: - filename = mm_cfg.SITE_PW_FILE + filename = config.SITE_PW_FILE else: - filename = mm_cfg.LISTCREATOR_PW_FILE + filename = config.LISTCREATOR_PW_FILE # rw-r----- omask = os.umask(026) try: @@ -371,9 +372,9 @@ def set_global_password(pw, siteadmin=True): def get_global_password(siteadmin=True): if siteadmin: - filename = mm_cfg.SITE_PW_FILE + filename = config.SITE_PW_FILE else: - filename = mm_cfg.LISTCREATOR_PW_FILE + filename = config.LISTCREATOR_PW_FILE try: fp = open(filename) challenge = fp.read()[:-1] # strip off trailing nl @@ -486,14 +487,14 @@ def findtext(templatefile, dict=None, raw=False, lang=None, mlist=None): languages.append(lang) if mlist is not None: languages.append(mlist.preferred_language) - languages.append(mm_cfg.DEFAULT_SERVER_LANGUAGE) + languages.append(config.DEFAULT_SERVER_LANGUAGE) # Calculate the locations to scan searchdirs = [] if mlist is not None: searchdirs.append(mlist.fullpath()) - searchdirs.append(os.path.join(mm_cfg.TEMPLATE_DIR, mlist.host_name)) - searchdirs.append(os.path.join(mm_cfg.TEMPLATE_DIR, 'site')) - searchdirs.append(mm_cfg.TEMPLATE_DIR) + searchdirs.append(os.path.join(config.TEMPLATE_DIR, mlist.host_name)) + searchdirs.append(os.path.join(config.TEMPLATE_DIR, 'site')) + searchdirs.append(config.TEMPLATE_DIR) # Start scanning quickexit = 'quickexit' fp = None @@ -514,7 +515,7 @@ def findtext(templatefile, dict=None, raw=False, lang=None, mlist=None): # Try one last time with the distro English template, which, unless # you've got a really broken installation, must be there. try: - filename = os.path.join(mm_cfg.TEMPLATE_DIR, 'en', templatefile) + filename = os.path.join(config.TEMPLATE_DIR, 'en', templatefile) fp = open(filename) except IOError, e: if e.errno <> errno.ENOENT: raise @@ -573,7 +574,7 @@ def is_administrivia(msg): break if line.strip(): linecnt += 1 - if linecnt > mm_cfg.DEFAULT_MAIL_COMMANDS_MAX_LINES: + if linecnt > config.DEFAULT_MAIL_COMMANDS_MAX_LINES: return False lines.append(line) bodytext = NL.join(lines) @@ -652,14 +653,14 @@ def reap(kids, func=None, once=False): def GetLanguageDescr(lang): - return mm_cfg.LC_DESCRIPTIONS[lang][0] + return config.LC_DESCRIPTIONS[lang][0] def GetCharSet(lang): - return mm_cfg.LC_DESCRIPTIONS[lang][1] + return config.LC_DESCRIPTIONS[lang][1] def IsLanguage(lang): - return mm_cfg.LC_DESCRIPTIONS.has_key(lang) + return config.LC_DESCRIPTIONS.has_key(lang) @@ -669,23 +670,23 @@ def get_domain(): # Strip off the port if there is one if port and host.endswith(':' + port): host = host[:-len(port)-1] - if mm_cfg.VIRTUAL_HOST_OVERVIEW and host: + if config.VIRTUAL_HOST_OVERVIEW and host: return host.lower() else: # See the note in Defaults.py concerning DEFAULT_URL # vs. DEFAULT_URL_HOST. - hostname = ((mm_cfg.DEFAULT_URL - and urlparse.urlparse(mm_cfg.DEFAULT_URL)[1]) - or mm_cfg.DEFAULT_URL_HOST) + hostname = ((config.DEFAULT_URL + and urlparse.urlparse(config.DEFAULT_URL)[1]) + or config.DEFAULT_URL_HOST) return hostname.lower() def get_site_email(hostname=None, extra=None): if hostname is None: - hostname = mm_cfg.VIRTUAL_HOSTS.get(get_domain(), get_domain()) + hostname = config.VIRTUAL_HOSTS.get(get_domain(), get_domain()) if extra is None: - return '%s@%s' % (mm_cfg.MAILMAN_SITE_LIST, hostname) - return '%s-%s@%s' % (mm_cfg.MAILMAN_SITE_LIST, extra, hostname) + return '%s@%s' % (config.MAILMAN_SITE_LIST, hostname) + return '%s-%s@%s' % (config.MAILMAN_SITE_LIST, extra, hostname) diff --git a/Mailman/Version.py b/Mailman/Version.py index 6c9aad05e..91fb82e65 100644 --- a/Mailman/Version.py +++ b/Mailman/Version.py @@ -46,3 +46,7 @@ PENDING_FILE_SCHEMA_VERSION = 2 # version number for the lists/<listname>/request.db file schema REQUESTS_FILE_SCHEMA_VERSION = 1 + +# Printable version string used by command line scripts +MAILMAN_VERSION = 'GNU Mailman ' + VERSION + diff --git a/Mailman/bin/list_lists.py b/Mailman/bin/list_lists.py index a3493c3b8..0feac2845 100644 --- a/Mailman/bin/list_lists.py +++ b/Mailman/bin/list_lists.py @@ -18,9 +18,11 @@ import sys import optparse +from Mailman import Defaults from Mailman import MailList from Mailman import Utils -from Mailman import mm_cfg +from Mailman import Version +from Mailman.configuration import config from Mailman.i18n import _ __i18n_templates__ = True @@ -28,7 +30,7 @@ __i18n_templates__ = True def parseargs(): - parser = optparse.OptionParser(version=mm_cfg.MAILMAN_VERSION, + parser = optparse.OptionParser(version=Version.MAILMAN_VERSION, usage=_("""\ %prog [options] @@ -46,6 +48,8 @@ Displays only the list name, with no description.""")) help=_("""\ List only those mailing lists that are homed to the given virtual domain. This only works if the VIRTUAL_HOST_OVERVIEW variable is set.""")) + parser.add_option('-C', '--config', + help=_('Alternative configuration file to use')) opts, args = parser.parse_args() if args: parser.print_help() @@ -57,6 +61,7 @@ This only works if the VIRTUAL_HOST_OVERVIEW variable is set.""")) def main(): parser, opts, args = parseargs() + config.load(opts.config) names = Utils.list_names() names.sort() @@ -67,7 +72,7 @@ def main(): mlist = MailList.MailList(n, lock=False) if opts.advertised and not mlist.advertised: continue - if opts.vhost and mm_cfg.VIRTUAL_HOST_OVERVIEW and \ + if opts.vhost and config.VIRTUAL_HOST_OVERVIEW and \ opts.vhost.find(mlist.web_page_url) == -1 and \ mlist.web_page_url.find(opts.vhost) == -1: continue diff --git a/Mailman/bin/mailmanctl.py b/Mailman/bin/mailmanctl.py new file mode 100644 index 000000000..a91b5651b --- /dev/null +++ b/Mailman/bin/mailmanctl.py @@ -0,0 +1,521 @@ +# Copyright (C) 2001-2006 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +import os +import grp +import pwd +import sys +import errno +import signal +import socket +import logging +import optparse + +from Mailman import Defaults +from Mailman import Errors +from Mailman import LockFile +from Mailman import Utils +from Mailman import Version +from Mailman import loginit +from Mailman.MailList import MailList +from Mailman.configuration import config +from Mailman.i18n import _ + +__i18n_templates__ = True + +COMMASPACE = ', ' +DOT = '.' + +# Since we wake up once per day and refresh the lock, the LOCK_LIFETIME +# needn't be (much) longer than SNOOZE. We pad it 6 hours just to be safe. +LOCK_LIFETIME = Defaults.days(1) + Defaults.hours(6) +SNOOZE = Defaults.days(1) +MAX_RESTARTS = 10 + + + +def parseargs(): + parser = optparse.OptionParser(version=Version.MAILMAN_VERSION, + usage=_("""\ +Primary start-up and shutdown script for Mailman's qrunner daemon. + +This script starts, stops, and restarts the main Mailman queue runners, making +sure that the various long-running qrunners are still alive and kicking. It +does this by forking and exec'ing the qrunners and waiting on their pids. +When it detects a subprocess has exited, it may restart it. + +The qrunners respond to SIGINT, SIGTERM, and SIGHUP. SIGINT and SIGTERM both +cause the qrunners to exit cleanly, but the master will only restart qrunners +that have exited due to a SIGINT. SIGHUP causes the master and the qrunners +to close their log files, and reopen then upon the next printed message. + +The master also responds to SIGINT, SIGTERM, and SIGHUP, which it simply +passes on to the qrunners (note that the master will close and reopen its own +log files on receipt of a SIGHUP). The master also leaves its own process id +in the file data/master-qrunner.pid but you normally don't need to use this +pid directly. The `start', `stop', `restart', and `reopen' commands handle +everything for you. + +Commands: + + start - Start the master daemon and all qrunners. Prints a message and + exits if the master daemon is already running. + + stop - Stops the master daemon and all qrunners. After stopping, no + more messages will be processed. + + restart - Restarts the qrunners, but not the master process. Use this + whenever you upgrade or update Mailman so that the qrunners will + use the newly installed code. + + reopen - This will close all log files, causing them to be re-opened the + next time a message is written to them + +Usage: %prog [options] [ start | stop | restart | reopen ]""")) + parser.add_option('-n', '--no-restart', + dest='restart', default=True, action='store_false', + help=_("""\ +Don't restart the qrunners when they exit because of an error or a SIGINT. +They are never restarted if they exit in response to a SIGTERM. Use this only +for debugging. Only useful if the `start' command is given.""")) + parser.add_option('-u', '--run-as-user', + dest='checkprivs', default=True, action='store_false', + help=_("""\ +Normally, this script will refuse to run if the user id and group id are not +set to the `mailman' user and group (as defined when you configured Mailman). +If run as root, this script will change to this user and group before the +check is made. + +This can be inconvenient for testing and debugging purposes, so the -u flag +means that the step that sets and checks the uid/gid is skipped, and the +program is run as the current user and group. This flag is not recommended +for normal production environments. + +Note though, that if you run with -u and are not in the mailman group, you may +have permission problems, such as begin unable to delete a list's archives +through the web. Tough luck!""")) + parser.add_option('-s', '--stale-lock-cleanup', + dest='force', default=False, action='store_true', + help=_("""\ +If mailmanctl finds an existing master lock, it will normally exit with an +error message. With this option, mailmanctl will perform an extra level of +checking. If a process matching the host/pid described in the lock file is +running, mailmanctl will still exit, but if no matching process is found, +mailmanctl will remove the apparently stale lock and make another attempt to +claim the master lock.""")) + parser.add_option('-q', '--quiet', + default=False, action='store_true', + help=_("""\ +Don't print status messages. Error messages are still printed to standard +error.""")) + parser.add_option('-C', '--config', + help=_('Alternative configuration file to use')) + opts, args = parser.parse_args() + if not args: + parser.print_help() + print >> sys.stderr, _('No command given.') + sys.exit(1) + if len(args) > 1: + parse.print_help() + commands = COMMASPACE.join(args) + print >> sys.stderr, _('Bad command: $commands') + sys.exit(1) + return parser, opts, args + + + +def kill_watcher(sig): + try: + fp = open(config.PIDFILE) + pidstr = fp.read() + fp.close() + pid = int(pidstr.strip()) + except (IOError, ValueError), e: + # For i18n convenience + pidfile = config.PIDFILE + print >> sys.stderr, _('PID unreadable in: $pidfile') + print >> sys.stderr, e + print >> sys.stderr, _('Is qrunner even running?') + return + try: + os.kill(pid, sig) + except OSError, e: + if e.errno <> errno.ESRCH: raise + print >> sys.stderr, _('No child with pid: $pid') + print >> sys.stderr, e + print >> sys.stderr, _('Stale pid file removed.') + os.unlink(config.PIDFILE) + + + +def get_lock_data(): + # Return the hostname, pid, and tempfile + fp = open(config.LOCKFILE) + try: + filename = os.path.split(fp.read().strip())[1] + finally: + fp.close() + parts = filename.split('.') + hostname = DOT.join(parts[1:-1]) + pid = int(parts[-1]) + return hostname, int(pid), filename + + +def qrunner_state(): + # 1 if proc exists on host (but is it qrunner? ;) + # 0 if host matches but no proc + # hostname if hostname doesn't match + hostname, pid, tempfile = get_lock_data() + if hostname <> socket.gethostname(): + return hostname + # Find out if the process exists by calling kill with a signal 0. + try: + os.kill(pid, 0) + except OSError, e: + if e.errno <> errno.ESRCH: + raise + return 0 + return 1 + + +def acquire_lock_1(force): + # Be sure we can acquire the master qrunner lock. If not, it means some + # other master qrunner daemon is already going. + lock = LockFile.LockFile(config.LOCK_FILE, LOCK_LIFETIME) + try: + lock.lock(0.1) + return lock + except LockFile.TimeOutError: + if not force: + raise + # Force removal of lock first + lock._disown() + hostname, pid, tempfile = get_lock_data() + os.unlink(config.LOCKFILE) + os.unlink(os.path.join(config.LOCK_DIR, tempfile)) + return acquire_lock_1(force=False) + + +def acquire_lock(force): + try: + lock = acquire_lock_1(force) + return lock + except LockFile.TimeOutError: + status = qrunner_state() + if status == 1: + # host matches and proc exists + print >> sys.stderr, _("""\ +The master qrunner lock could not be acquired because it appears as if another +master qrunner is already running. +""") + elif status == 0: + # host matches but no proc + print >> sys.stderr, _("""\ +The master qrunner lock could not be acquired. It appears as though there is +a stale master qrunner lock. Try re-running mailmanctl with the -s flag. +""") + else: + # host doesn't even match + print >> sys.stderr, _("""\ +The master qrunner lock could not be acquired, because it appears as if some +process on some other host may have acquired it. We can't test for stale +locks across host boundaries, so you'll have to do this manually. Or, if you +know the lock is stale, re-run mailmanctl with the -s flag. + +Lock file: $config.LOCKFILE +Lock host: $status + +Exiting.""") + + + +def start_runner(qrname, slice, count): + pid = os.fork() + if pid: + # parent + return pid + # child + # + # Craft the command line arguments for the exec() call. + rswitch = '--runner=%s:%d:%d' % (qrname, slice, count) + exe = os.path.join(config.BIN_DIR, 'qrunner') + # config.PYTHON, which is the absolute path to the Python interpreter, + # must be given as argv[0] due to Python's library search algorithm. + os.execl(config.PYTHON, config.PYTHON, exe, rswitch, '-s') + # Should never get here + raise RuntimeError, 'os.execl() failed' + + +def start_all_runners(): + kids = {} + for qrname, count in config.QRUNNERS: + for slice in range(count): + # queue runner name, slice, numslices, restart count + info = (qrname, slice, count, 0) + pid = start_runner(qrname, slice, count) + kids[pid] = info + return kids + + + +def check_for_site_list(): + sitelistname = config.MAILMAN_SITE_LIST + try: + sitelist = MailList(sitelistname, lock=False) + except Errors.MMUnknownListError: + print >> sys.stderr, _('Site list is missing: $sitelistname') + elog.error('Site list is missing: %s', config.MAILMAN_SITE_LIST) + sys.exit(1) + + +def check_privs(): + # If we're running as root (uid == 0), coerce the uid and gid to that + # which Mailman was configured for, and refuse to run if we didn't coerce + # the uid/gid. + gid = grp.getgrnam(config.MAILMAN_GROUP)[2] + uid = pwd.getpwnam(config.MAILMAN_USER)[2] + myuid = os.getuid() + if myuid == 0: + # Set the process's supplimental groups. + groups = [x[2] for x in grp.getgrall() if config.MAILMAN_USER in x[3]] + groups.append(gid) + os.setgroups(groups) + os.setgid(gid) + os.setuid(uid) + elif myuid <> uid: + name = config.MAILMAN_USER + usage(1, _( + 'Run this program as root or as the $name user, or use -u.')) + + + +def main(): + global elog, qlog + + parser, opts, args = parseargs() + config.load(opts.config) + + loginit.initialize() + elog = logging.getLogger('mailman.error') + qlog = logging.getLogger('mailman.qrunner') + + if opts.checkprivs: + check_privs() + else: + print _('Warning! You may encounter permission problems.') + + # Handle the commands + command = args[0].lower() + if command == 'stop': + # Sent the master qrunner process a SIGINT, which is equivalent to + # giving cron/qrunner a ctrl-c or KeyboardInterrupt. This will + # effectively shut everything down. + if not opts.quiet: + print _("Shutting down Mailman's master qrunner") + kill_watcher(signal.SIGTERM) + elif command == 'restart': + # Sent the master qrunner process a SIGHUP. This will cause the + # master qrunner to kill and restart all the worker qrunners, and to + # close and re-open its log files. + if not opts.quiet: + print _("Restarting Mailman's master qrunner") + kill_watcher(signal.SIGINT) + elif command == 'reopen': + if not opts.quiet: + print _('Re-opening all log files') + kill_watcher(signal.SIGHUP) + elif command == 'start': + # First, complain loudly if there's no site list. + check_for_site_list() + # Here's the scoop on the processes we're about to create. We'll need + # one for each qrunner, and one for a master child process watcher / + # lock refresher process. + # + # The child watcher process simply waits on the pids of the children + # qrunners. Unless explicitly disabled by a mailmanctl switch (or the + # children are killed with SIGTERM instead of SIGINT), the watcher + # will automatically restart any child process that exits. This + # allows us to be more robust, and also to implement restart by simply + # SIGINT'ing the qrunner children, and letting the watcher restart + # them. + # + # Under normal operation, we have a child per queue. This lets us get + # the most out of the available resources, since a qrunner with no + # files in its queue directory is pretty cheap, but having a separate + # runner process per queue allows for a very responsive system. Some + # people want a more traditional (i.e. MM2.0.x) cron-invoked qrunner. + # No problem, but using mailmanctl isn't the answer. So while + # mailmanctl hard codes some things, others, such as the number of + # qrunners per queue, are configurable. + # + # First, acquire the master mailmanctl lock + lock = acquire_lock(opts.force) + if not lock: + return + # Daemon process startup according to Stevens, Advanced Programming in + # the UNIX Environment, Chapter 13. + pid = os.fork() + if pid: + # parent + if not opts.quiet: + print _("Starting Mailman's master qrunner.") + # Give up the lock "ownership". This just means the foreground + # process won't close/unlock the lock when it finalizes this lock + # instance. We'll let the mater watcher subproc own the lock. + lock._transfer_to(pid) + return + # child + lock._take_possession() + # First, save our pid in a file for "mailmanctl stop" rendezvous. We + # want the perms on the .pid file to be rw-rw---- + omask = os.umask(6) + try: + fp = open(config.PIDFILE, 'w') + print >> fp, os.getpid() + fp.close() + finally: + os.umask(omask) + # Create a new session and become the session leader, but since we + # won't be opening any terminal devices, don't do the ultra-paranoid + # suggestion of doing a second fork after the setsid() call. + os.setsid() + # Instead of cd'ing to root, cd to the Mailman installation home + os.chdir(config.PREFIX) + # Set our file mode creation umask + os.umask(007) + # I don't think we have any unneeded file descriptors. + # + # Now start all the qrunners. This returns a dictionary where the + # keys are qrunner pids and the values are tuples of the following + # form: (qrname, slice, count). This does its own fork and exec, and + # sets up its own signal handlers. + kids = start_all_runners() + # Set up a SIGALRM handler to refresh the lock once per day. The lock + # lifetime is 1day+6hours so this should be plenty. + def sigalrm_handler(signum, frame): + lock.refresh() + signal.alarm(Defaults.days(1)) + signal.signal(signal.SIGALRM, sigalrm_handler) + signal.alarm(Defaults.days(1)) + # Set up a SIGHUP handler so that if we get one, we'll pass it along + # to all the qrunner children. This will tell them to close and + # reopen their log files + def sighup_handler(signum, frame): + loginit.reopen() + for pid in kids.keys(): + os.kill(pid, signal.SIGHUP) + # And just to tweak things... + qlog.info('Master watcher caught SIGHUP. Re-opening log files.') + signal.signal(signal.SIGHUP, sighup_handler) + # We also need to install a SIGTERM handler because that's what init + # will kill this process with when changing run levels. + def sigterm_handler(signum, frame): + for pid in kids.keys(): + try: + os.kill(pid, signal.SIGTERM) + except OSError, e: + if e.errno <> errno.ESRCH: raise + qlog.info('Master watcher caught SIGTERM. Exiting.') + signal.signal(signal.SIGTERM, sigterm_handler) + # Finally, we need a SIGINT handler which will cause the sub-qrunners + # to exit, but the master will restart SIGINT'd sub-processes unless + # the -n flag was given. + def sigint_handler(signum, frame): + for pid in kids.keys(): + os.kill(pid, signal.SIGINT) + qlog.info('Master watcher caught SIGINT. Restarting.') + signal.signal(signal.SIGINT, sigint_handler) + # Now we're ready to simply do our wait/restart loop. This is the + # master qrunner watcher. + try: + while True: + try: + pid, status = os.wait() + except OSError, e: + # No children? We're done + if e.errno == errno.ECHILD: + break + # If the system call got interrupted, just restart it. + elif e.errno <> errno.EINTR: + raise + continue + killsig = exitstatus = None + if os.WIFSIGNALED(status): + killsig = os.WTERMSIG(status) + if os.WIFEXITED(status): + exitstatus = os.WEXITSTATUS(status) + # We'll restart the process unless we were given the + # "no-restart" switch, or if the process was SIGTERM'd or + # exitted with a SIGTERM exit status. This lets us better + # handle runaway restarts (say, if the subproc had a syntax + # error!) + restarting = '' + if opts.restart: + if ((exitstatus == None and killsig <> signal.SIGTERM) or + (killsig == None and exitstatus <> signal.SIGTERM)): + # Then + restarting = '[restarting]' + qrname, slice, count, restarts = kids[pid] + del kids[pid] + qlog.info("""\ +Master qrunner detected subprocess exit +(pid: %d, sig: %s, sts: %s, class: %s, slice: %d/%d) %s""", + pid, killsig, exitstatus, qrname, + slice+1, count, restarting) + # See if we've reached the maximum number of allowable restarts + if exitstatus <> signal.SIGINT: + restarts += 1 + if restarts > MAX_RESTARTS: + qlog.info("""\ +Qrunner %s reached maximum restart limit of %d, not restarting.""", + qrname, MAX_RESTARTS) + restarting = '' + # Now perhaps restart the process unless it exited with a + # SIGTERM or we aren't restarting. + if restarting: + newpid = start_runner(qrname, slice, count) + kids[newpid] = (qrname, slice, count, restarts) + finally: + # Should we leave the main loop for any reason, we want to be sure + # all of our children are exited cleanly. Send SIGTERMs to all + # the child processes and wait for them all to exit. + for pid in kids.keys(): + try: + os.kill(pid, signal.SIGTERM) + except OSError, e: + if e.errno == errno.ESRCH: + # The child has already exited + qlog.info('ESRCH on pid: %d', pid) + del kids[pid] + # Wait for all the children to go away + while True: + try: + pid, status = os.wait() + except OSError, e: + if e.errno == errno.ECHILD: + break + elif e.errno <> errno.EINTR: + raise + continue + # Finally, give up the lock + lock.unlock(unconditionally=True) + os._exit(0) + + + +if __name__ == '__main__': + main() diff --git a/Mailman/bin/newlist.py b/Mailman/bin/newlist.py index 26c7cd42b..0f62f5373 100644 --- a/Mailman/bin/newlist.py +++ b/Mailman/bin/newlist.py @@ -25,8 +25,9 @@ from Mailman import Errors from Mailman import MailList from Mailman import Message from Mailman import Utils +from Mailman import Version from Mailman import i18n -from Mailman import mm_cfg +from Mailman.configuration import config _ = i18n._ @@ -35,7 +36,7 @@ __i18n_templates__ = True def parseargs(): - parser = optparse.OptionParser(version=mm_cfg.MAILMAN_VERSION, + parser = optparse.OptionParser(version=Version.MAILMAN_VERSION, usage=_("""\ %%prog [options] [listname [listadmin-addr [admin-password]]] @@ -87,7 +88,6 @@ defined in your Defaults.py file or overridden by settings in mm_cfg.py). Note that listnames are forced to lowercase.""")) parser.add_option('-l', '--language', type='string', action='store', - default=mm_cfg.DEFAULT_SERVER_LANGUAGE, help=_("""\ Make the list's preferred language LANGUAGE, which must be a two letter language code.""")) @@ -110,18 +110,27 @@ This option suppresses the prompt prior to administrator notification but still sends the notification. It can be used to make newlist totally non-interactive but still send the notification, assuming listname, listadmin-addr and admin-password are all specified on the command line.""")) + parser.add_option('-C', '--config', + help=_('Alternative configuration file to use')) opts, args = parser.parse_args() - # Is the language known? - if opts.language not in mm_cfg.LC_DESCRIPTIONS: - parser.print_help() - print >> sys.stderr, _('Unknown language: $opts.language') - sys.exit(1) + # Can't verify opts.language here because the configuration isn't loaded + # yet. return parser, opts, args def main(): parser, opts, args = parseargs() + config.load(opts.config) + + # Set up some defaults we couldn't set up in parseargs() + if opts.language is None: + opts.language = config.DEFAULT_SERVER_LANGUAGE + # Is the language known? + if opts.language not in config.LC_DESCRIPTIONS: + parser.print_help() + print >> sys.stderr, _('Unknown language: $opts.language') + sys.exit(1) # Handle variable number of positional arguments if args: @@ -134,12 +143,12 @@ def main(): # Note that --urlhost and --emailhost have precedence listname, domain = listname.split('@', 1) urlhost = opts.urlhost or domain - emailhost = opts.emailhost or mm_cfg.VIRTUAL_HOSTS.get(domain, domain) + emailhost = opts.emailhost or config.VIRTUAL_HOSTS.get(domain, domain) - urlhost = opts.urlhost or mm_cfg.DEFAULT_URL_HOST + urlhost = opts.urlhost or config.DEFAULT_URL_HOST host_name = (opts.emailhost or - mm_cfg.VIRTUAL_HOSTS.get(urlhost, mm_cfg.DEFAULT_EMAIL_HOST)) - web_page_url = mm_cfg.DEFAULT_URL_PATTERN % urlhost + config.VIRTUAL_HOSTS.get(urlhost, config.DEFAULT_EMAIL_HOST)) + web_page_url = config.DEFAULT_URL_PATTERN % urlhost if Utils.list_exists(listname): parser.print_help() @@ -154,7 +163,12 @@ def main(): if args: listpasswd = args.pop(0) else: - listpasswd = getpass.getpass(_('Initial $listname password: ')) + while True: + listpasswd = getpass.getpass(_('Initial $listname password: ')) + confirm = getpass.getpass(_('Confirm $listname password: ')) + if listpasswd == confirm: + break + print _('Passwords did not match, try again (Ctrl-C to quit)') # List passwords cannot be empty listpasswd = listpasswd.strip() @@ -198,8 +212,8 @@ def main(): mlist.Unlock() # Now do the MTA-specific list creation tasks - if mm_cfg.MTA: - modname = 'Mailman.MTA.' + mm_cfg.MTA + if config.MTA: + modname = 'Mailman.MTA.' + config.MTA __import__(modname) sys.modules[modname].create(mlist) diff --git a/Mailman/bin/qrunner.py b/Mailman/bin/qrunner.py new file mode 100644 index 000000000..1fab69d98 --- /dev/null +++ b/Mailman/bin/qrunner.py @@ -0,0 +1,258 @@ +# Copyright (C) 2001-2006 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +import sys +import signal +import logging +import optparse + +from Mailman import Version +from Mailman import loginit +from Mailman.configuration import config +from Mailman.i18n import _ + +__i18n_templates__ = True + +COMMASPACE = ', ' +log = None + + + +def r_callback(option, opt, value, parser): + dest = getattr(parser.values, option.dest) + parts = value.split(':') + if len(parts) == 1: + runner = parts[0] + rslice = rrange = 1 + elif len(parts) == 3: + runner = parts[0] + try: + rslice = int(parts[1]) + rrange = int(parts[2]) + except ValueError: + parser.print_help() + print >> sys.stderr, _('Bad runner specification: $value') + sys.exit(1) + else: + parser.print_help() + print >> sys.stderr, _('Bad runner specification: $value') + sys.exit(1) + if runner == 'All': + for runnername, slices in config.QRUNNERS: + dest.append((runnername, rslice, rrange)) + elif not runner.endswith('Runner'): + runner += 'Runner' + dest.append((runner, rslice, rrange)) + + + +def parseargs(): + parser = optparse.OptionParser(version=Version.MAILMAN_VERSION, + usage=_("""\ +Run one or more qrunners, once or repeatedly. + +Each named runner class is run in round-robin fashion. In other words, the +first named runner is run to consume all the files currently in its +directory. When that qrunner is done, the next one is run to consume all the +files in /its/ directory, and so on. The number of total iterations can be +given on the command line. + +Usage: %prog [options] + +-r is required unless -l or -h is given, and its argument must be one of the +names displayed by the -l switch. + +Normally, this script should be started from mailmanctl. Running it +separately or with -o is generally useful only for debugging. +""")) + parser.add_option('-r', '--runner', + metavar='runner[:slice:range]', dest='runners', + type='string', default=[], + action='callback', callback=r_callback, + help=_("""\ +Run the named qrunner, which must be one of the strings returned by the -l +option. Optional slice:range if given, is used to assign multiple qrunner +processes to a queue. range is the total number of qrunners for this queue +while slice is the number of this qrunner from [0..range). + +When using the slice:range form, you must ensure that each qrunner for the +queue is given the same range value. If slice:runner is not given, then 1:1 +is used. + +Multiple -r options may be given, in which case each qrunner will run once in +round-robin fashion. The special runner `All' is shorthand for a qrunner for +each listed by the -l option.""")) + parser.add_option('-o', '--once', + default=False, action='store_true', help=_("""\ +Run each named qrunner exactly once through its main loop. Otherwise, each +qrunner runs indefinitely, until the process receives a SIGTERM or SIGINT.""")) + parser.add_option('-l', '--list', + default=False, action='store_true', + help=_('List the available qrunner names and exit.')) + parser.add_option('-v', '--verbose', + default=0, action='count', help=_("""\ +Display more debugging information to the logs/qrunner log file.""")) + parser.add_option('-s', '--subproc', + default=False, action='store_true', help=_("""\ +This should only be used when running qrunner as a subprocess of the +mailmanctl startup script. It changes some of the exit-on-error behavior to +work better with that framework.""")) + parser.add_option('-C', '--config', + help=_('Alternative configuration file to use')) + opts, args = parser.parse_args() + if args: + parser.print_help() + print >> sys.stderr, _('Unexpected arguments') + sys.exit(1) + if not opts.runners and not opts.list: + parser.print_help() + print >> sys.stderr, _('No runner name given.') + sys.exit(1) + return parser, opts, args + + + +def make_qrunner(name, slice, range, once=False): + modulename = 'Mailman.Queue.' + name + try: + __import__(modulename) + except ImportError, e: + if opts.subproc: + # Exit with SIGTERM exit code so mailmanctl won't try to restart us + print >> sys.stderr, _('Cannot import runner module: $modulename') + print >> sys.stderr, e + sys.exit(signal.SIGTERM) + else: + print >> sys.stderr, e + sys.exit(1) + qrclass = getattr(sys.modules[modulename], name) + if once: + # Subclass to hack in the setting of the stop flag in _doperiodic() + class Once(qrclass): + def _doperiodic(self): + self.stop() + qrunner = Once(slice, range) + else: + qrunner = qrclass(slice, range) + return qrunner + + + +def set_signals(loop): + # Set up the SIGTERM handler for stopping the loop + def sigterm_handler(signum, frame): + # Exit the qrunner cleanly + loop.stop() + loop.status = signal.SIGTERM + log.info('%s qrunner caught SIGTERM. Stopping.', loop.name()) + signal.signal(signal.SIGTERM, sigterm_handler) + # Set up the SIGINT handler for stopping the loop. For us, SIGINT is + # the same as SIGTERM, but our parent treats the exit statuses + # differently (it restarts a SIGINT but not a SIGTERM). + def sigint_handler(signum, frame): + # Exit the qrunner cleanly + loop.stop() + loop.status = signal.SIGINT + log.info('%s qrunner caught SIGINT. Stopping.', loop.name()) + signal.signal(signal.SIGINT, sigint_handler) + # SIGHUP just tells us to rotate our log files. + def sighup_handler(signum, frame): + loginit.reopen() + log.info('%s qrunner caught SIGHUP. Reopening logs.', loop.name()) + signal.signal(signal.SIGHUP, sighup_handler) + + + +def main(): + global log, opts + + parser, opts, args = parseargs() + config.load(opts.config) + + # If we're not running as a subprocess of mailmanctl, then we'll log to + # stderr in addition to logging to the log files. We do this by passing a + # value of True to propagate, which allows the 'mailman' root logger to + # see the log messages. + loginit.initialize(propagate=not opts.subproc) + log = logging.getLogger('mailman.qrunner') + + if opts.list: + for runnername, slices in config.QRUNNERS: + if runnername.endswith('Runner'): + name = runnername[:-len('Runner')] + else: + name = runnername + print _('$name runs the $runnername qrunner') + print _('All runs all the above qrunners') + sys.exit(0) + + # Fast track for one infinite runner + if len(opts.runners) == 1 and not opts.once: + qrunner = make_qrunner(*opts.runners[0]) + class Loop: + status = 0 + def __init__(self, qrunner): + self._qrunner = qrunner + def name(self): + return self._qrunner.__class__.__name__ + def stop(self): + self._qrunner.stop() + loop = Loop(qrunner) + set_signals(loop) + # Now start up the main loop + log.info('%s qrunner started.', loop.name()) + qrunner.run() + log.info('%s qrunner exiting.', loop.name()) + else: + # Anything else we have to handle a bit more specially + qrunners = [] + for runner, rslice, rrange in opts.runners: + qrunner = make_qrunner(runner, rslice, rrange, once=True) + qrunners.append(qrunner) + # This class is used to manage the main loop + class Loop: + status = 0 + def __init__(self): + self._isdone = False + def name(self): + return 'Main loop' + def stop(self): + self._isdone = True + def isdone(self): + return self._isdone + loop = Loop() + set_signals(loop) + log.info('Main qrunner loop started.') + while not loop.isdone(): + for qrunner in qrunners: + # In case the SIGTERM came in the middle of this iteration + if loop.isdone(): + break + if opts.verbose: + log.info('Now doing a %s qrunner iteration', + qrunner.__class__.__bases__[0].__name__) + qrunner.run() + if opts.once: + break + log.info('Main qrunner loop exiting.') + # All done + sys.exit(loop.status) + + + +if __name__ == '__main__': + main() diff --git a/Mailman/bin/rmlist.py b/Mailman/bin/rmlist.py index aef927c88..9655327f0 100644 --- a/Mailman/bin/rmlist.py +++ b/Mailman/bin/rmlist.py @@ -22,7 +22,8 @@ import optparse from Mailman import MailList from Mailman import Utils -from Mailman import mm_cfg +from Mailman import Version +from Mailman.configuration import config from Mailman.i18n import _ __i18n_templates__ = True @@ -44,7 +45,7 @@ def remove_it(listname, filename, msg): def parseargs(): - parser = optparse.OptionParser(version=mm_cfg.MAILMAN_VERSION, + parser = optparse.OptionParser(version=Version.MAILMAN_VERSION, usage=_("""\ %prog [options] listname @@ -57,6 +58,8 @@ archives are not removed, which is very handy for retiring old lists. default=False, action='store_true', help=_("""\ Remove the list's archives too, or if the list has already been deleted, remove any residual archives.""")) + parser.add_option('-C', '--config', + help=_('Alternative configuration file to use')) opts, args = parser.parse_args() if not args: parser.print_help() @@ -72,6 +75,8 @@ remove any residual archives.""")) def main(): parser, opts, args = parseargs() + config.load(opts.config) + listname = args[0].lower().strip() if not Utils.list_exists(listname): if not opts.archives: @@ -89,18 +94,18 @@ def main(): if Utils.list_exists(listname): mlist = MailList.MailList(listname, lock=False) # Do the MTA-specific list deletion tasks - if mm_cfg.MTA: - modname = 'Mailman.MTA.' + mm_cfg.MTA + if config.MTA: + modname = 'Mailman.MTA.' + config.MTA __import__(modname) sys.modules[modname].remove(mlist) removeables.append((os.path.join('lists', listname), _('list info'))) # Remove any stale locks associated with the list - for filename in os.listdir(mm_cfg.LOCK_DIR): + for filename in os.listdir(config.LOCK_DIR): fn_listname = filename.split('.')[0] if fn_listname == listname: - removeables.append((os.path.join(mm_cfg.LOCK_DIR, filename), + removeables.append((os.path.join(config.LOCK_DIR, filename), _('stale lock file'))) if opts.archives: @@ -116,7 +121,7 @@ def main(): ]) for dirtmpl, msg in removeables: - path = os.path.join(mm_cfg.VAR_PREFIX, dirtmpl) + path = os.path.join(config.VAR_PREFIX, dirtmpl) remove_it(listname, path, msg) diff --git a/Mailman/bin/update.py b/Mailman/bin/update.py new file mode 100644 index 000000000..ea74abfd6 --- /dev/null +++ b/Mailman/bin/update.py @@ -0,0 +1,767 @@ +# Copyright (C) 1998-2006 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +import os +import md5 +import sys +import time +import email +import errno +import shutil +import cPickle +import marshal +import optparse + +from Mailman import MailList +from Mailman import Message +from Mailman import Pending +from Mailman import Utils +from Mailman import Version +from Mailman.LockFile import TimeOutError +from Mailman.MemberAdaptor import BYBOUNCE, ENABLED +from Mailman.OldStyleMemberships import OldStyleMemberships +from Mailman.Queue.Switchboard import Switchboard +from Mailman.configuration import config +from Mailman.i18n import _ + +__i18n_templates__ = True + +FRESH = 0 +NOTFRESH = -1 + + + +def parseargs(): + parser = optparse.OptionParser(version=Version.MAILMAN_VERSION, + usage=_("""\ +Perform all necessary upgrades. + +%prog [options]""")) + parser.add_option('-f', '--force', + default=False, action='store_true', help=_("""\ +Force running the upgrade procedures. Normally, if the version number of the +installed Mailman matches the current version number (or a 'downgrade' is +detected), nothing will be done.""")) + parser.add_option('-C', '--config', + help=_('Alternative configuration file to use')) + opts, args = parser.parse_args() + if args: + parser.print_help() + print >> sys.stderr, _('Unexpected arguments') + sys.exit(1) + return parser, opts, args + + + +def calcversions(): + # Returns a tuple of (lastversion, thisversion). If the last version + # could not be determined, lastversion will be FRESH or NOTFRESH, + # depending on whether this installation appears to be fresh or not. The + # determining factor is whether there are files in the $var_prefix/logs + # subdir or not. The version numbers are HEX_VERSIONs. + # + # See if we stored the last updated version + lastversion = None + thisversion = config.HEX_VERSION + try: + fp = open(os.path.join(config.DATA_DIR, 'last_mailman_version')) + data = fp.read() + fp.close() + lastversion = int(data, 16) + except (IOError, ValueError): + pass + # + # try to figure out if this is a fresh install + if lastversion is None: + lastversion = FRESH + try: + if os.listdir(config.LOG_DIR): + lastversion = NOTFRESH + except OSError: + pass + return (lastversion, thisversion) + + + +def makeabs(relpath): + return os.path.join(config.PREFIX, relpath) + + +def make_varabs(relpath): + return os.path.join(config.VAR_PREFIX, relpath) + + + +def move_language_templates(mlist): + listname = mlist.internal_name() + print _('Fixing language templates: $listname') + # Mailman 2.1 has a new cascading search for its templates, defined and + # described in Utils.py:maketext(). Putting templates in the top level + # templates/ subdir or the lists/<listname> subdir is deprecated and no + # longer searched.. + # + # What this means is that most templates can live in the global templates/ + # subdirectory, and only needs to be copied into the list-, vhost-, or + # site-specific language directories when needed. + # + # Also, by default all standard (i.e. English) templates must now live in + # the templates/en directory. This update cleans up all the templates, + # deleting more-specific duplicates (as calculated by md5 checksums) in + # favor of more-global locations. + # + # First, get rid of any lists/<list> template or lists/<list>/en template + # that is identical to the global templates/* default. + for gtemplate in os.listdir(os.path.join(config.TEMPLATE_DIR, 'en')): + # BAW: get rid of old templates, e.g. admlogin.txt and + # handle_opts.html + try: + fp = open(os.path.join(config.TEMPLATE_DIR, gtemplate)) + except IOError, e: + if e.errno <> errno.ENOENT: + raise + # No global template + continue + gcksum = md5.new(fp.read()).digest() + fp.close() + # Match against the lists/<list>/* template + try: + fp = open(os.path.join(mlist.fullpath(), gtemplate)) + except IOError, e: + if e.errno <> errno.ENOENT: + raise + else: + tcksum = md5.new(fp.read()).digest() + fp.close() + if gcksum == tcksum: + os.unlink(os.path.join(mlist.fullpath(), gtemplate)) + # Match against the lists/<list>/*.prev template + try: + fp = open(os.path.join(mlist.fullpath(), gtemplate + '.prev')) + except IOError, e: + if e.errno <> errno.ENOENT: + raise + else: + tcksum = md5.new(fp.read()).digest() + fp.close() + if gcksum == tcksum: + os.unlink(os.path.join(mlist.fullpath(), gtemplate + '.prev')) + # Match against the lists/<list>/en/* templates + try: + fp = open(os.path.join(mlist.fullpath(), 'en', gtemplate)) + except IOError, e: + if e.errno <> errno.ENOENT: + raise + else: + tcksum = md5.new(fp.read()).digest() + fp.close() + if gcksum == tcksum: + os.unlink(os.path.join(mlist.fullpath(), 'en', gtemplate)) + # Match against the templates/* template + try: + fp = open(os.path.join(config.TEMPLATE_DIR, gtemplate)) + except IOError, e: + if e.errno <> errno.ENOENT: + raise + else: + tcksum = md5.new(fp.read()).digest() + fp.close() + if gcksum == tcksum: + os.unlink(os.path.join(config.TEMPLATE_DIR, gtemplate)) + # Match against the templates/*.prev template + try: + fp = open(os.path.join(config.TEMPLATE_DIR, gtemplate + '.prev')) + except IOError, e: + if e.errno <> errno.ENOENT: + raise + else: + tcksum = md5.new(fp.read()).digest() + fp.close() + if gcksum == tcksum: + os.unlink(os.path.join(config.TEMPLATE_DIR, + gtemplate + '.prev')) + + + +def dolist(listname): + mlist = MailList.MailList(listname, lock=False) + try: + mlist.Lock(0.5) + except TimeOutError: + print >> sys.stderr, _( + 'WARNING: could not acquire lock for list: $listname') + return 1 + # Sanity check the invariant that every BYBOUNCE disabled member must have + # bounce information. Some earlier betas broke this. BAW: we're + # submerging below the MemberAdaptor interface, so skip this if we're not + # using OldStyleMemberships. + if isinstance(mlist._memberadaptor, OldStyleMemberships): + noinfo = {} + for addr, (reason, when) in mlist.delivery_status.items(): + if reason == BYBOUNCE and not mlist.bounce_info.has_key(addr): + noinfo[addr] = reason, when + # What to do about these folks with a BYBOUNCE delivery status and no + # bounce info? This number should be very small, and I think it's + # fine to simple re-enable them and let the bounce machinery + # re-disable them if necessary. + n = len(noinfo) + if n > 0: + print _( + 'Resetting $n BYBOUNCEs disabled addrs with no bounce info') + for addr in noinfo.keys(): + mlist.setDeliveryStatus(addr, ENABLED) + + # Update the held requests database + print _("""Updating the held requests database.""") + mlist._UpdateRecords() + + mbox_dir = make_varabs('archives/private/%s.mbox' % (listname)) + mbox_file = make_varabs('archives/private/%s.mbox/%s' % (listname, + listname)) + o_pub_mbox_file = make_varabs('archives/public/%s' % (listname)) + o_pri_mbox_file = make_varabs('archives/private/%s' % (listname)) + html_dir = o_pri_mbox_file + o_html_dir = makeabs('public_html/archives/%s' % (listname)) + # Make the mbox directory if it's not there. + if not os.path.exists(mbox_dir): + ou = os.umask(0) + os.mkdir(mbox_dir, 02775) + os.umask(ou) + else: + # This shouldn't happen, but hey, just in case + if not os.path.isdir(mbox_dir): + print _("""\ +For some reason, $mbox_dir exists as a file. This won't work with b6, so I'm +renaming it to ${mbox_dir}.tmp and proceeding.""") + os.rename(mbox_dir, "%s.tmp" % (mbox_dir)) + ou = os.umask(0) + os.mkdir(mbox_dir, 02775) + os.umask(ou) + # Move any existing mboxes around, but watch out for both a public and a + # private one existing + if os.path.isfile(o_pri_mbox_file) and os.path.isfile(o_pub_mbox_file): + if mlist.archive_private: + print _("""\ + +$listname has both public and private mbox archives. Since this list +currently uses private archiving, I'm installing the private mbox archive -- +$o_pri_mbox_file -- as the active archive, and renaming + $o_pub_mbox_file +to + ${o_pub_mbox_file}.preb6 + +You can integrate that into the archives if you want by using the 'arch' +script. +""") % (mlist._internal_name, o_pri_mbox_file, o_pub_mbox_file, + o_pub_mbox_file) + os.rename(o_pub_mbox_file, "%s.preb6" % (o_pub_mbox_file)) + else: + print _("""\ +$mlist._internal_name has both public and private mbox archives. Since this +list currently uses public archiving, I'm installing the public mbox file +archive file ($o_pub_mbox_file) as the active one, and renaming +$o_pri_mbox_file to ${o_pri_mbox_file}.preb6 + +You can integrate that into the archives if you want by using the 'arch' +script. +""") + os.rename(o_pri_mbox_file, "%s.preb6" % (o_pri_mbox_file)) + # Move private archive mbox there if it's around + # and take into account all sorts of absurdities + print _('- updating old private mbox file') + if os.path.exists(o_pri_mbox_file): + if os.path.isfile(o_pri_mbox_file): + os.rename(o_pri_mbox_file, mbox_file) + elif not os.path.isdir(o_pri_mbox_file): + newname = "%s.mm_install-dunno_what_this_was_but_its_in_the_way" \ + % o_pri_mbox_file + os.rename(o_pri_mbox_file, newname) + print _("""\ + unknown file in the way, moving + $o_pri_mbox_file + to + $newname""") + else: + # directory + print _("""\ + looks like you have a really recent CVS installation... + you're either one brave soul, or you already ran me""") + # Move public archive mbox there if it's around + # and take into account all sorts of absurdities. + print _('- updating old public mbox file') + if os.path.exists(o_pub_mbox_file): + if os.path.isfile(o_pub_mbox_file): + os.rename(o_pub_mbox_file, mbox_file) + elif not os.path.isdir(o_pub_mbox_file): + newname = "%s.mm_install-dunno_what_this_was_but_its_in_the_way" \ + % o_pub_mbox_file + os.rename(o_pub_mbox_file, newname) + print _("""\ + unknown file in the way, moving + $o_pub_mbox_file + to + $newname""") + else: # directory + print _("""\ + looks like you have a really recent CVS installation... + you're either one brave soul, or you already ran me""") + # Move the html archives there + if os.path.isdir(o_html_dir): + os.rename(o_html_dir, html_dir) + # chmod the html archives + os.chmod(html_dir, 02775) + # BAW: Is this still necessary?! + mlist.Save() + # Check to see if pre-b4 list-specific templates are around + # and move them to the new place if there's not already + # a new one there + tmpl_dir = os.path.join(config.PREFIX, "templates") + list_dir = os.path.join(config.PREFIX, "lists") + b4_tmpl_dir = os.path.join(tmpl_dir, mlist._internal_name) + new_tmpl_dir = os.path.join(list_dir, mlist._internal_name) + if os.path.exists(b4_tmpl_dir): + print _("""\ +- This list looks like it might have <= b4 list templates around""") + for f in os.listdir(b4_tmpl_dir): + o_tmpl = os.path.join(b4_tmpl_dir, f) + n_tmpl = os.path.join(new_tmpl_dir, f) + if os.path.exists(o_tmpl): + if not os.path.exists(n_tmpl): + os.rename(o_tmpl, n_tmpl) + print _('- moved $o_tmpl to $n_tmpl') + else: + print _("""\ +- both $o_tmpl and $n_tmpl exist, leaving untouched""") + else: + print _("""\ +- $o_tmpl doesn't exist, leaving untouched""") + # Move all the templates to the en language subdirectory as required for + # Mailman 2.1 + move_language_templates(mlist) + # Avoid eating filehandles with the list lockfiles + mlist.Unlock() + return 0 + + + +def archive_path_fixer(unused_arg, dir, files): + # Passed to os.path.walk to fix the perms on old html archives. + for f in files: + abs = os.path.join(dir, f) + if os.path.isdir(abs): + if f == "database": + os.chmod(abs, 02770) + else: + os.chmod(abs, 02775) + elif os.path.isfile(abs): + os.chmod(abs, 0664) + + +def remove_old_sources(module): + # Also removes old directories. + src = '%s/%s' % (config.PREFIX, module) + pyc = src + "c" + if os.path.isdir(src): + print _('removing directory $src and everything underneath') + shutil.rmtree(src) + elif os.path.exists(src): + print _('removing $src') + try: + os.unlink(src) + except os.error, rest: + print _("Warning: couldn't remove $src -- $rest") + if module.endswith('.py') and os.path.exists(pyc): + try: + os.unlink(pyc) + except OSError, rest: + print _("couldn't remove old file $pyc -- $rest") + + + +def update_qfiles(): + print _('updating old qfiles') + prefix = `time.time()` + '+' + # Be sure the qfiles/in directory exists (we don't really need the + # switchboard object, but it's convenient for creating the directory). + sb = Switchboard(config.INQUEUE_DIR) + for filename in os.listdir(config.QUEUE_DIR): + # Updating means just moving the .db and .msg files to qfiles/in where + # it should be dequeued, converted, and processed normally. + if os.path.splitext(filename) == '.msg': + oldmsgfile = os.path.join(config.QUEUE_DIR, filename) + newmsgfile = os.path.join(config.INQUEUE_DIR, prefix + filename) + os.rename(oldmsgfile, newmsgfile) + elif os.path.splitext(filename) == '.db': + olddbfile = os.path.join(config.QUEUE_DIR, filename) + newdbfile = os.path.join(config.INQUEUE_DIR, prefix + filename) + os.rename(olddbfile, newdbfile) + # Now update for the Mailman 2.1.5 qfile format. For every filebase in + # the qfiles/* directories that has both a .pck and a .db file, pull the + # data out and re-queue them. + for dirname in os.listdir(config.QUEUE_DIR): + dirpath = os.path.join(config.QUEUE_DIR, dirname) + if dirpath == config.BADQUEUE_DIR: + # The files in qfiles/bad can't possibly be pickles + continue + sb = Switchboard(dirpath) + try: + for filename in os.listdir(dirpath): + filepath = os.path.join(dirpath, filename) + filebase, ext = os.path.splitext(filepath) + # Handle the .db metadata files as part of the handling of the + # .pck or .msg message files. + if ext not in ('.pck', '.msg'): + continue + msg, data = dequeue(filebase) + if msg is not None and data is not None: + sb.enqueue(msg, data) + except EnvironmentError, e: + if e.errno <> errno.ENOTDIR: + raise + print _('Warning! Not a directory: $dirpath') + + + +# Implementations taken from the pre-2.1.5 Switchboard +def ext_read(filename): + fp = open(filename) + d = marshal.load(fp) + # Update from version 2 files + if d.get('version', 0) == 2: + del d['filebase'] + # Do the reverse conversion (repr -> float) + for attr in ['received_time']: + try: + sval = d[attr] + except KeyError: + pass + else: + # Do a safe eval by setting up a restricted execution + # environment. This may not be strictly necessary since we + # know they are floats, but it can't hurt. + d[attr] = eval(sval, {'__builtins__': {}}) + fp.close() + return d + + +def dequeue(filebase): + # Calculate the .db and .msg filenames from the given filebase. + msgfile = os.path.join(filebase + '.msg') + pckfile = os.path.join(filebase + '.pck') + dbfile = os.path.join(filebase + '.db') + # Now we are going to read the message and metadata for the given + # filebase. We want to read things in this order: first, the metadata + # file to find out whether the message is stored as a pickle or as + # plain text. Second, the actual message file. However, we want to + # first unlink the message file and then the .db file, because the + # qrunner only cues off of the .db file + msg = None + try: + data = ext_read(dbfile) + os.unlink(dbfile) + except EnvironmentError, e: + if e.errno <> errno.ENOENT: + raise + data = {} + # Between 2.1b4 and 2.1b5, the `rejection-notice' key in the metadata + # was renamed to `rejection_notice', since dashes in the keys are not + # supported in METAFMT_ASCII. + if data.has_key('rejection-notice'): + data['rejection_notice'] = data['rejection-notice'] + del data['rejection-notice'] + msgfp = None + try: + try: + msgfp = open(pckfile) + msg = cPickle.load(msgfp) + os.unlink(pckfile) + except EnvironmentError, e: + if e.errno <> errno.ENOENT: raise + msgfp = None + try: + msgfp = open(msgfile) + msg = email.message_from_file(msgfp, Message.Message) + os.unlink(msgfile) + except EnvironmentError, e: + if e.errno <> errno.ENOENT: raise + except (email.Errors.MessageParseError, ValueError), e: + # This message was unparsable, most likely because its + # MIME encapsulation was broken. For now, there's not + # much we can do about it. + print _('message is unparsable: $filebase') + msgfp.close() + msgfp = None + if config.QRUNNER_SAVE_BAD_MESSAGES: + # Cheapo way to ensure the directory exists w/ the + # proper permissions. + sb = Switchboard(config.BADQUEUE_DIR) + os.rename(msgfile, os.path.join( + config.BADQUEUE_DIR, filebase + '.txt')) + else: + os.unlink(msgfile) + msg = data = None + except EOFError: + # For some reason the pckfile was empty. Just delete it. + print _('Warning! Deleting empty .pck file: $pckfile') + os.unlink(pckfile) + finally: + if msgfp: + msgfp.close() + return msg, data + + + +def update_pending(): + file20 = os.path.join(config.DATA_DIR, 'pending_subscriptions.db') + file214 = os.path.join(config.DATA_DIR, 'pending.pck') + db = None + # Try to load the Mailman 2.0 file + try: + fp = open(file20) + except IOError, e: + if e.errno <> errno.ENOENT: + raise + else: + print _('Updating Mailman 2.0 pending_subscriptions.db database') + db = marshal.load(fp) + # Convert to the pre-Mailman 2.1.5 format + db = Pending._update(db) + if db is None: + # Try to load the Mailman 2.1.x where x < 5, file + try: + fp = open(file214) + except IOError, e: + if e.errno <> errno.ENOENT: + raise + else: + print _('Updating Mailman 2.1.4 pending.pck database') + db = cPickle.load(fp) + if db is None: + print _('Nothing to do.') + return + # Now upgrade the database to the 2.1.5 format. Each list now has its own + # pending.pck file, but only the RE_ENABLE operation actually recorded the + # listname in the request. For the SUBSCRIPTION, UNSUBSCRIPTION, and + # CHANGE_OF_ADDRESS operations, we know the address of the person making + # the request so we can repend this request just for the lists the person + # is a member of. For the HELD_MESSAGE operation, we can check the list's + # requests.pck file for correlation. Evictions will take care of any + # misdirected pendings. + reenables_by_list = {} + addrops_by_address = {} + holds_by_id = {} + subs_by_address = {} + for key, val in db.items(): + if key in ('evictions', 'version'): + continue + try: + op = val[0] + data = val[1:] + except (IndexError, ValueError): + print _('Ignoring bad pended data: $key: $val') + continue + if op in (Pending.UNSUBSCRIPTION, Pending.CHANGE_OF_ADDRESS): + # data[0] is the address being unsubscribed + addrops_by_address.setdefault(data[0], []).append((key, val)) + elif op == Pending.SUBSCRIPTION: + # data[0] is a UserDesc object + addr = data[0].address + subs_by_address.setdefault(addr, []).append((key, val)) + elif op == Pending.RE_ENABLE: + # data[0] is the mailing list's internal name + reenables_by_list.setdefault(data[0], []).append((key, val)) + elif op == Pending.HELD_MESSAGE: + # data[0] is the hold id. There better only be one entry per id + id = data[0] + if holds_by_id.has_key(id): + print _('WARNING: Ignoring duplicate pending ID: $id.') + else: + holds_by_id[id] = (key, val) + # Now we have to lock every list and re-pend all the appropriate + # requests. Note that this will reset all the expiration dates, but that + # should be fine. + for listname in Utils.list_names(): + mlist = MailList.MailList(listname) + # This is not the most efficient way to do this because it loads and + # saves the pending.pck file each time. :( + try: + for cookie, data in reenables_by_list.get(listname, []): + mlist.pend_repend(cookie, data) + for id, (cookie, data) in holds_by_id.items(): + try: + rec = mlist.GetRecord(id) + except KeyError: + # Not for this list + pass + else: + mlist.pend_repend(cookie, data) + del holds_by_id[id] + for addr, recs in subs_by_address.items(): + # We shouldn't have a subscription confirmation if the address + # is already a member of the mailing list. + if mlist.isMember(addr): + continue + for cookie, data in recs: + mlist.pend_repend(cookie, data) + for addr, recs in addrops_by_address.items(): + # We shouldn't have unsubscriptions or change of address + # requests for addresses which aren't members of the list. + if not mlist.isMember(addr): + continue + for cookie, data in recs: + mlist.pend_repend(cookie, data) + mlist.Save() + finally: + mlist.Unlock() + try: + os.unlink(file20) + except OSError, e: + if e.errno <> errno.ENOENT: + raise + try: + os.unlink(file214) + except OSError, e: + if e.errno <> errno.ENOENT: + raise + + + +def main(): + parser, opts, args = parseargs() + config.load(opts.config) + + # calculate the versions + lastversion, thisversion = calcversions() + hexlversion = hex(lastversion) + hextversion = hex(thisversion) + if lastversion == thisversion and not opts.force: + # nothing to do + print _('No updates are necessary.') + sys.exit(0) + if lastversion > thisversion and not opts.force: + print _("""\ +Downgrade detected, from version $hexlversion to version $hextversion +This is probably not safe. +Exiting.""") + sys.exit(1) + print _('Upgrading from version $hexlversion to $hextversion') + errors = 0 + # get rid of old stuff + print _('getting rid of old source files') + for mod in ('Mailman/Archiver.py', 'Mailman/HyperArch.py', + 'Mailman/HyperDatabase.py', 'Mailman/pipermail.py', + 'Mailman/smtplib.py', 'Mailman/Cookie.py', + 'bin/update_to_10b6', 'scripts/mailcmd', + 'scripts/mailowner', 'mail/wrapper', 'Mailman/pythonlib', + 'cgi-bin/archives', 'Mailman/MailCommandHandler'): + remove_old_sources(mod) + listnames = Utils.list_names() + if not listnames: + print _('no lists == nothing to do, exiting') + return + # + # for people with web archiving, make sure the directories + # in the archiving are set with proper perms for b6. + # + if os.path.isdir("%s/public_html/archives" % config.PREFIX): + print _("""\ +fixing all the perms on your old html archives to work with b6 +If your archives are big, this could take a minute or two...""") + os.path.walk("%s/public_html/archives" % config.PREFIX, + archive_path_fixer, "") + print _('done') + for listname in listnames: + print _('Updating mailing list: $listname') + errors = errors + dolist(listname) + print + print _('Updating Usenet watermarks') + wmfile = os.path.join(config.DATA_DIR, 'gate_watermarks') + try: + fp = open(wmfile) + except IOError: + print _('- nothing to update here') + else: + d = marshal.load(fp) + fp.close() + for listname in d.keys(): + if listname not in listnames: + # this list no longer exists + continue + mlist = MailList.MailList(listname, lock=0) + try: + mlist.Lock(0.5) + except TimeOutError: + print >> sys.stderr, _( + 'WARNING: could not acquire lock for list: $listname') + errors = errors + 1 + else: + # Pre 1.0b7 stored 0 in the gate_watermarks file to indicate + # that no gating had been done yet. Without coercing this to + # None, the list could now suddenly get flooded. + mlist.usenet_watermark = d[listname] or None + mlist.Save() + mlist.Unlock() + os.unlink(wmfile) + print _('- usenet watermarks updated and gate_watermarks removed') + # In Mailman 2.1, the pending database format and file name changed, but + # in Mailman 2.1.5 it changed again. This should update all existing + # files to the 2.1.5 format. + update_pending() + # In Mailman 2.1, the qfiles directory has a different structure and a + # different content. Also, in Mailman 2.1.5 we collapsed the message + # files from separate .msg (pickled Message objects) and .db (marshalled + # dictionaries) to a shared .pck file containing two pickles. + update_qfiles() + # This warning was necessary for the upgrade from 1.0b9 to 1.0b10. + # There's no good way of figuring this out for releases prior to 2.0beta2 + # :( + if lastversion == NOTFRESH: + print _(""" + +NOTE NOTE NOTE NOTE NOTE + + You are upgrading an existing Mailman installation, but I can't tell what + version you were previously running. + + If you are upgrading from Mailman 1.0b9 or earlier you will need to + manually update your mailing lists. For each mailing list you need to + copy the file templates/options.html lists/<listname>/options.html. + + However, if you have edited this file via the Web interface, you will have + to merge your changes into this file, otherwise you will lose your + changes. + +NOTE NOTE NOTE NOTE NOTE + +""") + if not errors: + # Record the version we just upgraded to + fp = open(os.path.join(config.DATA_DIR, 'last_mailman_version'), 'w') + fp.write(hex(config.HEX_VERSION) + '\n') + fp.close() + else: + lockdir = config.LOCK_DIR + print _('''\ + +ERROR: + +The locks for some lists could not be acquired. This means that either +Mailman was still active when you upgraded, or there were stale locks in the +$lockdir directory. + +You must put Mailman into a quiescent state and remove all stale locks, then +re-run "make update" manually. See the INSTALL and UPGRADE files for details. +''') diff --git a/Mailman/configuration.py b/Mailman/configuration.py new file mode 100644 index 000000000..cf3af03f3 --- /dev/null +++ b/Mailman/configuration.py @@ -0,0 +1,103 @@ +# Copyright (C) 2006 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +"""Configuration file loading and management.""" + +import os +import errno + +from Mailman import Defaults + +_missing = object() + + + +class Configuration(object): + def load(self, filename=None): + # Load the configuration from the named file, or if not given, search + # in VAR_PREFIX for an etc/mailman.cfg file. If that file is missing, + # use Mailman/mm_cfg.py for backward compatibility. + # + # Whatever you find, create a namespace and execfile that file in it. + # The values in that namespace are exposed as attributes on this + # Configuration instance. + if filename is None: + filename = os.path.join(Defaults.VAR_PREFIX, 'etc', 'mailman.cfg') + # Set up the execfile namespace + ns = Defaults.__dict__.copy() + # Prune a few things + del ns['__file__'] + del ns['__name__'] + del ns['__doc__'] + # Attempt our first choice + path = os.path.abspath(os.path.expanduser(filename)) + try: + execfile(path, ns, ns) + except EnvironmentError, e: + if e.errno <> errno.ENOENT: + raise + # The file didn't exist, so try mm_cfg.py + from Mailman import mm_cfg + ns = mm_cfg.__dict__.copy() + # Pull out the defaults + PREFIX = ns['PREFIX'] + VAR_PREFIX = ns['VAR_PREFIX'] + EXEC_PREFIX = ns['EXEC_PREFIX'] + # Now that we've loaded all the configuration files we're going to + # load, set up some useful directories. + self.LIST_DATA_DIR = os.path.join(VAR_PREFIX, 'lists') + self.LOG_DIR = os.path.join(VAR_PREFIX, 'logs') + self.LOCK_DIR = lockdir = os.path.join(VAR_PREFIX, 'locks') + self.DATA_DIR = datadir = os.path.join(VAR_PREFIX, 'data') + self.ETC_DIR = etcdir = os.path.join(VAR_PREFIX, 'etc') + self.SPAM_DIR = os.path.join(VAR_PREFIX, 'spam') + self.WRAPPER_DIR = os.path.join(EXEC_PREFIX, 'mail') + self.BIN_DIR = os.path.join(PREFIX, 'bin') + self.SCRIPTS_DIR = os.path.join(PREFIX, 'scripts') + self.TEMPLATE_DIR = os.path.join(PREFIX, 'templates') + self.MESSAGES_DIR = os.path.join(PREFIX, 'messages') + self.PUBLIC_ARCHIVE_FILE_DIR = os.path.join(VAR_PREFIX, + 'archives', 'public') + self.PRIVATE_ARCHIVE_FILE_DIR = os.path.join(VAR_PREFIX, + 'archives', 'private') + # Directories used by the qrunner subsystem + self.QUEUE_DIR = qdir = os.path.join(VAR_PREFIX, 'qfiles') + self.INQUEUE_DIR = os.path.join(qdir, 'in') + self.OUTQUEUE_DIR = os.path.join(qdir, 'out') + self.CMDQUEUE_DIR = os.path.join(qdir, 'commands') + self.BOUNCEQUEUE_DIR = os.path.join(qdir, 'bounces') + self.NEWSQUEUE_DIR = os.path.join(qdir, 'news') + self.ARCHQUEUE_DIR = os.path.join(qdir, 'archive') + self.SHUNTQUEUE_DIR = os.path.join(qdir, 'shunt') + self.VIRGINQUEUE_DIR = os.path.join(qdir, 'virgin') + self.BADQUEUE_DIR = os.path.join(qdir, 'bad') + self.RETRYQUEUE_DIR = os.path.join(qdir, 'retry') + self.MAILDIR_DIR = os.path.join(qdir, 'maildir') + # Other useful files + self.PIDFILE = os.path.join(datadir, + 'master-qrunner.pid') + self.SITE_PW_FILE = os.path.join(datadir, 'adm.pw') + self.LISTCREATOR_PW_FILE = os.path.join(datadir, 'creator.pw') + self.CONFIG_FILE = os.path.join(etcdir, 'mailman.cfg') + self.LOCK_FILE = os.path.join(lockdir, 'master-qrunner') + # Now update our dict so attribute syntax just works + self.__dict__.update(ns) + + + +config = Configuration() + diff --git a/Mailman/i18n.py b/Mailman/i18n.py index 7d36e22d3..69f9801d1 100644 --- a/Mailman/i18n.py +++ b/Mailman/i18n.py @@ -20,8 +20,8 @@ import time import string import gettext -from Mailman import mm_cfg from Mailman.SafeDict import SafeDict +from Mailman.configuration import config _translation = None _missing = object() @@ -47,7 +47,7 @@ def set_language(language=None): if language is not None: language = [language] try: - _translation = gettext.translation('mailman', mm_cfg.MESSAGES_DIR, + _translation = gettext.translation('mailman', config.MESSAGES_DIR, language) except IOError: # The selected language was not installed in messages, so fall back to @@ -67,7 +67,7 @@ def set_translation(translation): # Set up the global translation based on environment variables. Mostly used # for command line scripts. if _translation is None: - set_language() + _translation = gettext.NullTranslations() diff --git a/Mailman/loginit.py b/Mailman/loginit.py index bcb016f08..fac01c0fb 100644 --- a/Mailman/loginit.py +++ b/Mailman/loginit.py @@ -24,7 +24,7 @@ import os import codecs import logging -from Mailman import mm_cfg +from Mailman.configuration import config FMT = '%(asctime)s (%(process)d) %(message)s' DATEFMT = '%b %d %H:%M:%S %Y' @@ -116,7 +116,7 @@ def initialize(propagate=False): # Propagation to the root logger is how we handle logging to stderr # when the qrunners are not run as a subprocess of mailmanctl. log.propagate = propagate - handler = ReopenableFileHandler(os.path.join(mm_cfg.LOG_DIR, logger)) + handler = ReopenableFileHandler(os.path.join(config.LOG_DIR, logger)) _handlers.append(handler) handler.setFormatter(formatter) log.addHandler(handler) diff --git a/Mailman/testing/emailbase.py b/Mailman/testing/emailbase.py index 115ea6839..6c290c8e5 100644 --- a/Mailman/testing/emailbase.py +++ b/Mailman/testing/emailbase.py @@ -17,14 +17,18 @@ """Base class for tests that email things.""" +import os import smtpd import socket import asyncore +import tempfile import subprocess from Mailman import mm_cfg from Mailman.testing.base import TestBase +TESTPORT = 10825 + MSGTEXT = None @@ -49,15 +53,23 @@ class SinkServer(smtpd.SMTPServer): class EmailBase(TestBase): def setUp(self): TestBase.setUp(self) - # Find an unused non-root requiring port to listen on - oldport = mm_cfg.SMTPPORT - mm_cfg.SMTPPORT = port = 10825 + # Find an unused non-root requiring port to listen on. Set up a + # configuration file that causes the underlying outgoing runner to use + # the same port, then start Mailman. + fd, self._configfile = tempfile.mkstemp(suffix='.cfg') + print 'config file:', self._configfile + fp = os.fdopen(fd, 'w') + print >> fp, 'SMTPPORT =', TESTPORT + fp.close() # Second argument is ignored. - self._server = SinkServer(('localhost', port), None) + self._server = SinkServer(('localhost', TESTPORT), None) + os.system('bin/mailmanctl start') def tearDown(self): + os.system('bin/mailmanctl stop') self._server.close() TestBase.tearDown(self) + os.remove(self._configfile) def _readmsg(self): global MSGTEXT diff --git a/Makefile.in b/Makefile.in index aa49d9310..9c5fe7807 100644 --- a/Makefile.in +++ b/Makefile.in @@ -48,7 +48,7 @@ VAR_DIRS= \ archives/private archives/public ARCH_INDEP_DIRS= \ - bin templates scripts cron pythonlib \ + bin etc templates scripts cron pythonlib \ Mailman Mailman/bin Mailman/Cgi Mailman/Archiver \ Mailman/Handlers Mailman/Queue Mailman/Bouncers \ Mailman/MTA Mailman/Gui Mailman/Commands messages icons \ diff --git a/bin/Makefile.in b/bin/Makefile.in index 40fba3ed4..37084d1f7 100644 --- a/bin/Makefile.in +++ b/bin/Makefile.in @@ -45,17 +45,18 @@ SCRIPTSDIR= $(prefix)/bin SHELL= /bin/sh SCRIPTS= mmshell \ - remove_members clone_member update \ + remove_members clone_member \ sync_members check_db withlist check_perms \ config_list dumpdb cleanarch \ - list_admins genaliases mailmanctl qrunner \ + list_admins genaliases \ fix_url.py convert.py transcheck \ msgfmt.py discard \ reset_pw.py templ2pot.py po2templ.py LN_SCRIPTS= add_members arch change_pw find_member inject list_lists \ - list_members list_owners mmsitepass newlist rmlist \ - show_qfiles show_mm_cfg testall unshunt version + list_members list_owners mailmanctl mmsitepass newlist \ + qrunner rmlist show_qfiles show_mm_cfg testall unshunt \ + update version BUILDDIR= ../build/bin diff --git a/bin/mailmanctl b/bin/mailmanctl index 36c3ab545..e69de29bb 100644 --- a/bin/mailmanctl +++ b/bin/mailmanctl @@ -1,545 +0,0 @@ -#! @PYTHON@ - -# Copyright (C) 2001-2006 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""Primary start-up and shutdown script for Mailman's qrunner daemon. - -This script starts, stops, and restarts the main Mailman queue runners, making -sure that the various long-running qrunners are still alive and kicking. It -does this by forking and exec'ing the qrunners and waiting on their pids. -When it detects a subprocess has exited, it may restart it. - -The qrunners respond to SIGINT, SIGTERM, and SIGHUP. SIGINT and SIGTERM both -cause the qrunners to exit cleanly, but the master will only restart qrunners -that have exited due to a SIGINT. SIGHUP causes the master and the qrunners -to close their log files, and reopen then upon the next printed message. - -The master also responds to SIGINT, SIGTERM, and SIGHUP, which it simply -passes on to the qrunners (note that the master will close and reopen its own -log files on receipt of a SIGHUP). The master also leaves its own process id -in the file data/master-qrunner.pid but you normally don't need to use this -pid directly. The `start', `stop', `restart', and `reopen' commands handle -everything for you. - -Usage: %(PROGRAM)s [options] [ start | stop | restart | reopen ] - -Options: - - -n/--no-restart - Don't restart the qrunners when they exit because of an error or a - SIGINT. They are never restarted if they exit in response to a - SIGTERM. Use this only for debugging. Only useful if the `start' - command is given. - - -u/--run-as-user - Normally, this script will refuse to run if the user id and group id - are not set to the `mailman' user and group (as defined when you - configured Mailman). If run as root, this script will change to this - user and group before the check is made. - - This can be inconvenient for testing and debugging purposes, so the -u - flag means that the step that sets and checks the uid/gid is skipped, - and the program is run as the current user and group. This flag is - not recommended for normal production environments. - - Note though, that if you run with -u and are not in the mailman group, - you may have permission problems, such as begin unable to delete a - list's archives through the web. Tough luck! - - -s/--stale-lock-cleanup - If mailmanctl finds an existing master lock, it will normally exit - with an error message. With this option, mailmanctl will perform an - extra level of checking. If a process matching the host/pid described - in the lock file is running, mailmanctl will still exit, but if no - matching process is found, mailmanctl will remove the apparently stale - lock and make another attempt to claim the master lock. - - -q/--quiet - Don't print status messages. Error messages are still printed to - standard error. - - -h/--help - Print this message and exit. - -Commands: - - start - Start the master daemon and all qrunners. Prints a message and - exits if the master daemon is already running. - - stop - Stops the master daemon and all qrunners. After stopping, no - more messages will be processed. - - restart - Restarts the qrunners, but not the master process. Use this - whenever you upgrade or update Mailman so that the qrunners will - use the newly installed code. - - reopen - This will close all log files, causing them to be re-opened the - next time a message is written to them -""" - -import os -import grp -import pwd -import sys -import errno -import getopt -import signal -import socket -import logging - -import paths -from Mailman import Errors -from Mailman import LockFile -from Mailman import Utils -from Mailman import loginit -from Mailman import mm_cfg -from Mailman.MailList import MailList -from Mailman.i18n import _ - -PROGRAM = sys.argv[0] -COMMASPACE = ', ' -DOT = '.' - -# Locking contantsa -LOCKFILE = os.path.join(mm_cfg.LOCK_DIR, 'master-qrunner') -# Since we wake up once per day and refresh the lock, the LOCK_LIFETIME -# needn't be (much) longer than SNOOZE. We pad it 6 hours just to be safe. -LOCK_LIFETIME = mm_cfg.days(1) + mm_cfg.hours(6) -SNOOZE = mm_cfg.days(1) -MAX_RESTARTS = 10 - -loginit.initialize() -elog = logging.getLogger('mailman.error') -qlog = logging.getLogger('mailman.qrunner') - - - -def usage(code, msg=''): - if code: - fd = sys.stderr - else: - fd = sys.stdout - print >> fd, _(__doc__) - if msg: - print >> fd, msg - sys.exit(code) - - - -def kill_watcher(sig): - try: - fp = open(mm_cfg.PIDFILE) - pidstr = fp.read() - fp.close() - pid = int(pidstr.strip()) - except (IOError, ValueError), e: - # For i18n convenience - pidfile = mm_cfg.PIDFILE - print >> sys.stderr, _('PID unreadable in: %(pidfile)s') - print >> sys.stderr, e - print >> sys.stderr, _('Is qrunner even running?') - return - try: - os.kill(pid, sig) - except OSError, e: - if e.errno <> errno.ESRCH: raise - print >> sys.stderr, _('No child with pid: %(pid)s') - print >> sys.stderr, e - print >> sys.stderr, _('Stale pid file removed.') - os.unlink(mm_cfg.PIDFILE) - - - -def get_lock_data(): - # Return the hostname, pid, and tempfile - fp = open(LOCKFILE) - try: - filename = os.path.split(fp.read().strip())[1] - finally: - fp.close() - parts = filename.split('.') - hostname = DOT.join(parts[1:-1]) - pid = int(parts[-1]) - return hostname, int(pid), filename - - -def qrunner_state(): - # 1 if proc exists on host (but is it qrunner? ;) - # 0 if host matches but no proc - # hostname if hostname doesn't match - hostname, pid, tempfile = get_lock_data() - if hostname <> socket.gethostname(): - return hostname - # Find out if the process exists by calling kill with a signal 0. - try: - os.kill(pid, 0) - except OSError, e: - if e.errno <> errno.ESRCH: - raise - return 0 - return 1 - - -def acquire_lock_1(force): - # Be sure we can acquire the master qrunner lock. If not, it means some - # other master qrunner daemon is already going. - lock = LockFile.LockFile(LOCKFILE, LOCK_LIFETIME) - try: - lock.lock(0.1) - return lock - except LockFile.TimeOutError: - if not force: - raise - # Force removal of lock first - lock._disown() - hostname, pid, tempfile = get_lock_data() - os.unlink(LOCKFILE) - os.unlink(os.path.join(mm_cfg.LOCK_DIR, tempfile)) - return acquire_lock_1(force=0) - - -def acquire_lock(force): - try: - lock = acquire_lock_1(force) - return lock - except LockFile.TimeOutError: - status = qrunner_state() - if status == 1: - # host matches and proc exists - print >> sys.stderr, _("""\ -The master qrunner lock could not be acquired because it appears as if another -master qrunner is already running. -""") - elif status == 0: - # host matches but no proc - print >> sys.stderr, _("""\ -The master qrunner lock could not be acquired. It appears as though there is -a stale master qrunner lock. Try re-running mailmanctl with the -s flag. -""") - else: - # host doesn't even match - print >> sys.stderr, _("""\ -The master qrunner lock could not be acquired, because it appears as if some -process on some other host may have acquired it. We can't test for stale -locks across host boundaries, so you'll have to do this manually. Or, if you -know the lock is stale, re-run mailmanctl with the -s flag. - -Lock file: %(LOCKFILE)s -Lock host: %(status)s - -Exiting.""") - - - -def start_runner(qrname, slice, count): - pid = os.fork() - if pid: - # parent - return pid - # child - # - # Craft the command line arguments for the exec() call. - rswitch = '--runner=%s:%d:%d' % (qrname, slice, count) - exe = os.path.join(mm_cfg.BIN_DIR, 'qrunner') - # mm_cfg.PYTHON, which is the absolute path to the Python interpreter, - # must be given as argv[0] due to Python's library search algorithm. - os.execl(mm_cfg.PYTHON, mm_cfg.PYTHON, exe, rswitch, '-s') - # Should never get here - raise RuntimeError, 'os.execl() failed' - - -def start_all_runners(): - kids = {} - for qrname, count in mm_cfg.QRUNNERS: - for slice in range(count): - # queue runner name, slice, numslices, restart count - info = (qrname, slice, count, 0) - pid = start_runner(qrname, slice, count) - kids[pid] = info - return kids - - - -def check_for_site_list(): - sitelistname = mm_cfg.MAILMAN_SITE_LIST - try: - sitelist = MailList(sitelistname, lock=False) - except Errors.MMUnknownListError: - print >> sys.stderr, _('Site list is missing: %(sitelistname)s') - elog.error('Site list is missing: %s', mm_cfg.MAILMAN_SITE_LIST) - sys.exit(1) - - -def check_privs(): - # If we're running as root (uid == 0), coerce the uid and gid to that - # which Mailman was configured for, and refuse to run if we didn't coerce - # the uid/gid. - gid = grp.getgrnam(mm_cfg.MAILMAN_GROUP)[2] - uid = pwd.getpwnam(mm_cfg.MAILMAN_USER)[2] - myuid = os.getuid() - if myuid == 0: - # Set the process's supplimental groups. - groups = [x[2] for x in grp.getgrall() if mm_cfg.MAILMAN_USER in x[3]] - groups.append(gid) - os.setgroups(groups) - os.setgid(gid) - os.setuid(uid) - elif myuid <> uid: - name = mm_cfg.MAILMAN_USER - usage(1, _( - 'Run this program as root or as the %(name)s user, or use -u.')) - - - -def main(): - global quiet - try: - opts, args = getopt.getopt(sys.argv[1:], 'hnusq', - ['help', 'no-start', 'run-as-user', - 'stale-lock-cleanup', 'quiet']) - except getopt.error, msg: - usage(1, msg) - - restart = 1 - checkprivs = 1 - force = 0 - quiet = 0 - for opt, arg in opts: - if opt in ('-h', '--help'): - usage(0) - elif opt in ('-n', '--no-restart'): - restart = 0 - elif opt in ('-u', '--run-as-user'): - checkprivs = 0 - elif opt in ('-s', '--stale-lock-cleanup'): - force = 1 - elif opt in ('-q', '--quiet'): - quiet = 1 - - if len(args) < 1: - usage(1, _('No command given.')) - elif len(args) > 1: - command = COMMASPACE.join(args) - usage(1, _('Bad command: %(command)s')) - - if checkprivs: - check_privs() - else: - print _('Warning! You may encounter permission problems.') - - # Handle the commands - command = args[0].lower() - if command == 'stop': - # Sent the master qrunner process a SIGINT, which is equivalent to - # giving cron/qrunner a ctrl-c or KeyboardInterrupt. This will - # effectively shut everything down. - if not quiet: - print _("Shutting down Mailman's master qrunner") - kill_watcher(signal.SIGTERM) - elif command == 'restart': - # Sent the master qrunner process a SIGHUP. This will cause the - # master qrunner to kill and restart all the worker qrunners, and to - # close and re-open its log files. - if not quiet: - print _("Restarting Mailman's master qrunner") - kill_watcher(signal.SIGINT) - elif command == 'reopen': - if not quiet: - print _('Re-opening all log files') - kill_watcher(signal.SIGHUP) - elif command == 'start': - # First, complain loudly if there's no site list. - check_for_site_list() - # Here's the scoop on the processes we're about to create. We'll need - # one for each qrunner, and one for a master child process watcher / - # lock refresher process. - # - # The child watcher process simply waits on the pids of the children - # qrunners. Unless explicitly disabled by a mailmanctl switch (or the - # children are killed with SIGTERM instead of SIGINT), the watcher - # will automatically restart any child process that exits. This - # allows us to be more robust, and also to implement restart by simply - # SIGINT'ing the qrunner children, and letting the watcher restart - # them. - # - # Under normal operation, we have a child per queue. This lets us get - # the most out of the available resources, since a qrunner with no - # files in its queue directory is pretty cheap, but having a separate - # runner process per queue allows for a very responsive system. Some - # people want a more traditional (i.e. MM2.0.x) cron-invoked qrunner. - # No problem, but using mailmanctl isn't the answer. So while - # mailmanctl hard codes some things, others, such as the number of - # qrunners per queue, is configurable in mm_cfg.py. - # - # First, acquire the master mailmanctl lock - lock = acquire_lock(force) - if not lock: - return - # Daemon process startup according to Stevens, Advanced Programming in - # the UNIX Environment, Chapter 13. - pid = os.fork() - if pid: - # parent - if not quiet: - print _("Starting Mailman's master qrunner.") - # Give up the lock "ownership". This just means the foreground - # process won't close/unlock the lock when it finalizes this lock - # instance. We'll let the mater watcher subproc own the lock. - lock._transfer_to(pid) - return - # child - lock._take_possession() - # First, save our pid in a file for "mailmanctl stop" rendezvous. We - # want the perms on the .pid file to be rw-rw---- - omask = os.umask(6) - try: - fp = open(mm_cfg.PIDFILE, 'w') - print >> fp, os.getpid() - fp.close() - finally: - os.umask(omask) - # Create a new session and become the session leader, but since we - # won't be opening any terminal devices, don't do the ultra-paranoid - # suggestion of doing a second fork after the setsid() call. - os.setsid() - # Instead of cd'ing to root, cd to the Mailman installation home - os.chdir(mm_cfg.PREFIX) - # Set our file mode creation umask - os.umask(007) - # I don't think we have any unneeded file descriptors. - # - # Now start all the qrunners. This returns a dictionary where the - # keys are qrunner pids and the values are tuples of the following - # form: (qrname, slice, count). This does its own fork and exec, and - # sets up its own signal handlers. - kids = start_all_runners() - # Set up a SIGALRM handler to refresh the lock once per day. The lock - # lifetime is 1day+6hours so this should be plenty. - def sigalrm_handler(signum, frame, lock=lock): - lock.refresh() - signal.alarm(mm_cfg.days(1)) - signal.signal(signal.SIGALRM, sigalrm_handler) - signal.alarm(mm_cfg.days(1)) - # Set up a SIGHUP handler so that if we get one, we'll pass it along - # to all the qrunner children. This will tell them to close and - # reopen their log files - def sighup_handler(signum, frame, kids=kids): - loginit.reopen() - for pid in kids.keys(): - os.kill(pid, signal.SIGHUP) - # And just to tweak things... - qlog.info('Master watcher caught SIGHUP. Re-opening log files.') - signal.signal(signal.SIGHUP, sighup_handler) - # We also need to install a SIGTERM handler because that's what init - # will kill this process with when changing run levels. - def sigterm_handler(signum, frame, kids=kids): - for pid in kids.keys(): - try: - os.kill(pid, signal.SIGTERM) - except OSError, e: - if e.errno <> errno.ESRCH: raise - qlog.info('Master watcher caught SIGTERM. Exiting.') - signal.signal(signal.SIGTERM, sigterm_handler) - # Finally, we need a SIGINT handler which will cause the sub-qrunners - # to exit, but the master will restart SIGINT'd sub-processes unless - # the -n flag was given. - def sigint_handler(signum, frame, kids=kids): - for pid in kids.keys(): - os.kill(pid, signal.SIGINT) - qlog.info('Master watcher caught SIGINT. Restarting.') - signal.signal(signal.SIGINT, sigint_handler) - # Now we're ready to simply do our wait/restart loop. This is the - # master qrunner watcher. - try: - while True: - try: - pid, status = os.wait() - except OSError, e: - # No children? We're done - if e.errno == errno.ECHILD: - break - # If the system call got interrupted, just restart it. - elif e.errno <> errno.EINTR: - raise - continue - killsig = exitstatus = None - if os.WIFSIGNALED(status): - killsig = os.WTERMSIG(status) - if os.WIFEXITED(status): - exitstatus = os.WEXITSTATUS(status) - # We'll restart the process unless we were given the - # "no-restart" switch, or if the process was SIGTERM'd or - # exitted with a SIGTERM exit status. This lets us better - # handle runaway restarts (say, if the subproc had a syntax - # error!) - restarting = '' - if restart: - if (exitstatus == None and killsig <> signal.SIGTERM) or \ - (killsig == None and exitstatus <> signal.SIGTERM): - # Then - restarting = '[restarting]' - qrname, slice, count, restarts = kids[pid] - del kids[pid] - qlog.info("""\ -Master qrunner detected subprocess exit -(pid: %d, sig: %s, sts: %s, class: %s, slice: %d/%d) %s""", - pid, killsig, exitstatus, qrname, - slice+1, count, restarting) - # See if we've reached the maximum number of allowable restarts - if exitstatus <> signal.SIGINT: - restarts += 1 - if restarts > MAX_RESTARTS: - qlog.info("""\ -Qrunner %s reached maximum restart limit of %d, not restarting.""", - qrname, MAX_RESTARTS) - restarting = '' - # Now perhaps restart the process unless it exited with a - # SIGTERM or we aren't restarting. - if restarting: - newpid = start_runner(qrname, slice, count) - kids[newpid] = (qrname, slice, count, restarts) - finally: - # Should we leave the main loop for any reason, we want to be sure - # all of our children are exited cleanly. Send SIGTERMs to all - # the child processes and wait for them all to exit. - for pid in kids.keys(): - try: - os.kill(pid, signal.SIGTERM) - except OSError, e: - if e.errno == errno.ESRCH: - # The child has already exited - qlog.info('ESRCH on pid: %d', pid) - del kids[pid] - # Wait for all the children to go away - while True: - try: - pid, status = os.wait() - except OSError, e: - if e.errno == errno.ECHILD: - break - elif e.errno <> errno.EINTR: - raise - continue - # Finally, give up the lock - lock.unlock(unconditionally=True) - os._exit(0) - - - -if __name__ == '__main__': - main() diff --git a/bin/qrunner b/bin/qrunner index 42248e472..e69de29bb 100644 --- a/bin/qrunner +++ b/bin/qrunner @@ -1,278 +0,0 @@ -#! @PYTHON@ - -# Copyright (C) 2001-2006 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""Run one or more qrunners, once or repeatedly. - -Each named runner class is run in round-robin fashion. In other words, the -first named runner is run to consume all the files currently in its -directory. When that qrunner is done, the next one is run to consume all the -files in /its/ directory, and so on. The number of total iterations can be -given on the command line. - -Usage: %(PROGRAM)s [options] - -Options: - - -r runner[:slice:range] - --runner=runner[:slice:range] - Run the named qrunner, which must be one of the strings returned by - the -l option. Optional slice:range if given, is used to assign - multiple qrunner processes to a queue. range is the total number of - qrunners for this queue while slice is the number of this qrunner from - [0..range). - - If using the slice:range form, you better make sure that each qrunner - for the queue is given the same range value. If slice:runner is not - given, then 1:1 is used. - - Multiple -r options may be given, in which case each qrunner will run - once in round-robin fashion. The special runner `All' is shorthand - for a qrunner for each listed by the -l option. - - --once - -o - Run each named qrunner exactly once through its main loop. Otherwise, - each qrunner runs indefinitely, until the process receives a SIGTERM - or SIGINT. - - -l/--list - Shows the available qrunner names and exit. - - -v/--verbose - Spit out more debugging information to the logs/qrunner log file. - - -s/--subproc - This should only be used when running qrunner as a subprocess of the - mailmanctl startup script. It changes some of the exit-on-error - behavior to work better with that framework. - - -h/--help - Print this message and exit. - -runner is required unless -l or -h is given, and it must be one of the names -displayed by the -l switch. - -Note also that this script should be started up from mailmanctl as a normal -operation. It is only useful for debugging if it is run separately. -""" - -import sys -import getopt -import signal -import logging - -import paths -from Mailman import mm_cfg -from Mailman import loginit -from Mailman.i18n import _ - -PROGRAM = sys.argv[0] -COMMASPACE = ', ' - -# Flag which says whether we're running under mailmanctl or not. -AS_SUBPROC = 0 - -log = logging.getLogger('mailman.qrunner') - - - -def usage(code, msg=''): - if code: - fd = sys.stderr - else: - fd = sys.stdout - print >> fd, _(__doc__) - if msg: - print >> fd, msg - sys.exit(code) - - - -def make_qrunner(name, slice, range, once=0): - modulename = 'Mailman.Queue.' + name - try: - __import__(modulename) - except ImportError, e: - if AS_SUBPROC: - # Exit with SIGTERM exit code so mailmanctl won't try to restart us - print >> sys.stderr, 'Cannot import runner module', modulename - print >> sys.stderr, e - sys.exit(signal.SIGTERM) - else: - usage(1, e) - qrclass = getattr(sys.modules[modulename], name) - if once: - # Subclass to hack in the setting of the stop flag in _doperiodic() - class Once(qrclass): - def _doperiodic(self): - self.stop() - qrunner = Once(slice, range) - else: - qrunner = qrclass(slice, range) - return qrunner - - - -def set_signals(loop): - # Set up the SIGTERM handler for stopping the loop - def sigterm_handler(signum, frame, loop=loop): - # Exit the qrunner cleanly - loop.stop() - loop.status = signal.SIGTERM - log.info('%s qrunner caught SIGTERM. Stopping.', loop.name()) - signal.signal(signal.SIGTERM, sigterm_handler) - # Set up the SIGINT handler for stopping the loop. For us, SIGINT is - # the same as SIGTERM, but our parent treats the exit statuses - # differently (it restarts a SIGINT but not a SIGTERM). - def sigint_handler(signum, frame, loop=loop): - # Exit the qrunner cleanly - loop.stop() - loop.status = signal.SIGINT - log.info('%s qrunner caught SIGINT. Stopping.', loop.name()) - signal.signal(signal.SIGINT, sigint_handler) - # SIGHUP just tells us to rotate our log files. - def sighup_handler(signum, frame, loop=loop): - loginit.reopen() - log.info('%s qrunner caught SIGHUP. Reopening logs.', loop.name()) - signal.signal(signal.SIGHUP, sighup_handler) - - - -def main(): - global AS_SUBPROC - try: - opts, args = getopt.getopt( - sys.argv[1:], 'hlor:vs', - ['help', 'list', 'once', 'runner=', 'verbose', 'subproc']) - except getopt.error, msg: - usage(1, msg) - - once = 0 - runners = [] - verbose = 0 - for opt, arg in opts: - if opt in ('-h', '--help'): - usage(0) - elif opt in ('-l', '--list'): - for runnername, slices in mm_cfg.QRUNNERS: - if runnername.endswith('Runner'): - name = runnername[:-len('Runner')] - else: - name = runnername - print _('%(name)s runs the %(runnername)s qrunner') - print _('All runs all the above qrunners') - sys.exit(0) - elif opt in ('-o', '--once'): - once = 1 - elif opt in ('-r', '--runner'): - runnerspec = arg - parts = runnerspec.split(':') - if len(parts) == 1: - runner = parts[0] - slice = 1 - range = 1 - elif len(parts) == 3: - runner = parts[0] - try: - slice = int(parts[1]) - range = int(parts[2]) - except ValueError: - usage(1, 'Bad runner specification: %(runnerspec)s') - else: - usage(1, 'Bad runner specification: %(runnerspec)s') - if runner == 'All': - for runnername, slices in mm_cfg.QRUNNERS: - runners.append((runnername, slice, range)) - else: - if runner.endswith('Runner'): - runners.append((runner, slice, range)) - else: - runners.append((runner + 'Runner', slice, range)) - elif opt in ('-s', '--subproc'): - AS_SUBPROC = 1 - elif opt in ('-v', '--verbose'): - verbose = 1 - - if len(args) <> 0: - usage(1) - if len(runners) == 0: - usage(1, _('No runner name given.')) - - # If we're not running as a subprocess of mailmanctl, then we'll log to - # stderr in addition to logging to the log files. We do this by passing a - # value of True to propagate, which allows the 'mailman' root logger to - # see the log messages. - loginit.initialize(propagate=not AS_SUBPROC) - - # Fast track for one infinite runner - if len(runners) == 1 and not once: - qrunner = make_qrunner(*runners[0]) - class Loop: - status = 0 - def __init__(self, qrunner): - self.__qrunner = qrunner - def name(self): - return self.__qrunner.__class__.__name__ - def stop(self): - self.__qrunner.stop() - loop = Loop(qrunner) - set_signals(loop) - # Now start up the main loop - log.info('%s qrunner started.', loop.name()) - qrunner.run() - log.info('%s qrunner exiting.', loop.name()) - else: - # Anything else we have to handle a bit more specially - qrunners = [] - for runner, slice, range in runners: - qrunner = make_qrunner(runner, slice, range, 1) - qrunners.append(qrunner) - # This class is used to manage the main loop - class Loop: - status = 0 - def __init__(self): - self.__isdone = 0 - def name(self): - return 'Main loop' - def stop(self): - self.__isdone = 1 - def isdone(self): - return self.__isdone - loop = Loop() - set_signals(loop) - log.info('Main qrunner loop started.') - while not loop.isdone(): - for qrunner in qrunners: - # In case the SIGTERM came in the middle of this iteration - if loop.isdone(): - break - if verbose: - log.info('Now doing a %s qrunner iteration', - qrunner.__class__.__bases__[0].__name__) - qrunner.run() - if once: - break - log.info('Main qrunner loop exiting.') - # All done - sys.exit(loop.status) - - - -if __name__ == '__main__': - main() diff --git a/bin/update b/bin/update index 23429508e..e69de29bb 100755 --- a/bin/update +++ b/bin/update @@ -1,807 +0,0 @@ -#! @PYTHON@ -# -# Copyright (C) 1998-2005 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -"""Perform all necessary upgrades. - -Usage: %(PROGRAM)s [options] - -Options: - -f/--force - Force running the upgrade procedures. Normally, if the version number - of the installed Mailman matches the current version number (or a - `downgrade' is detected), nothing will be done. - - -h/--help - Print this text and exit. - -Use this script to help you update to the latest release of Mailman from -some previous version. It knows about versions back to 1.0b4 (?). -""" - -import os -import md5 -import sys -import time -import errno -import getopt -import shutil -import cPickle -import marshal - -import paths -import email - -from Mailman import mm_cfg -from Mailman import Utils -from Mailman import MailList -from Mailman import Message -from Mailman import Pending -from Mailman.LockFile import TimeOutError -from Mailman.i18n import _ -from Mailman.Queue.Switchboard import Switchboard -from Mailman.OldStyleMemberships import OldStyleMemberships -from Mailman.MemberAdaptor import BYBOUNCE, ENABLED - -FRESH = 0 -NOTFRESH = -1 - -LMVFILE = os.path.join(mm_cfg.DATA_DIR, 'last_mailman_version') -PROGRAM = sys.argv[0] - - - -def calcversions(): - # Returns a tuple of (lastversion, thisversion). If the last version - # could not be determined, lastversion will be FRESH or NOTFRESH, - # depending on whether this installation appears to be fresh or not. The - # determining factor is whether there are files in the $var_prefix/logs - # subdir or not. The version numbers are HEX_VERSIONs. - # - # See if we stored the last updated version - lastversion = None - thisversion = mm_cfg.HEX_VERSION - try: - fp = open(LMVFILE) - data = fp.read() - fp.close() - lastversion = int(data, 16) - except (IOError, ValueError): - pass - # - # try to figure out if this is a fresh install - if lastversion is None: - lastversion = FRESH - try: - if os.listdir(mm_cfg.LOG_DIR): - lastversion = NOTFRESH - except OSError: - pass - return (lastversion, thisversion) - - - -def makeabs(relpath): - return os.path.join(mm_cfg.PREFIX, relpath) - -def make_varabs(relpath): - return os.path.join(mm_cfg.VAR_PREFIX, relpath) - - -def move_language_templates(mlist): - listname = mlist.internal_name() - print _('Fixing language templates: %(listname)s') - # Mailman 2.1 has a new cascading search for its templates, defined and - # described in Utils.py:maketext(). Putting templates in the top level - # templates/ subdir or the lists/<listname> subdir is deprecated and no - # longer searched.. - # - # What this means is that most templates can live in the global templates/ - # subdirectory, and only needs to be copied into the list-, vhost-, or - # site-specific language directories when needed. - # - # Also, by default all standard (i.e. English) templates must now live in - # the templates/en directory. This update cleans up all the templates, - # deleting more-specific duplicates (as calculated by md5 checksums) in - # favor of more-global locations. - # - # First, get rid of any lists/<list> template or lists/<list>/en template - # that is identical to the global templates/* default. - for gtemplate in os.listdir(os.path.join(mm_cfg.TEMPLATE_DIR, 'en')): - # BAW: get rid of old templates, e.g. admlogin.txt and - # handle_opts.html - try: - fp = open(os.path.join(mm_cfg.TEMPLATE_DIR, gtemplate)) - except IOError, e: - if e.errno <> errno.ENOENT: raise - # No global template - continue - - gcksum = md5.new(fp.read()).digest() - fp.close() - # Match against the lists/<list>/* template - try: - fp = open(os.path.join(mlist.fullpath(), gtemplate)) - except IOError, e: - if e.errno <> errno.ENOENT: raise - else: - tcksum = md5.new(fp.read()).digest() - fp.close() - if gcksum == tcksum: - os.unlink(os.path.join(mlist.fullpath(), gtemplate)) - # Match against the lists/<list>/*.prev template - try: - fp = open(os.path.join(mlist.fullpath(), gtemplate + '.prev')) - except IOError, e: - if e.errno <> errno.ENOENT: raise - else: - tcksum = md5.new(fp.read()).digest() - fp.close() - if gcksum == tcksum: - os.unlink(os.path.join(mlist.fullpath(), gtemplate + '.prev')) - # Match against the lists/<list>/en/* templates - try: - fp = open(os.path.join(mlist.fullpath(), 'en', gtemplate)) - except IOError, e: - if e.errno <> errno.ENOENT: raise - else: - tcksum = md5.new(fp.read()).digest() - fp.close() - if gcksum == tcksum: - os.unlink(os.path.join(mlist.fullpath(), 'en', gtemplate)) - # Match against the templates/* template - try: - fp = open(os.path.join(mm_cfg.TEMPLATE_DIR, gtemplate)) - except IOError, e: - if e.errno <> errno.ENOENT: raise - else: - tcksum = md5.new(fp.read()).digest() - fp.close() - if gcksum == tcksum: - os.unlink(os.path.join(mm_cfg.TEMPLATE_DIR, gtemplate)) - # Match against the templates/*.prev template - try: - fp = open(os.path.join(mm_cfg.TEMPLATE_DIR, gtemplate + '.prev')) - except IOError, e: - if e.errno <> errno.ENOENT: raise - else: - tcksum = md5.new(fp.read()).digest() - fp.close() - if gcksum == tcksum: - os.unlink(os.path.join(mm_cfg.TEMPLATE_DIR, - gtemplate + '.prev')) - - - -def dolist(listname): - errors = 0 - mlist = MailList.MailList(listname, lock=0) - try: - mlist.Lock(0.5) - except TimeOutError: - print >> sys.stderr, _('WARNING: could not acquire lock for list: ' - '%(listname)s') - return 1 - - # Sanity check the invariant that every BYBOUNCE disabled member must have - # bounce information. Some earlier betas broke this. BAW: we're - # submerging below the MemberAdaptor interface, so skip this if we're not - # using OldStyleMemberships. - if isinstance(mlist._memberadaptor, OldStyleMemberships): - noinfo = {} - for addr, (reason, when) in mlist.delivery_status.items(): - if reason == BYBOUNCE and not mlist.bounce_info.has_key(addr): - noinfo[addr] = reason, when - # What to do about these folks with a BYBOUNCE delivery status and no - # bounce info? This number should be very small, and I think it's - # fine to simple re-enable them and let the bounce machinery - # re-disable them if necessary. - n = len(noinfo) - if n > 0: - print _( - 'Resetting %(n)s BYBOUNCEs disabled addrs with no bounce info') - for addr in noinfo.keys(): - mlist.setDeliveryStatus(addr, ENABLED) - - # Update the held requests database - print _("""Updating the held requests database.""") - mlist._UpdateRecords() - - mbox_dir = make_varabs('archives/private/%s.mbox' % (listname)) - mbox_file = make_varabs('archives/private/%s.mbox/%s' % (listname, - listname)) - - o_pub_mbox_file = make_varabs('archives/public/%s' % (listname)) - o_pri_mbox_file = make_varabs('archives/private/%s' % (listname)) - - html_dir = o_pri_mbox_file - o_html_dir = makeabs('public_html/archives/%s' % (listname)) - # - # make the mbox directory if it's not there. - # - if not os.path.exists(mbox_dir): - ou = os.umask(0) - os.mkdir(mbox_dir, 02775) - os.umask(ou) - else: - # this shouldn't happen, but hey, just in case - if not os.path.isdir(mbox_dir): - print _("""\ -For some reason, %(mbox_dir)s exists as a file. This won't work with -b6, so I'm renaming it to %(mbox_dir)s.tmp and proceeding.""") - os.rename(mbox_dir, "%s.tmp" % (mbox_dir)) - ou = os.umask(0) - os.mkdir(mbox_dir, 02775) - os.umask(ou) - - # Move any existing mboxes around, but watch out for both a public and a - # private one existing - if os.path.isfile(o_pri_mbox_file) and os.path.isfile(o_pub_mbox_file): - if mlist.archive_private: - print _("""\ - -%(listname)s has both public and private mbox archives. Since this list -currently uses private archiving, I'm installing the private mbox archive --- %(o_pri_mbox_file)s -- as the active archive, and renaming - %(o_pub_mbox_file)s -to - %(o_pub_mbox_file)s.preb6 - -You can integrate that into the archives if you want by using the 'arch' -script. -""") % (mlist._internal_name, o_pri_mbox_file, o_pub_mbox_file, - o_pub_mbox_file) - os.rename(o_pub_mbox_file, "%s.preb6" % (o_pub_mbox_file)) - else: - print _("""\ -%s has both public and private mbox archives. Since this list -currently uses public archiving, I'm installing the public mbox file -archive file (%s) as the active one, and renaming - %s - to - %s.preb6 - -You can integrate that into the archives if you want by using the 'arch' -script. -""") % (mlist._internal_name, o_pub_mbox_file, o_pri_mbox_file, - o_pri_mbox_file) - os.rename(o_pri_mbox_file, "%s.preb6" % (o_pri_mbox_file)) - # - # move private archive mbox there if it's around - # and take into account all sorts of absurdities - # - print _('- updating old private mbox file') - if os.path.exists(o_pri_mbox_file): - if os.path.isfile(o_pri_mbox_file): - os.rename(o_pri_mbox_file, mbox_file) - elif not os.path.isdir(o_pri_mbox_file): - newname = "%s.mm_install-dunno_what_this_was_but_its_in_the_way" \ - % o_pri_mbox_file - os.rename(o_pri_mbox_file, newname) - print _("""\ - unknown file in the way, moving - %(o_pri_mbox_file)s - to - %(newname)s""") - else: - # directory - print _("""\ - looks like you have a really recent CVS installation... - you're either one brave soul, or you already ran me""") - - - # - # move public archive mbox there if it's around - # and take into account all sorts of absurdities. - # - print _('- updating old public mbox file') - if os.path.exists(o_pub_mbox_file): - if os.path.isfile(o_pub_mbox_file): - os.rename(o_pub_mbox_file, mbox_file) - elif not os.path.isdir(o_pub_mbox_file): - newname = "%s.mm_install-dunno_what_this_was_but_its_in_the_way" \ - % o_pub_mbox_file - os.rename(o_pub_mbox_file, newname) - print _("""\ - unknown file in the way, moving - %(o_pub_mbox_file)s - to - %(newname)s""") - else: # directory - print _("""\ - looks like you have a really recent CVS installation... - you're either one brave soul, or you already ran me""") - - # - # move the html archives there - # - if os.path.isdir(o_html_dir): - os.rename(o_html_dir, html_dir) - # - # chmod the html archives - # - os.chmod(html_dir, 02775) - # BAW: Is this still necessary?! - mlist.Save() - # - # check to see if pre-b4 list-specific templates are around - # and move them to the new place if there's not already - # a new one there - # - tmpl_dir = os.path.join(mm_cfg.PREFIX, "templates") - list_dir = os.path.join(mm_cfg.PREFIX, "lists") - b4_tmpl_dir = os.path.join(tmpl_dir, mlist._internal_name) - new_tmpl_dir = os.path.join(list_dir, mlist._internal_name) - if os.path.exists(b4_tmpl_dir): - print _("""\ -- This list looks like it might have <= b4 list templates around""") - for f in os.listdir(b4_tmpl_dir): - o_tmpl = os.path.join(b4_tmpl_dir, f) - n_tmpl = os.path.join(new_tmpl_dir, f) - if os.path.exists(o_tmpl): - if not os.path.exists(n_tmpl): - os.rename(o_tmpl, n_tmpl) - print _('- moved %(o_tmpl)s to %(n_tmpl)s') - else: - print _("""\ -- both %(o_tmpl)s and %(n_tmpl)s exist, leaving untouched""") - else: - print _("""\ -- %(o_tmpl)s doesn't exist, leaving untouched""") - # - # Move all the templates to the en language subdirectory as required for - # Mailman 2.1 - # - move_language_templates(mlist) - # Avoid eating filehandles with the list lockfiles - mlist.Unlock() - return 0 - - - -def archive_path_fixer(unused_arg, dir, files): - # Passed to os.path.walk to fix the perms on old html archives. - for f in files: - abs = os.path.join(dir, f) - if os.path.isdir(abs): - if f == "database": - os.chmod(abs, 02770) - else: - os.chmod(abs, 02775) - elif os.path.isfile(abs): - os.chmod(abs, 0664) - -def remove_old_sources(module): - # Also removes old directories. - src = '%s/%s' % (mm_cfg.PREFIX, module) - pyc = src + "c" - if os.path.isdir(src): - print _('removing directory %(src)s and everything underneath') - shutil.rmtree(src) - elif os.path.exists(src): - print _('removing %(src)s') - try: - os.unlink(src) - except os.error, rest: - print _("Warning: couldn't remove %(src)s -- %(rest)s") - if module.endswith('.py') and os.path.exists(pyc): - try: - os.unlink(pyc) - except os.error, rest: - print _("couldn't remove old file %(pyc)s -- %(rest)s") - - -def update_qfiles(): - print _('updating old qfiles') - prefix = `time.time()` + '+' - # Be sure the qfiles/in directory exists (we don't really need the - # switchboard object, but it's convenient for creating the directory). - sb = Switchboard(mm_cfg.INQUEUE_DIR) - for filename in os.listdir(mm_cfg.QUEUE_DIR): - # Updating means just moving the .db and .msg files to qfiles/in where - # it should be dequeued, converted, and processed normally. - if filename.endswith('.msg'): - oldmsgfile = os.path.join(mm_cfg.QUEUE_DIR, filename) - newmsgfile = os.path.join(mm_cfg.INQUEUE_DIR, prefix + filename) - os.rename(oldmsgfile, newmsgfile) - elif filename.endswith('.db'): - olddbfile = os.path.join(mm_cfg.QUEUE_DIR, filename) - newdbfile = os.path.join(mm_cfg.INQUEUE_DIR, prefix + filename) - os.rename(olddbfile, newdbfile) - # Now update for the Mailman 2.1.5 qfile format. For every filebase in - # the qfiles/* directories that has both a .pck and a .db file, pull the - # data out and re-queue them. - for dirname in os.listdir(mm_cfg.QUEUE_DIR): - dirpath = os.path.join(mm_cfg.QUEUE_DIR, dirname) - if dirpath == mm_cfg.BADQUEUE_DIR: - # The files in qfiles/bad can't possibly be pickles - continue - sb = Switchboard(dirpath) - try: - for filename in os.listdir(dirpath): - filepath = os.path.join(dirpath, filename) - filebase, ext = os.path.splitext(filepath) - # Handle the .db metadata files as part of the handling of the - # .pck or .msg message files. - if ext not in ('.pck', '.msg'): - continue - msg, data = dequeue(filebase) - if msg is not None and data is not None: - sb.enqueue(msg, data) - except EnvironmentError, e: - if e.errno <> errno.ENOTDIR: - raise - print _('Warning! Not a directory: %(dirpath)s') - - - -# Implementations taken from the pre-2.1.5 Switchboard -def ext_read(filename): - fp = open(filename) - d = marshal.load(fp) - # Update from version 2 files - if d.get('version', 0) == 2: - del d['filebase'] - # Do the reverse conversion (repr -> float) - for attr in ['received_time']: - try: - sval = d[attr] - except KeyError: - pass - else: - # Do a safe eval by setting up a restricted execution - # environment. This may not be strictly necessary since we - # know they are floats, but it can't hurt. - d[attr] = eval(sval, {'__builtins__': {}}) - fp.close() - return d - - -def dequeue(filebase): - # Calculate the .db and .msg filenames from the given filebase. - msgfile = os.path.join(filebase + '.msg') - pckfile = os.path.join(filebase + '.pck') - dbfile = os.path.join(filebase + '.db') - # Now we are going to read the message and metadata for the given - # filebase. We want to read things in this order: first, the metadata - # file to find out whether the message is stored as a pickle or as - # plain text. Second, the actual message file. However, we want to - # first unlink the message file and then the .db file, because the - # qrunner only cues off of the .db file - msg = None - try: - data = ext_read(dbfile) - os.unlink(dbfile) - except EnvironmentError, e: - if e.errno <> errno.ENOENT: raise - data = {} - # Between 2.1b4 and 2.1b5, the `rejection-notice' key in the metadata - # was renamed to `rejection_notice', since dashes in the keys are not - # supported in METAFMT_ASCII. - if data.has_key('rejection-notice'): - data['rejection_notice'] = data['rejection-notice'] - del data['rejection-notice'] - msgfp = None - try: - try: - msgfp = open(pckfile) - msg = cPickle.load(msgfp) - os.unlink(pckfile) - except EnvironmentError, e: - if e.errno <> errno.ENOENT: raise - msgfp = None - try: - msgfp = open(msgfile) - msg = email.message_from_file(msgfp, Message.Message) - os.unlink(msgfile) - except EnvironmentError, e: - if e.errno <> errno.ENOENT: raise - except (email.Errors.MessageParseError, ValueError), e: - # This message was unparsable, most likely because its - # MIME encapsulation was broken. For now, there's not - # much we can do about it. - print _('message is unparsable: %(filebase)s') - msgfp.close() - msgfp = None - if mm_cfg.QRUNNER_SAVE_BAD_MESSAGES: - # Cheapo way to ensure the directory exists w/ the - # proper permissions. - sb = Switchboard(mm_cfg.BADQUEUE_DIR) - os.rename(msgfile, os.path.join( - mm_cfg.BADQUEUE_DIR, filebase + '.txt')) - else: - os.unlink(msgfile) - msg = data = None - except EOFError: - # For some reason the pckfile was empty. Just delete it. - print _('Warning! Deleting empty .pck file: %(pckfile)s') - os.unlink(pckfile) - finally: - if msgfp: - msgfp.close() - return msg, data - - - -def update_pending(): - file20 = os.path.join(mm_cfg.DATA_DIR, 'pending_subscriptions.db') - file214 = os.path.join(mm_cfg.DATA_DIR, 'pending.pck') - db = None - # Try to load the Mailman 2.0 file - try: - fp = open(file20) - except IOError, e: - if e.errno <> errno.ENOENT: raise - else: - print _('Updating Mailman 2.0 pending_subscriptions.db database') - db = marshal.load(fp) - # Convert to the pre-Mailman 2.1.5 format - db = Pending._update(db) - if db is None: - # Try to load the Mailman 2.1.x where x < 5, file - try: - fp = open(file214) - except IOError, e: - if e.errno <> errno.ENOENT: raise - else: - print _('Updating Mailman 2.1.4 pending.pck database') - db = cPickle.load(fp) - if db is None: - print _('Nothing to do.') - return - # Now upgrade the database to the 2.1.5 format. Each list now has its own - # pending.pck file, but only the RE_ENABLE operation actually recorded the - # listname in the request. For the SUBSCRIPTION, UNSUBSCRIPTION, and - # CHANGE_OF_ADDRESS operations, we know the address of the person making - # the request so we can repend this request just for the lists the person - # is a member of. For the HELD_MESSAGE operation, we can check the list's - # requests.pck file for correlation. Evictions will take care of any - # misdirected pendings. - reenables_by_list = {} - addrops_by_address = {} - holds_by_id = {} - subs_by_address = {} - for key, val in db.items(): - if key in ('evictions', 'version'): - continue - try: - op = val[0] - data = val[1:] - except (IndexError, ValueError): - print _('Ignoring bad pended data: %(key)s: %(val)s') - continue - if op in (Pending.UNSUBSCRIPTION, Pending.CHANGE_OF_ADDRESS): - # data[0] is the address being unsubscribed - addrops_by_address.setdefault(data[0], []).append((key, val)) - elif op == Pending.SUBSCRIPTION: - # data[0] is a UserDesc object - addr = data[0].address - subs_by_address.setdefault(addr, []).append((key, val)) - elif op == Pending.RE_ENABLE: - # data[0] is the mailing list's internal name - reenables_by_list.setdefault(data[0], []).append((key, val)) - elif op == Pending.HELD_MESSAGE: - # data[0] is the hold id. There better only be one entry per id - id = data[0] - if holds_by_id.has_key(id): - print _('WARNING: Ignoring duplicate pending ID: %(id)s.') - else: - holds_by_id[id] = (key, val) - # Now we have to lock every list and re-pend all the appropriate - # requests. Note that this will reset all the expiration dates, but that - # should be fine. - for listname in Utils.list_names(): - mlist = MailList.MailList(listname) - # This is not the most efficient way to do this because it loads and - # saves the pending.pck file each time. :( - try: - for cookie, data in reenables_by_list.get(listname, []): - mlist.pend_repend(cookie, data) - for id, (cookie, data) in holds_by_id.items(): - try: - rec = mlist.GetRecord(id) - except KeyError: - # Not for this list - pass - else: - mlist.pend_repend(cookie, data) - del holds_by_id[id] - for addr, recs in subs_by_address.items(): - # We shouldn't have a subscription confirmation if the address - # is already a member of the mailing list. - if mlist.isMember(addr): - continue - for cookie, data in recs: - mlist.pend_repend(cookie, data) - for addr, recs in addrops_by_address.items(): - # We shouldn't have unsubscriptions or change of address - # requests for addresses which aren't members of the list. - if not mlist.isMember(addr): - continue - for cookie, data in recs: - mlist.pend_repend(cookie, data) - mlist.Save() - finally: - mlist.Unlock() - try: - os.unlink(file20) - except OSError, e: - if e.errno <> errno.ENOENT: raise - try: - os.unlink(file214) - except OSError, e: - if e.errno <> errno.ENOENT: raise - - - -def main(): - errors = 0 - # get rid of old stuff - print _('getting rid of old source files') - for mod in ('Mailman/Archiver.py', 'Mailman/HyperArch.py', - 'Mailman/HyperDatabase.py', 'Mailman/pipermail.py', - 'Mailman/smtplib.py', 'Mailman/Cookie.py', - 'bin/update_to_10b6', 'scripts/mailcmd', - 'scripts/mailowner', 'mail/wrapper', 'Mailman/pythonlib', - 'cgi-bin/archives', 'Mailman/MailCommandHandler'): - remove_old_sources(mod) - listnames = Utils.list_names() - if not listnames: - print _('no lists == nothing to do, exiting') - return - # - # for people with web archiving, make sure the directories - # in the archiving are set with proper perms for b6. - # - if os.path.isdir("%s/public_html/archives" % mm_cfg.PREFIX): - print _("""\ -fixing all the perms on your old html archives to work with b6 -If your archives are big, this could take a minute or two...""") - os.path.walk("%s/public_html/archives" % mm_cfg.PREFIX, - archive_path_fixer, "") - print _('done') - for listname in listnames: - print _('Updating mailing list: %(listname)s') - errors = errors + dolist(listname) - print - print _('Updating Usenet watermarks') - wmfile = os.path.join(mm_cfg.DATA_DIR, 'gate_watermarks') - try: - fp = open(wmfile) - except IOError: - print _('- nothing to update here') - else: - d = marshal.load(fp) - fp.close() - for listname in d.keys(): - if listname not in listnames: - # this list no longer exists - continue - mlist = MailList.MailList(listname, lock=0) - try: - mlist.Lock(0.5) - except TimeOutError: - print >> sys.stderr, _( - 'WARNING: could not acquire lock for list: %(listname)s') - errors = errors + 1 - else: - # Pre 1.0b7 stored 0 in the gate_watermarks file to indicate - # that no gating had been done yet. Without coercing this to - # None, the list could now suddenly get flooded. - mlist.usenet_watermark = d[listname] or None - mlist.Save() - mlist.Unlock() - os.unlink(wmfile) - print _('- usenet watermarks updated and gate_watermarks removed') - # In Mailman 2.1, the pending database format and file name changed, but - # in Mailman 2.1.5 it changed again. This should update all existing - # files to the 2.1.5 format. - update_pending() - # In Mailman 2.1, the qfiles directory has a different structure and a - # different content. Also, in Mailman 2.1.5 we collapsed the message - # files from separate .msg (pickled Message objects) and .db (marshalled - # dictionaries) to a shared .pck file containing two pickles. - update_qfiles() - # This warning was necessary for the upgrade from 1.0b9 to 1.0b10. - # There's no good way of figuring this out for releases prior to 2.0beta2 - # :( - if lastversion == NOTFRESH: - print _(""" - -NOTE NOTE NOTE NOTE NOTE - - You are upgrading an existing Mailman installation, but I can't tell what - version you were previously running. - - If you are upgrading from Mailman 1.0b9 or earlier you will need to - manually update your mailing lists. For each mailing list you need to - copy the file templates/options.html lists/<listname>/options.html. - - However, if you have edited this file via the Web interface, you will have - to merge your changes into this file, otherwise you will lose your - changes. - -NOTE NOTE NOTE NOTE NOTE - -""") - return errors - - - -def usage(code, msg=''): - if code: - fd = sys.stderr - else: - fd = sys.stdout - print >> fd, _(__doc__) % globals() - if msg: - print >> sys.stderr, msg - sys.exit(code) - - - -if __name__ == '__main__': - try: - opts, args = getopt.getopt(sys.argv[1:], 'hf', - ['help', 'force']) - except getopt.error, msg: - usage(1, msg) - - if args: - usage(1, 'Unexpected arguments: %s' % args) - - force = 0 - for opt, arg in opts: - if opt in ('-h', '--help'): - usage(0) - elif opt in ('-f', '--force'): - force = 1 - - # calculate the versions - lastversion, thisversion = calcversions() - hexlversion = hex(lastversion) - hextversion = hex(thisversion) - if lastversion == thisversion and not force: - # nothing to do - print _('No updates are necessary.') - sys.exit(0) - if lastversion > thisversion and not force: - print _("""\ -Downgrade detected, from version %(hexlversion)s to version %(hextversion)s -This is probably not safe. -Exiting.""") - sys.exit(1) - print _('Upgrading from version %(hexlversion)s to %(hextversion)s') - errors = main() - if not errors: - # Record the version we just upgraded to - fp = open(LMVFILE, 'w') - fp.write(hex(mm_cfg.HEX_VERSION) + '\n') - fp.close() - else: - lockdir = mm_cfg.LOCK_DIR - print _('''\ - -ERROR: - -The locks for some lists could not be acquired. This means that either -Mailman was still active when you upgraded, or there were stale locks in the -%(lockdir)s directory. - -You must put Mailman into a quiescent state and remove all stale locks, then -re-run "make update" manually. See the INSTALL and UPGRADE files for details. -''') @@ -1,5 +1,5 @@ #! /bin/sh -# From configure.in Revision: 7897 . +# From configure.in Revision: 7899 . # Guess values for system-dependent variables and create Makefiles. # Generated by GNU Autoconf 2.59 for GNU Mailman 2.2.0a0. # @@ -4295,16 +4295,13 @@ build/bin/dumpdb:bin/dumpdb \ build/bin/fix_url.py:bin/fix_url.py \ build/bin/genaliases:bin/genaliases \ build/bin/list_admins:bin/list_admins \ -build/bin/mailmanctl:bin/mailmanctl \ build/bin/mmshell:bin/mmshell \ build/bin/msgfmt.py:bin/msgfmt.py \ build/bin/pygettext.py:bin/pygettext.py \ -build/bin/qrunner:bin/qrunner \ build/bin/remove_members:bin/remove_members \ build/bin/reset_pw.py:bin/reset_pw.py \ build/bin/sync_members:bin/sync_members \ build/bin/transcheck:bin/transcheck \ -build/bin/update:bin/update \ build/bin/withlist:bin/withlist \ build/bin/templ2pot.py:bin/templ2pot.py \ build/bin/po2templ.py:bin/po2templ.py \ diff --git a/configure.in b/configure.in index 3cd34d5aa..4cfeba5d3 100644 --- a/configure.in +++ b/configure.in @@ -15,7 +15,7 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. dnl Process this file with autoconf to produce a configure script. -AC_REVISION($Revision: 7899 $) +AC_REVISION($Revision: 7926 $) AC_PREREQ(2.0) AC_INIT([GNU Mailman], [2.2.0a0]) @@ -602,16 +602,13 @@ bin/dumpdb \ bin/fix_url.py \ bin/genaliases \ bin/list_admins \ -bin/mailmanctl \ bin/mmshell \ bin/msgfmt.py \ bin/pygettext.py \ -bin/qrunner \ bin/remove_members \ bin/reset_pw.py \ bin/sync_members \ bin/transcheck \ -bin/update \ bin/withlist \ bin/templ2pot.py \ bin/po2templ.py \ diff --git a/misc/Makefile.in b/misc/Makefile.in index 20d90d640..6c1a3a55e 100644 --- a/misc/Makefile.in +++ b/misc/Makefile.in @@ -42,6 +42,7 @@ OPT= @OPT@ CFLAGS= $(OPT) $(DEFS) PACKAGEDIR= $(prefix)/Mailman DATADIR= $(var_prefix)/data +ETCDIR= $(var_prefix)/etc ICONDIR= $(prefix)/icons SCRIPTSDIR= $(prefix)/scripts @@ -83,6 +84,7 @@ install-other: done $(INSTALL) -m $(EXEMODE) mailman $(DESTDIR)$(SCRIPTSDIR) $(INSTALL) -m $(FILEMODE) sitelist.cfg $(DESTDIR)$(DATADIR) + $(INSTALL) -m $(FILEMODE) mailman.cfg.sample $(DESTDIR)$(ETCDIR) install-packages: for p in $(PACKAGES); \ diff --git a/misc/mailman.cfg.sample b/misc/mailman.cfg.sample new file mode 100644 index 000000000..4cc4ec280 --- /dev/null +++ b/misc/mailman.cfg.sample @@ -0,0 +1,66 @@ +# -*- python -*- + +# Copyright (C) 2006 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +"""This module contains your site-specific settings. + +Use this to override the default settings in Mailman/Defaults.py. You only +need to include those settings that you want to change, and unlike the old +mm_cfg.py file, you do /not/ need to import Defaults. Its variables will +automatically be available in this module's namespace. + +You should consult Defaults.py though for a complete listing of configuration +variables that you can change. + +To use this, copy this file to $VAR_PREFIX/etc/mailman.cfg + +Mailman's installation procedure will never overwrite mailman.cfg. +""" +# -*- python -*- + +# Copyright (C) 2006 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +"""This module contains your site-specific settings. + +Use this to override the default settings in Mailman/Defaults.py. You only +need to include those settings that you want to change, and unlike the old +mm_cfg.py file, you do /not/ need to import Defaults. Its variables will +automatically be available in this module's namespace. + +You should consult Defaults.py though for a complete listing of configuration +variables that you can change. + +To use this, copy this file to $VAR_PREFIX/etc/mailman.cfg + +Mailman's installation procedure will never overwrite mailman.cfg. +""" |