diff options
61 files changed, 2334 insertions, 1333 deletions
diff --git a/Mailman/Archiver/Archiver.py b/Mailman/Archiver/Archiver.py index c8e208518..328920335 100644 --- a/Mailman/Archiver/Archiver.py +++ b/Mailman/Archiver/Archiver.py @@ -25,6 +25,7 @@ archival. import os import errno import traceback +import re from cStringIO import StringIO from Mailman import mm_cfg @@ -35,22 +36,26 @@ from Mailman.SafeDict import SafeDict from Mailman.Logging.Syslog import syslog from Mailman.i18n import _ +try: + True, False +except NameError: + True = 1 + False = 0 + def makelink(old, new): try: os.symlink(old, new) - except os.error, e: - code, msg = e - if code <> errno.EEXIST: + except OSError, e: + if e.errno <> errno.EEXIST: raise def breaklink(link): try: os.unlink(link) - except os.error, e: - code, msg = e - if code <> errno.ENOENT: + except OSError, e: + if e.errno <> errno.ENOENT: raise @@ -107,13 +112,16 @@ class Archiver: fp = open(indexfile) except IOError, e: if e.errno <> errno.ENOENT: raise - else: + omask = os.umask(002) + try: fp = open(indexfile, 'w') - fp.write(Utils.maketext( - 'emptyarchive.html', - {'listname': self.real_name, - 'listinfo': self.GetScriptURL('listinfo', absolute=1), - }, mlist=self)) + finally: + os.umask(omask) + fp.write(Utils.maketext( + 'emptyarchive.html', + {'listname': self.real_name, + 'listinfo': self.GetScriptURL('listinfo', absolute=1), + }, mlist=self)) if fp: fp.close() finally: @@ -128,15 +136,15 @@ class Archiver: self.internal_name() + '.mbox') def GetBaseArchiveURL(self): + url = self.GetScriptURL('private', absolute=1) + '/' if self.archive_private: - return self.GetScriptURL('private', absolute=1) + '/' + return url else: - inv = {} - for k, v in mm_cfg.VIRTUAL_HOSTS.items(): - inv[v] = k + hostname = re.match('[^:]*://([^/]*)/.*', url).group(1)\ + or mm_cfg.DEFAULT_URL_HOST url = mm_cfg.PUBLIC_ARCHIVE_URL % { 'listname': self.internal_name(), - 'hostname': inv.get(self.host_name, mm_cfg.DEFAULT_URL_HOST), + 'hostname': hostname } if not url.endswith('/'): url += '/' @@ -220,7 +228,7 @@ class Archiver: if mm_cfg.ARCHIVE_TO_MBOX == -1: # Archiving is completely disabled, don't require the skeleton. return - pubdir = Site.get_archpath(self.internal_name(), public=1) + pubdir = Site.get_archpath(self.internal_name(), public=True) privdir = self.archive_dir() pubmbox = pubdir + '.mbox' privmbox = privdir + '.mbox' @@ -231,4 +239,6 @@ class Archiver: # BAW: privdir or privmbox could be nonexistant. We'd get an # OSError, ENOENT which should be caught and reported properly. makelink(privdir, pubdir) - makelink(privmbox, pubmbox) + # Only make this link if the site has enabled public mbox files + if mm_cfg.PUBLIC_MBOX: + makelink(privmbox, pubmbox) diff --git a/Mailman/Archiver/HyperArch.py b/Mailman/Archiver/HyperArch.py index 61725b2e3..c0695f518 100644 --- a/Mailman/Archiver/HyperArch.py +++ b/Mailman/Archiver/HyperArch.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2005 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -40,6 +40,8 @@ import weakref import binascii from email.Header import decode_header, make_header +from email.Errors import HeaderParseError +from email.Charset import Charset from Mailman import mm_cfg from Mailman import Utils @@ -287,10 +289,9 @@ class Article(pipermail.Article): self.ctype = ctype.lower() self.cenc = cenc.lower() self.decoded = {} - charset = message.get_param('charset') - if isinstance(charset, types.TupleType): - # An RFC 2231 charset - charset = unicode(charset[2], charset[0]) + cset = Utils.GetCharSet(mlist.preferred_language) + cset_out = Charset(cset).output_charset or cset + charset = message.get_content_charset(cset_out) if charset: charset = charset.lower().strip() if charset[0]=='"' and charset[-1]=='"': @@ -298,7 +299,7 @@ class Article(pipermail.Article): if charset[0]=="'" and charset[-1]=="'": charset = charset[1:-1] try: - body = message.get_payload(decode=1) + body = message.get_payload(decode=True) except binascii.Error: body = None if body and charset != Utils.GetCharSet(self._lang): @@ -402,22 +403,35 @@ class Article(pipermail.Article): self.decoded['email'] = email if subject: self.decoded['subject'] = subject + self.decoded['stripped'] = self.strip_subject(subject or self.subject) + + def strip_subject(self, subject): + # Strip subject_prefix and Re: for subject sorting + # This part was taken from CookHeaders.py (TK) + prefix = self._mlist.subject_prefix.strip() + if prefix: + prefix_pat = re.escape(prefix) + prefix_pat = '%'.join(prefix_pat.split(r'\%')) + prefix_pat = re.sub(r'%\d*d', r'\s*\d+\s*', prefix_pat) + subject = re.sub(prefix_pat, '', subject) + subject = subject.lstrip() + strip_pat = re.compile('^((RE|AW|SV)(\[\d+\])?:\s*)+', re.I) + stripped = strip_pat.sub('', subject) + return stripped def decode_charset(self, field): - if field.find("=?") == -1: - return None - # Get the decoded header as a list of (s, charset) tuples - pairs = decode_header(field) - # Use __unicode__() until we can guarantee Python 2.2 + # TK: This function was rewritten for unifying to Unicode. + # Convert 'field' into Unicode one line string. try: - # Use a large number for maxlinelen so it won't get wrapped - h = make_header(pairs, 99999) - return h.__unicode__() - except (UnicodeError, LookupError): - # Unknown encoding - return None - # The last value for c will have the proper charset in it - return EMPTYSTRING.join([s for s, c in pairs]) + pairs = decode_header(field) + ustr = make_header(pairs).__unicode__() + except (LookupError, UnicodeError, ValueError, HeaderParseError): + # assume list's language + cset = Utils.GetCharSet(self._mlist.preferred_language) + if cset == 'us-ascii': + cset = 'iso-8859-1' # assume this for English list + ustr = unicode(field, cset, 'replace') + return u''.join(ustr.splitlines()) def as_html(self): d = self.__dict__.copy() @@ -538,7 +552,15 @@ class Article(pipermail.Article): body = EMPTYSTRING.join(self.body) if isinstance(body, types.UnicodeType): body = body.encode(Utils.GetCharSet(self._lang), 'replace') - return NL.join(headers) % d + '\n\n' + body + if mm_cfg.ARCHIVER_OBSCURES_EMAILADDRS: + otrans = i18n.get_translation() + try: + i18n.set_language(self._lang) + body = re.sub(r'([-+,.\w]+)@([-+.\w]+)', + '\g<1>' + _(' at ') + '\g<2>', body) + finally: + i18n.set_translation(otrans) + return NL.join(headers) % d + '\n\n' + body + '\n' def _set_date(self, message): self.__super_set_date(message) @@ -559,6 +581,12 @@ class Article(pipermail.Article): break self.body.append(line) + def finished_update_article(self): + self.body = [] + try: + del self.html_body + except AttributeError: + pass class HyperArchive(pipermail.T): @@ -735,13 +763,14 @@ class HyperArchive(pipermail.T): d["archive_listing"] = EMPTYSTRING.join(accum) finally: i18n.set_translation(otrans) - # The TOC is always in the charset of the list's preferred language d['meta'] += html_charset % Utils.GetCharSet(mlist.preferred_language) - - return quick_maketext( - 'archtoc.html', d, - mlist=mlist) + # The site can disable public access to the mbox file. + if mm_cfg.PUBLIC_MBOX: + template = 'archtoc.html' + else: + template = 'archtocnombox.html' + return quick_maketext(template, d, mlist=mlist) def html_TOC_entry(self, arch): # Check to see if the archive is gzip'd or not @@ -996,7 +1025,11 @@ class HyperArchive(pipermail.T): subject = self.get_header("subject", article) author = self.get_header("author", article) if mm_cfg.ARCHIVER_OBSCURES_EMAILADDRS: - author = re.sub('@', _(' at '), author) + try: + author = re.sub('@', _(' at '), author) + except UnicodeError: + # Non-ASCII author contains '@' ... no valid email anyway + pass subject = CGIescape(subject, self.lang) author = CGIescape(author, self.lang) @@ -1121,6 +1154,10 @@ class HyperArchive(pipermail.T): # 1. use lines directly, rather than source and dest # 2. make it clearer # 3. make it faster + # TK: Prepare for unicode obscure. + atmark = _(' at ') + if lines and isinstance(lines[0], types.UnicodeType): + atmark = unicode(atmark, Utils.GetCharSet(self.lang), 'replace') source = lines[:] dest = lines last_line_was_quoted = 0 @@ -1161,7 +1198,7 @@ class HyperArchive(pipermail.T): text = jr.group(1) length = len(text) if mm_cfg.ARCHIVER_OBSCURES_EMAILADDRS: - text = re.sub('@', _(' at '), text) + text = re.sub('@', atmark, text) URL = self.maillist.GetScriptURL( 'listinfo', absolute=1) else: diff --git a/Mailman/Archiver/pipermail.py b/Mailman/Archiver/pipermail.py index 210030ed8..fac7e5ed6 100644 --- a/Mailman/Archiver/pipermail.py +++ b/Mailman/Archiver/pipermail.py @@ -7,7 +7,7 @@ import os import re import sys import time -from email.Utils import parseaddr, parsedate_tz, mktime_tz +from email.Utils import parseaddr, parsedate_tz, mktime_tz, formatdate import cPickle as pickle from cStringIO import StringIO from string import lowercase @@ -126,9 +126,13 @@ class Database(DatabaseInterface): """Store article without message body to save space""" # TBD this is not thread safe! temp = article.body + temp2 = article.html_body article.body = [] + del article.html_body self.articleIndex[article.msgid] = pickle.dumps(article) article.body = temp + article.html_body = temp2 + # The Article class encapsulates a single posting. The attributes # are: @@ -213,7 +217,8 @@ class Article: self.headers[i] = message[i] # Read the message body - s = StringIO(message.get_payload()) + s = StringIO(message.get_payload(decode=1)\ + or message.as_string().split('\n\n',1)[1]) self.body = s.readlines() def _set_date(self, message): @@ -235,10 +240,16 @@ class Article: date = self._last_article_time + 1 self._last_article_time = date self.date = '%011i' % date + self.datestr = message.get('date') \ + or message.get('x-list-received-date') \ + or formatdate(date) def __repr__(self): return '<Article ID = '+repr(self.msgid)+'>' + def finished_update_article(self): + pass + # Pipermail formatter class class T: @@ -486,6 +497,8 @@ class T: self.update_article(arcdir, a1, L[0], L[2]) else: del self.database.changed[key] + if L[0]: + L[0].finished_update_article() L = L[1:] # Rotate the list if msgid is None: L.append(msgid) @@ -600,8 +613,14 @@ class T: self.write_article(arch, temp, os.path.join(archivedir, filename)) - author = fixAuthor(article.author) - subject = article.subject.lower() + if article.decoded.has_key('author'): + author = fixAuthor(article.decoded['author']) + else: + author = fixAuthor(article.author) + if article.decoded.has_key('stripped'): + subject = article.decoded['stripped'].lower() + else: + subject = article.subject.lower() article.parentID = parentID = self.get_parent_info(arch, article) if parentID: diff --git a/Mailman/Bouncer.py b/Mailman/Bouncer.py index 360e1ff06..e2a9de6c3 100644 --- a/Mailman/Bouncer.py +++ b/Mailman/Bouncer.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2004 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -52,9 +52,9 @@ _ = i18n._ class _BounceInfo: - def __init__(self, member, score, date, noticesleft, cookie): + def __init__(self, member, score, date, noticesleft): self.member = member - self.cookie = cookie + self.cookie = None self.reset(score, date, noticesleft) def reset(self, score, date, noticesleft): @@ -111,11 +111,8 @@ class Bouncer: day = time.localtime()[:3] if not isinstance(info, _BounceInfo): # This is the first bounce we've seen from this member - cookie = Pending.new(Pending.RE_ENABLE, self.internal_name(), - member) info = _BounceInfo(member, weight, day, - self.bounce_you_are_disabled_warnings, - cookie) + self.bounce_you_are_disabled_warnings) self.setBounceInfo(member, info) syslog('bounce', '%s: %s bounce score: %s', self.internal_name(), member, info.score) @@ -131,7 +128,7 @@ class Bouncer: # We've already scored any bounces for this day, so ignore it. syslog('bounce', '%s: %s already scored a bounce for date %s', self.internal_name(), member, - time.strftime('%d-%b-%Y', day + (0,)*6)) + time.strftime('%d-%b-%Y', day + (0,0,0,0,1,0))) # Continue to check phase below else: # See if this member's bounce information is stale. @@ -154,13 +151,29 @@ class Bouncer: # Now that we've adjusted the bounce score for this bounce, let's # check to see if the disable-by-bounce threshold has been reached. if info.score >= self.bounce_score_threshold: - self.disableBouncingMember(member, info, msg) + if mm_cfg.VERP_PROBES: + syslog('bounce', + 'sending %s list probe to: %s (score %s >= %s)', + self.internal_name(), member, info.score, + self.bounce_score_threshold) + self.sendProbe(member, msg) + info.reset(0, info.date, info.noticesleft) + else: + self.disableBouncingMember(member, info, msg) def disableBouncingMember(self, member, info, msg): + # Initialize their confirmation cookie. If we do it when we get the + # first bounce, it'll expire by the time we get the disabling bounce. + cookie = self.pend_new(Pending.RE_ENABLE, self.internal_name(), member) + info.cookie = cookie # Disable them - syslog('bounce', '%s: %s disabling due to bounce score %s >= %s', - self.internal_name(), member, - info.score, self.bounce_score_threshold) + if mm_cfg.VERP_PROBES: + syslog('bounce', '%s: %s disabling due to probe bounce received', + self.internal_name(), member) + else: + syslog('bounce', '%s: %s disabling due to bounce score %s >= %s', + self.internal_name(), member, + info.score, self.bounce_score_threshold) self.setDeliveryStatus(member, MemberAdaptor.BYBOUNCE) self.sendNextNotification(member) if self.bounce_notify_owner_on_disable: @@ -211,7 +224,7 @@ class Bouncer: userack=1) # Expunge the pending cookie for the user. We throw away the # returned data. - Pending.confirm(info.cookie) + self.pend_confirm(info.cookie) if reason == MemberAdaptor.BYBOUNCE: syslog('bounce', '%s: %s deleted after exhausting notices', self.internal_name(), member) @@ -264,6 +277,8 @@ class Bouncer: # provided in the exception argument. sender = msg.get_sender() subject = msg.get('subject', _('(no subject)')) + subject = Utils.oneline(subject, + Utils.GetCharSet(self.preferred_language)) if e is None: notice = _('[No bounce details are available]') else: diff --git a/Mailman/Bouncers/Postfix.py b/Mailman/Bouncers/Postfix.py index 0231434fe..1fab8666f 100644 --- a/Mailman/Bouncers/Postfix.py +++ b/Mailman/Bouncers/Postfix.py @@ -71,7 +71,7 @@ def findaddr(msg): def process(msg): - if msg.get_type() <> 'multipart/mixed': + if msg.get_type() not in ('multipart/mixed', 'multipart/report'): return None # We're looking for the plain/text subpart with a Content-Description: of # `notification'. diff --git a/Mailman/Cgi/admin.py b/Mailman/Cgi/admin.py index 94ed8bda1..eb63e0081 100644 --- a/Mailman/Cgi/admin.py +++ b/Mailman/Cgi/admin.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2004 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -14,9 +14,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -"""Process and produce the list-administration options forms. - -""" +"""Process and produce the list-administration options forms.""" # For Python 2.1.x compatibility from __future__ import nested_scopes @@ -51,6 +49,12 @@ i18n.set_language(mm_cfg.DEFAULT_SERVER_LANGUAGE) NL = '\n' OPTCOLUMNS = 11 +try: + True, False +except NameError: + True = 1 + False = 0 + def main(): @@ -232,8 +236,9 @@ def admin_overview(msg=''): # List is for different identity of this host - skip it. continue else: - advertised.append(mlist) - + advertised.append((mlist.GetScriptURL('admin'), + mlist.real_name, + mlist.description)) # Greeting depends on whether there was an error or not if msg: greeting = FontAttr(msg, color="ff5060", size="+1") @@ -283,10 +288,10 @@ def admin_overview(msg=''): Bold(FontAttr(_('Description'), size='+2')) ]) highlight = 1 - for mlist in advertised: + for url, real_name, description in advertised: table.AddRow( - [Link(mlist.GetScriptURL('admin'), Bold(mlist.real_name)), - mlist.description or Italic(_('[no description available]'))]) + [Link(url, Bold(real_name)), + description or Italic(_('[no description available]'))]) if highlight and mm_cfg.WEB_HIGHLIGHT_COLOR: table.AddRowInfo(table.GetCurrentRowIndex(), bgcolor=mm_cfg.WEB_HIGHLIGHT_COLOR) @@ -406,7 +411,7 @@ def show_results(mlist, doc, category, subcat, cgidata): otherlinks.AddItem(Link(mlist.GetScriptURL('listinfo'), _('Go to the general list information page'))) otherlinks.AddItem(Link(mlist.GetScriptURL('edithtml'), - _('Edit the public HTML pages'))) + _('Edit the public HTML pages and text files'))) otherlinks.AddItem(Link(mlist.GetBaseArchiveURL(), _('Go to list archives')).Format() + '<br> <br>') @@ -678,7 +683,7 @@ def get_item_gui_value(mlist, category, kind, varname, params, extra): # and a delete button. Yeesh! params are ignored. table = Table(border=0) # This adds the html for the entry widget - def makebox(i, name, pattern, desc, empty=0, table=table): + def makebox(i, name, pattern, desc, empty=False, table=table): deltag = 'topic_delete_%02d' % i boxtag = 'topic_box_%02d' % i reboxtag = 'topic_rebox_%02d' % i @@ -718,7 +723,71 @@ def get_item_gui_value(mlist, category, kind, varname, params, extra): # Add one more non-deleteable widget as the first blank entry, but # only if there are no real entries. if i == 1: - makebox(i, '', '', '', empty=1) + makebox(i, '', '', '', empty=True) + return table + elif kind == mm_cfg.HeaderFilter: + # A complex and specialized widget type that allows for setting of a + # spam filter rule including, a mark button, a regexp text box, an + # "add after mark", up and down buttons, and a delete button. Yeesh! + # params are ignored. + table = Table(border=0) + # This adds the html for the entry widget + def makebox(i, pattern, action, empty=False, table=table): + deltag = 'hdrfilter_delete_%02d' % i + reboxtag = 'hdrfilter_rebox_%02d' % i + actiontag = 'hdrfilter_action_%02d' % i + wheretag = 'hdrfilter_where_%02d' % i + addtag = 'hdrfilter_add_%02d' % i + newtag = 'hdrfilter_new_%02d' % i + uptag = 'hdrfilter_up_%02d' % i + downtag = 'hdrfilter_down_%02d' % i + if empty: + table.AddRow([Center(Bold(_('Spam Filter Rule %(i)d'))), + Hidden(newtag)]) + else: + table.AddRow([Center(Bold(_('Spam Filter Rule %(i)d'))), + SubmitButton(deltag, _('Delete'))]) + table.AddRow([Label(_('Spam Filter Regexp:')), + TextArea(reboxtag, text=pattern, + rows=4, cols=30, wrap='off')]) + values = [mm_cfg.DEFER, mm_cfg.HOLD, mm_cfg.REJECT, + mm_cfg.DISCARD, mm_cfg.ACCEPT] + try: + checked = values.index(action) + except ValueError: + checked = 0 + radio = RadioButtonArray( + actiontag, + (_('Defer'), _('Hold'), _('Reject'), + _('Discard'), _('Accept')), + values=values, + checked=checked).Format() + table.AddRow([Label(_('Action:')), radio]) + if not empty: + table.AddRow([SubmitButton(addtag, _('Add new item...')), + SelectOptions(wheretag, ('before', 'after'), + (_('...before this one.'), + _('...after this one.')), + selected=1), + ]) + # BAW: IWBNI we could disable the up and down buttons for the + # first and last item respectively, but it's not easy to know + # which is the last item, so let's not worry about that for + # now. + table.AddRow([SubmitButton(uptag, _('Move rule up')), + SubmitButton(downtag, _('Move rule down'))]) + table.AddRow(['<hr>']) + table.AddCellInfo(table.GetCurrentRowIndex(), 0, colspan=2) + # Now for each element in the existing data, create a widget + i = 1 + data = getattr(mlist, varname) + for pattern, action, empty in data: + makebox(i, pattern, action, empty) + i += 1 + # Add one more non-deleteable widget as the first blank entry, but + # only if there are no real entries. + if i == 1: + makebox(i, '', mm_cfg.DEFER, empty=True) return table elif kind == mm_cfg.Checkbox: return CheckBoxArray(varname, *params) @@ -1262,7 +1331,8 @@ def change_options(mlist, category, subcat, cgidata, doc): mlist.InviteNewMember(userdesc, invitation) else: mlist.ApprovedAddMember(userdesc, send_welcome_msg, - send_admin_notif, invitation) + send_admin_notif, invitation, + whence='admin mass sub') except Errors.MMAlreadyAMember: subscribe_errors.append((entry, _('Already a member'))) except Errors.MMBadEmailError: @@ -1353,7 +1423,7 @@ def change_options(mlist, category, subcat, cgidata, doc): for user in users: if cgidata.has_key('%s_unsub' % user): try: - mlist.ApprovedDeleteMember(user) + mlist.ApprovedDeleteMember(user, whence='member mgt page') removes.append(user) except Errors.NotAMemberError: errors.append((user, _('Not subscribed'))) diff --git a/Mailman/Cgi/admindb.py b/Mailman/Cgi/admindb.py index 1af5b2581..1dd0e28a3 100644 --- a/Mailman/Cgi/admindb.py +++ b/Mailman/Cgi/admindb.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2004 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -155,7 +155,7 @@ def main(): realname = mlist.real_name if not cgidata.keys(): # If this is not a form submission (i.e. there are no keys in the - # form), then all we don't need to do much special. + # form), then we don't need to do much special. doc.SetTitle(_('%(realname)s Administrative Database')) elif not details: # This is a form submission @@ -179,7 +179,7 @@ def main(): admindburl = mlist.GetScriptURL('admindb', absolute=1) form = Form(admindburl) # Add the instructions template - if details: + if details == 'instructions': doc.AddItem(Header( 2, _('Detailed instructions for the administrative database'))) else: @@ -187,8 +187,14 @@ def main(): 2, _('Administrative requests for mailing list:') + ' <em>%s</em>' % mlist.real_name)) - if not details: + if details <> 'instructions': form.AddItem(Center(SubmitButton('submit', _('Submit All Data')))) + if not (sender or msgid): + form.AddItem(Center( + CheckBox('discardalldefersp', 0).Format() + + ' ' + + _('Discard all messages marked <em>Defer</em>') + )) # Add a link back to the overview, if we're not viewing the overview! adminurl = mlist.GetScriptURL('admin', absolute=1) d = {'listname' : mlist.real_name, @@ -231,6 +237,12 @@ def main(): if addform: doc.AddItem(form) form.AddItem('<hr>') + if not (sender or msgid): + form.AddItem(Center( + CheckBox('discardalldefersp', 0).Format() + + ' ' + + _('Discard all messages marked <em>Defer</em>') + )) form.AddItem(Center(SubmitButton('submit', _('Submit All Data')))) doc.AddItem(mlist.GetMailmanFooter()) print doc.Format() @@ -298,7 +310,9 @@ def show_pending_subs(mlist, form): if addr not in mlist.ban_list: radio += '<br>' + CheckBox('ban-%d' % id, 1).Format() + \ ' ' + _('Permanently ban from this list') - table.AddRow(['%s<br><em>%s</em>' % (addr, fullname), + # While the address may be a unicode, it must be ascii + paddr = addr.encode('us-ascii', 'replace') + table.AddRow(['%s<br><em>%s</em>' % (paddr, fullname), radio, TextBox('comment-%d' % id, size=40) ]) @@ -470,10 +484,12 @@ def show_helds_overview(mlist, form): # handled by the time we got here. mlist.HandleRequest(id, mm_cfg.DISCARD) continue + dispsubj = Utils.oneline( + subject, Utils.GetCharSet(mlist.preferred_language)) t = Table(border=0) t.AddRow([Link(admindburl + '?msgid=%d' % id, '[%d]' % counter), Bold(_('Subject:')), - Utils.websafe(subject) + Utils.websafe(dispsubj) ]) t.AddRow([' ', Bold(_('Size:')), str(size) + _(' bytes')]) if reason: @@ -589,6 +605,14 @@ def show_post_requests(mlist, id, info, total, count, form): body = EMPTYSTRING.join(lines)[:mm_cfg.ADMINDB_PAGE_TEXT_LIMIT] else: body = EMPTYSTRING.join(lines) + # Get message charset and try encode in list charset + mcset = msg.get_param('charset', 'us-ascii').lower() + lcset = Utils.GetCharSet(mlist.preferred_language) + if mcset <> lcset: + try: + body = unicode(body, mcset).encode(lcset) + except (LookupError, UnicodeError, ValueError): + pass hdrtxt = NL.join(['%s: %s' % (k, v) for k, v in msg.items()]) hdrtxt = Utils.websafe(hdrtxt) # Okay, we've reconstituted the message just fine. Now for the fun part! @@ -596,7 +620,8 @@ def show_post_requests(mlist, id, info, total, count, form): t.AddRow([Bold(_('From:')), sender]) row, col = t.GetCurrentRowIndex(), t.GetCurrentCellIndex() t.AddCellInfo(row, col-1, align='right') - t.AddRow([Bold(_('Subject:')), Utils.websafe(subject)]) + t.AddRow([Bold(_('Subject:')), + Utils.websafe(Utils.oneline(subject, lcset))]) t.AddCellInfo(row+1, col-1, align='right') t.AddRow([Bold(_('Reason:')), _(reason)]) t.AddCellInfo(row+2, col-1, align='right') @@ -661,6 +686,11 @@ def process_form(mlist, doc, cgidata): sender = unquote_plus(k[len(prefix):]) value = cgidata.getvalue(k) senderactions.setdefault(sender, {})[action] = value + # discard-all-defers + try: + discardalldefersp = cgidata.getvalue('discardalldefersp', 0) + except ValueError: + discardalldefersp = 0 for sender in senderactions.keys(): actions = senderactions[sender] # Handle what to do about all this sender's held messages @@ -668,6 +698,8 @@ def process_form(mlist, doc, cgidata): action = int(actions.get('senderaction', mm_cfg.DEFER)) except ValueError: action = mm_cfg.DEFER + if action == mm_cfg.DEFER and discardalldefersp: + action = mm_cfg.DISCARD if action in (mm_cfg.DEFER, mm_cfg.APPROVE, mm_cfg.REJECT, mm_cfg.DISCARD): preserve = actions.get('senderpreserve', 0) diff --git a/Mailman/Cgi/confirm.py b/Mailman/Cgi/confirm.py index 096596cc5..d2cf0cce7 100644 --- a/Mailman/Cgi/confirm.py +++ b/Mailman/Cgi/confirm.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2003 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2004 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -33,6 +33,12 @@ from Mailman.Logging.Syslog import syslog _ = i18n._ i18n.set_language(mm_cfg.DEFAULT_SERVER_LANGUAGE) +try: + True, False +except NameError: + True = 1 + False = 0 + def main(): @@ -95,7 +101,7 @@ def main(): Otherwise, <a href="%(confirmurl)s">re-enter</a> your confirmation string.''') - content = Pending.confirm(cookie, expunge=0) + content = mlist.pend_confirm(cookie, expunge=False) if content is None: bad_confirmation(doc, badconfirmstr) doc.AddItem(mlist.GetMailmanFooter()) @@ -122,8 +128,8 @@ def main(): doc.addError(_("""The address requesting unsubscription is not a member of the mailing list. Perhaps you have already been unsubscribed, e.g. by the list administrator?""")) - # And get rid of this confirmation cookie - Pending.confirm(cookie) + # Expunge this record from the pending database. + expunge(mlist, cookie) elif content[0] == Pending.CHANGE_OF_ADDRESS: if cgidata.getvalue('cancel'): addrchange_cancel(mlist, doc, cookie) @@ -138,7 +144,8 @@ def main(): doc.addError(_("""The address requesting to be changed has been subsequently unsubscribed. This request has been cancelled.""")) - Pending.confirm(cookie, expunge=1) + # Expunge this record from the pending database. + expunge(mlist, cookie) elif content[0] == Pending.HELD_MESSAGE: if cgidata.getvalue('cancel'): heldmsg_cancel(mlist, doc, cookie) @@ -170,6 +177,17 @@ def bad_confirmation(doc, extra=''): doc.AddItem(extra) +def expunge(mlist, cookie): + # Expunge this record from the list's pending database. This requires + # that the list lock be acquired, however the list doesn't need to be + # saved because this operation doesn't touch the config.pck file. + mlist.Lock() + try: + mlist.pend_confirm(cookie, expunge=True) + finally: + mlist.Unlock() + + def ask_for_cookie(mlist, doc, extra=''): title = _('Enter confirmation cookie') @@ -235,8 +253,8 @@ def subscription_prompt(mlist, doc, cookie, userdesc): <p>Note: your password will be emailed to you once your subscription is confirmed. You can change it by visiting your personal options page. - <p>Or hit <em>Cancel and discard</em> to cancel this subscription - request.""") + '<p><hr>' + <p>Or hit <em>Cancel my subscription request</em> if you no longer want to + subscribe to this list.""") + '<p><hr>' if mlist.subscribe_policy in (2, 3): # Confirmation is required result = _("""Your confirmation is required in order to continue with @@ -290,8 +308,12 @@ def subscription_prompt(mlist, doc, cookie, userdesc): def subscription_cancel(mlist, doc, cookie): - # Discard this cookie - userdesc = Pending.confirm(cookie, expunge=1)[1] + mlist.Lock() + try: + # Discard this cookie + userdesc = mlist.pend_confirm(cookie)[1] + finally: + mlist.Unlock() lang = userdesc.language i18n.set_language(lang) doc.set_language(lang) @@ -324,7 +346,7 @@ def subscription_confirm(mlist, doc, cookie, cgidata): digest = None else: digest = None - userdesc = Pending.confirm(cookie, expunge=0)[1] + userdesc = mlist.pend_confirm(cookie, expunge=False)[1] fullname = cgidata.getvalue('realname', None) if fullname is not None: fullname = Utils.canonstr(fullname, lang) @@ -379,8 +401,8 @@ def subscription_confirm(mlist, doc, cookie, cgidata): def unsubscription_cancel(mlist, doc, cookie): - # Discard this cookie - Pending.confirm(cookie, expunge=1) + # Expunge this record from the pending database + expunge(mlist, cookie) doc.AddItem(_('You have canceled your unsubscription request.')) @@ -397,7 +419,7 @@ def unsubscription_confirm(mlist, doc, cookie): try: # Do this in two steps so we can get the preferred language for # the user who is unsubscribing. - op, addr = Pending.confirm(cookie, expunge=0) + op, addr = mlist.pend_confirm(cookie, expunge=False) lang = mlist.getMemberLanguage(addr) i18n.set_language(lang) doc.set_language(lang) @@ -467,8 +489,8 @@ def unsubscription_prompt(mlist, doc, cookie, addr): def addrchange_cancel(mlist, doc, cookie): - # Discard this cookie - Pending.confirm(cookie, expunge=1) + # Expunge this record from the pending database + expunge(mlist, cookie) doc.AddItem(_('You have canceled your change of address request.')) @@ -485,7 +507,8 @@ def addrchange_confirm(mlist, doc, cookie): try: # Do this in two steps so we can get the preferred language for # the user who is unsubscribing. - op, oldaddr, newaddr, globally = Pending.confirm(cookie, expunge=0) + op, oldaddr, newaddr, globally = mlist.pend_confirm( + cookie, expunge=False) lang = mlist.getMemberLanguage(oldaddr) i18n.set_language(lang) doc.set_language(lang) @@ -565,14 +588,14 @@ def addrchange_prompt(mlist, doc, cookie, oldaddr, newaddr, globally): def heldmsg_cancel(mlist, doc, cookie): - # Discard this cookie title = _('Continue awaiting approval') doc.SetTitle(title) table = Table(border=0, width='100%') table.AddRow([Center(Bold(FontAttr(title, size='+1')))]) table.AddCellInfo(table.GetCurrentRowIndex(), 0, bgcolor=mm_cfg.WEB_HEADER_COLOR) - Pending.confirm(cookie, expunge=1) + # Expunge this record from the pending database. + expunge(mlist, cookie) table.AddRow([_('''Okay, the list moderator will still have the opportunity to approve or reject this message.''')]) doc.AddItem(table) @@ -591,7 +614,7 @@ def heldmsg_confirm(mlist, doc, cookie): try: # Do this in two steps so we can get the preferred language for # the user who posted the message. - op, id = Pending.confirm(cookie, expunge=1) + op, id = mlist.pend_confirm(cookie) ign, sender, msgsubject, ign, ign, ign = mlist.GetRecord(id) subject = Utils.websafe(msgsubject) lang = mlist.getMemberLanguage(sender) @@ -708,7 +731,7 @@ def reenable_confirm(mlist, doc, cookie): try: # Do this in two steps so we can get the preferred language for # the user who is unsubscribing. - op, listname, addr = Pending.confirm(cookie, expunge=0) + op, listname, addr = mlist.pend_confirm(cookie, expunge=False) lang = mlist.getMemberLanguage(addr) i18n.set_language(lang) doc.set_language(lang) @@ -758,7 +781,8 @@ def reenable_prompt(mlist, doc, cookie, list, member): <a href="%(listinfourl)s">list information page</a>.""")]) return - date = time.strftime('%A, %B %d, %Y', info.date + (0,) * 6) + date = time.strftime('%A, %B %d, %Y', + time.localtime(time.mktime(info.date + (0,)*6))) daysleft = int(info.noticesleft * mlist.bounce_you_are_disabled_warnings_interval / mm_cfg.days(1)) diff --git a/Mailman/Cgi/create.py b/Mailman/Cgi/create.py index 90e849ba4..540d9d6cc 100644 --- a/Mailman/Cgi/create.py +++ b/Mailman/Cgi/create.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2003 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2004 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -93,7 +93,7 @@ def process_request(doc, cgidata): auth = cgidata.getvalue('auth', '').strip() langs = cgidata.getvalue('langs', [mm_cfg.DEFAULT_SERVER_LANGUAGE]) - if type(langs) <> ListType: + if not isinstance(langs, ListType): langs = [langs] # Sanity check safelistname = Utils.websafe(listname) @@ -125,7 +125,8 @@ def process_request(doc, cgidata): blank if you want Mailman to autogenerate the list passwords.''')) return - password = confirm = Utils.MakeRandomPassword(length=8) + password = confirm = Utils.MakeRandomPassword( + mm_cfg.ADMIN_PASSWORD_LENGTH) else: if password <> confirm: request_creation(doc, cgidata, @@ -152,6 +153,15 @@ def process_request(doc, cgidata): doc, cgidata, _('You are not authorized to create new mailing lists')) return + # Make sure the web hostname matches one of our virtual domains + hostname = Utils.get_domain() + if mm_cfg.VIRTUAL_HOST_OVERVIEW and \ + not mm_cfg.VIRTUAL_HOSTS.has_key(hostname): + safehostname = Utils.websafe(hostname) + request_creation(doc, cgidata, + _('Unknown virtual host: %(safehostname)s')) + return + emailhost = mm_cfg.VIRTUAL_HOSTS.get(hostname, mm_cfg.DEFAULT_EMAIL_HOST) # We've got all the data we need, so go ahead and try to create the list # See admin.py for why we need to set up the signal handler. mlist = MailList.MailList() @@ -175,10 +185,10 @@ def process_request(doc, cgidata): oldmask = os.umask(002) try: try: - mlist.Create(listname, owner, pw, langs) + mlist.Create(listname, owner, pw, langs, emailhost) finally: os.umask(oldmask) - except Errors.MMBadEmailError, s: + except Errors.EmailAddressError, s: request_creation(doc, cgidata, _('Bad owner email address: %(s)s')) return @@ -199,11 +209,9 @@ def process_request(doc, cgidata): # Initialize the host_name and web_page_url attributes, based on # virtual hosting settings and the request environment variables. - hostname = Utils.get_domain() mlist.default_member_moderation = moderate mlist.web_page_url = mm_cfg.DEFAULT_URL_PATTERN % hostname - mlist.host_name = mm_cfg.VIRTUAL_HOSTS.get( - hostname, mm_cfg.DEFAULT_EMAIL_HOST) + mlist.host_name = emailhost mlist.Save() finally: # Now be sure to unlock the list. It's okay if we get a signal here @@ -220,7 +228,7 @@ def process_request(doc, cgidata): # And send the notice to the list owner. if notify: - siteadmin = Utils.get_site_email(mlist.host_name, 'admin') + siteowner = Utils.get_site_email(mlist.host_name, 'owner') text = Utils.maketext( 'newlist.txt', {'listname' : listname, @@ -228,10 +236,10 @@ def process_request(doc, cgidata): 'admin_url' : mlist.GetScriptURL('admin', absolute=1), 'listinfo_url': mlist.GetScriptURL('listinfo', absolute=1), 'requestaddr' : mlist.GetRequestEmail(), - 'siteowner' : siteadmin, + 'siteowner' : siteowner, }, mlist=mlist) msg = Message.UserNotification( - owner, siteadmin, + owner, siteowner, _('Your new mailing list: %(listname)s'), text, mlist.preferred_language) msg.send(mlist) diff --git a/Mailman/Cgi/edithtml.py b/Mailman/Cgi/edithtml.py index d52fda1fc..9a07cdc0e 100644 --- a/Mailman/Cgi/edithtml.py +++ b/Mailman/Cgi/edithtml.py @@ -1,17 +1,17 @@ -# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2005 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software +# along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. """Script which implements admin editing of the list's html templates.""" @@ -43,6 +43,7 @@ def main(): ('listinfo.html', _('General list information page')), ('subscribe.html', _('Subscribe results page')), ('options.html', _('User specific options page')), + ('subscribeack.txt', _('Welcome email text file')), ) _ = i18n._ @@ -157,10 +158,14 @@ def ChangeHTML(mlist, cgi_info, template_name, doc): code = cgi_info['html_code'].value langdir = os.path.join(mlist.fullpath(), mlist.preferred_language) # Make sure the directory exists + omask = os.umask(0) try: - os.mkdir(langdir, 02775) - except OSError, e: - if e.errno <> errno.EEXIST: raise + try: + os.mkdir(langdir, 02775) + except OSError, e: + if e.errno <> errno.EEXIST: raise + finally: + os.umask(omask) fp = open(os.path.join(langdir, template_name), 'w') try: fp.write(code) diff --git a/Mailman/Cgi/listinfo.py b/Mailman/Cgi/listinfo.py index 10e0c41e9..abbf570b9 100644 --- a/Mailman/Cgi/listinfo.py +++ b/Mailman/Cgi/listinfo.py @@ -91,8 +91,9 @@ def listinfo_overview(msg=''): # List is for different identity of this host - skip it. continue else: - advertised.append(mlist) - + advertised.append((mlist.GetScriptURL('listinfo'), + mlist.real_name, + mlist.description)) if msg: greeting = FontAttr(msg, color="ff5060", size="+1") else: @@ -135,10 +136,10 @@ def listinfo_overview(msg=''): Bold(FontAttr(_('Description'), size='+2')) ]) highlight = 1 - for mlist in advertised: + for url, real_name, description in advertised: table.AddRow( - [Link(mlist.GetScriptURL('listinfo'), Bold(mlist.real_name)), - mlist.description or Italic(_('[no description available]'))]) + [Link(url, Bold(real_name)), + description or Italic(_('[no description available]'))]) if highlight and mm_cfg.WEB_HIGHLIGHT_COLOR: table.AddRowInfo(table.GetCurrentRowIndex(), bgcolor=mm_cfg.WEB_HIGHLIGHT_COLOR) diff --git a/Mailman/Cgi/options.py b/Mailman/Cgi/options.py index 09f629874..0ebd76329 100644 --- a/Mailman/Cgi/options.py +++ b/Mailman/Cgi/options.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2004 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -39,6 +39,12 @@ SETLANGUAGE = -1 _ = i18n._ i18n.set_language(mm_cfg.DEFAULT_SERVER_LANGUAGE) +try: + True, False +except NameError: + True = 1 + False = 0 + def main(): @@ -157,8 +163,23 @@ def main(): # Because they can't supply a password for unsubscribing, we'll need # to do the confirmation dance. if mlist.isMember(user): - mlist.ConfirmUnsubscription(user, userlang) - doc.addError(_('The confirmation email has been sent.'), tag='') + # We must acquire the list lock in order to pend a request. + try: + mlist.Lock() + # If unsubs require admin approval, then this request has to + # be held. Otherwise, send a confirmation. + if mlist.unsubscribe_policy: + mlist.HoldUnsubscription(user) + doc.addError(_("""Your unsubscription request has been + forwarded to the list administrator for approval."""), + tag='') + else: + mlist.ConfirmUnsubscription(user, userlang) + doc.addError(_('The confirmation email has been sent.'), + tag='') + mlist.Save() + finally: + mlist.Unlock() else: # Not a member if mlist.private_roster == 0: @@ -199,7 +220,6 @@ def main(): # Authenticate, possibly using the password supplied in the login page password = cgidata.getvalue('password', '').strip() - if not mlist.WebAuthenticate((mm_cfg.AuthUser, mm_cfg.AuthListAdmin, mm_cfg.AuthSiteAdmin), @@ -416,13 +436,13 @@ address. Upon confirmation, any other mailing list containing the address # list admin?) is informed of the removal. signal.signal(signal.SIGTERM, sigterm_handler) mlist.Lock() - needapproval = 0 + needapproval = False try: try: mlist.DeleteMember( user, 'via the member options page', userack=1) except Errors.MMNeedApproval: - needapproval = 1 + needapproval = True mlist.Save() finally: mlist.Unlock() @@ -609,7 +629,10 @@ address. Upon confirmation, any other mailing list containing the address print doc.Format() return - options_page(mlist, doc, user, cpuser, userlang) + if mlist.isMember(user): + options_page(mlist, doc, user, cpuser, userlang) + else: + loginpage(mlist, doc, user, userlang) print doc.Format() diff --git a/Mailman/Cgi/private.py b/Mailman/Cgi/private.py index 790d351c6..cbab3b066 100644 --- a/Mailman/Cgi/private.py +++ b/Mailman/Cgi/private.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2005 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -35,13 +35,16 @@ from Mailman.Logging.Syslog import syslog _ = i18n._ i18n.set_language(mm_cfg.DEFAULT_SERVER_LANGUAGE) +SLASH = '/' + def true_path(path): "Ensure that the path is safe by removing .." - path = path.replace('../', '') - path = path.replace('./', '') - return path[1:] + # Workaround for path traverse vulnerability. Unsuccessful attempts will + # be logged in logs/error. + parts = [x for x in path.split(SLASH) if x not in ('.', '..')] + return SLASH.join(parts)[1:] diff --git a/Mailman/Commands/cmd_confirm.py b/Mailman/Commands/cmd_confirm.py index e0e7b1c19..6574355d7 100644 --- a/Mailman/Commands/cmd_confirm.py +++ b/Mailman/Commands/cmd_confirm.py @@ -67,6 +67,9 @@ your email address?""")) res.results.append(_("""\ You were not invited to this mailing list. The invitation has been discarded, and both list administrators have been alerted.""")) + except Errors.MMBadPasswordError: + res.results.append(_("""\ +Bad approval password given. Held message is still being held.""")) else: if ((results[0] == Pending.SUBSCRIPTION and mlist.send_welcome_msg) or diff --git a/Mailman/Commands/cmd_password.py b/Mailman/Commands/cmd_password.py index 4d0a8e3e0..19093c0c4 100644 --- a/Mailman/Commands/cmd_password.py +++ b/Mailman/Commands/cmd_password.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002 by the Free Software Foundation, Inc. +# Copyright (C) 2002-2004 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -49,6 +49,8 @@ def process(res, args): if mlist.isMember(address): password = mlist.getMemberPassword(address) res.results.append(_('Your password is: %(password)s')) + # Prohibit multiple password retrievals. + return STOP else: listname = mlist.real_name res.results.append( @@ -62,6 +64,8 @@ def process(res, args): if mlist.isMember(address): password = mlist.getMemberPassword(address) res.results.append(_('Your password is: %(password)s')) + # Prohibit multiple password retrievals. + return STOP else: listname = mlist.real_name res.results.append( diff --git a/Mailman/Commands/cmd_set.py b/Mailman/Commands/cmd_set.py index b29095e6f..c68a9067d 100644 --- a/Mailman/Commands/cmd_set.py +++ b/Mailman/Commands/cmd_set.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002 by the Free Software Foundation, Inc. +# Copyright (C) 2002-2003 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -292,10 +292,16 @@ class SetCommands: status = self._status(res, args[0]) if status < 0: return STOP - # sense is reversed - mlist.setMemberOption(self.__address, mm_cfg.DisableDelivery, - not status) - res.results.append(_('delivery option set')) + # Delivery status is handled differently than other options. If + # status is true (set delivery on), then we enable delivery. + # Otherwise, we have to use the setDeliveryStatus() interface to + # specify that delivery was disabled by the user. + if status: + mlist.setDeliveryStatus(self.__address, MemberAdaptor.ENABLED) + res.results.append(_('delivery enabled')) + else: + mlist.setDeliveryStatus(self.__address, MemberAdaptor.BYUSER) + res.results.append(_('delivery disabled by user')) def set_myposts(self, res, args): mlist = res.mlist @@ -308,7 +314,7 @@ class SetCommands: mlist.setMemberOption(self.__address, mm_cfg.DontReceiveOwnPosts, not status) res.results.append(_('myposts option set')) - + def set_hide(self, res, args): mlist = res.mlist if len(args) <> 1: @@ -343,7 +349,7 @@ class SetCommands: mlist.setMemberOption(self.__address, mm_cfg.SuppressPasswordReminder, not status) res.results.append(_('reminder option set')) - + def process(res, args): diff --git a/Mailman/Commands/cmd_subscribe.py b/Mailman/Commands/cmd_subscribe.py index ee4b75a65..a653158a3 100644 --- a/Mailman/Commands/cmd_subscribe.py +++ b/Mailman/Commands/cmd_subscribe.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002 by the Free Software Foundation, Inc. +# Copyright (C) 2002-2005 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -84,9 +84,12 @@ def process(res, args): res.results.append(_('No valid address found to subscribe')) return STOP # Watch for encoded names - h = make_header(decode_header(realname)) - # BAW: in Python 2.2, use just unicode(h) - realname = h.__unicode__() + try: + h = make_header(decode_header(realname)) + # BAW: in Python 2.2, use just unicode(h) + realname = h.__unicode__() + except UnicodeError: + realname = u'' # Coerce to byte string if uh contains only ascii try: realname = realname.encode('us-ascii') diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in index 7cf454a9b..ac5c1f5f7 100644 --- a/Mailman/Defaults.py.in +++ b/Mailman/Defaults.py.in @@ -1,6 +1,6 @@ # -*- python -*- -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2005 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -146,6 +146,10 @@ def add_virtualhost(urlhost, emailhost=None): # And set the default add_virtualhost(DEFAULT_URL_HOST, DEFAULT_EMAIL_HOST) +# Note that you will want to run bin/fix_url.py to change the domain of an +# existing list. bin/fix_url.py must be run within the bin/withlist script, +# like so: bin/withlist -l -r bin/fix_url.py <listname> + ##### @@ -188,16 +192,6 @@ WEB_VLINK_COLOR = '' # If true, forces VLINK= WEB_HIGHLIGHT_COLOR = '#dddddd' # If true, alternating rows # in listinfo & admin display -# The driver script prints out a lot of information when a Mailman bug is -# encountered. This really helps for development, but it also reveals -# information about the host system that some administrators are not -# comfortable with. By setting STEALTH_MODE to On, you disable the printing -# of this information on the web pages. Set STEALTH_MODE to Off to enable -# better debugging. This information is still, and always, printed in the -# error logs. -STEALTH_MODE = On - - ##### # Archive defaults @@ -266,6 +260,16 @@ PRIVATE_EXTERNAL_ARCHIVER = No # should modify the Message object as necessary. ARCHIVE_SCRUBBER = 'Mailman.Handlers.Scrubber' +# Mailman.Handlers.Scrubber uses attachment's filename as is. +# If you don't like this (extremely long mime-encoded filename) then set +# this True. +SCRUBBER_DONT_USE_ATTACHMENT_FILENAME = False + +# Use of attachment filename extension per se is may be dangerous because +# virus fakes it. You can set this True if you filter the attachment by +# filename extension +SCRUBBER_USE_ATTACHMENT_FILENAME_EXTENSION = False + # This variable defines what happens to text/html subparts. They can be # stripped completely, escaped, or filtered through an external program. The # legal values are: @@ -338,6 +342,10 @@ DEFAULT_CHARSET = None # HTML-quoted in the archives. VERBATIM_ENCODING = ['iso-2022-jp'] +# When the archive is public, should Mailman also make the raw Unix mbox file +# publically available? +PUBLIC_MBOX = No + ##### @@ -459,6 +467,7 @@ GLOBAL_PIPELINE = [ 'Moderate', 'Hold', 'MimeDel', + 'Scrubber', 'Emergency', 'Tagger', 'CalcRecips', @@ -526,7 +535,7 @@ OWNER_PIPELINE = [ # printing of this log message. SMTP_LOG_EVERY_MESSAGE = ( 'smtp', - '%(msg_message-id)s smtp for %(#recips)d recips, completed in %(time).3f seconds') + '%(msg_message-id)s smtp to %(listname)s for %(#recips)d recips, completed in %(time).3f seconds') # This will only be printed if there were no immediate smtp failures. # Mutually exclusive with SMTP_LOG_REFUSED. @@ -593,6 +602,12 @@ VERP_FORMAT = '%(bounces)s+%(mailbox)s=%(host)s' # compiled case-insensitively. VERP_REGEXP = r'^(?P<bounces>[^+]+?)\+(?P<mailbox>[^=]+)=(?P<host>[^@]+)@.*$' +# VERP format and regexp for probe messages +VERP_PROBE_FORMAT = '%(bounces)s+%(token)s' +VERP_PROBE_REGEXP = r'^(?P<bounces>[^+]+?)\+(?P<token>[^@]+)@.*$' +# Set this Yes to activate VERP probe for disabling by bounce +VERP_PROBES = No + # A perfect opportunity for doing VERP is the password reminders, which are # already addressed individually to each recipient. Set this to Yes to enable # VERPs on all password reminders. @@ -617,7 +632,7 @@ VERP_DELIVERY_INTERVAL = 0 # friendly Subject: on the message, but requires cooperation from the MTA. # Format is like VERP_FORMAT above, but with the following substitutions: # -# %(confirm)s -- the list-confirm mailbox will be set here +# %(addr)s -- the list-confirm mailbox will be set here # %(cookie)s -- the confirmation cookie will be set here VERP_CONFIRM_FORMAT = '%(addr)s+%(cookie)s' @@ -755,22 +770,6 @@ OWNERS_CAN_ENABLE_PERSONALIZATION = No # want to edit the held message on disk. HOLD_MESSAGES_AS_PICKLES = Yes -# These define the available types of external message metadata formats, and -# the one to use by default. MARSHAL format uses Python's built-in marshal -# module. BSDDB_NATIVE uses the bsddb module compiled into Python, which -# links with whatever version of Berkeley db you've got on your system (in -# Python 2.0 this is included by default if configure can find it). ASCII -# format is a dumb repr()-based format with "key = value" Python assignments. -# It is human readable and editable (as Python source code) and is appropriate -# for execfile() food. -# -# Note! Make sure your queues are empty before you change this. -METAFMT_MARSHAL = 1 -METAFMT_BSDDB_NATIVE = 2 -METAFMT_ASCII = 3 - -METADATA_FORMAT = METAFMT_MARSHAL - # This variable controls the order in which list-specific category options are # presented in the admin cgi page. ADMIN_CATEGORIES = [ @@ -788,6 +787,21 @@ ADMIN_CATEGORIES = [ # list's config variable default_member_moderation. DEFAULT_NEW_MEMBER_OPTIONS = 256 +# Specify the type of passwords to use, when Mailman generates the passwords +# itself, as would be the case for membership requests where the user did not +# fill in a password, or during list creation, when auto-generation of admin +# passwords was selected. +# +# Set this value to Yes for classic Mailman user-friendly(er) passwords. +# These generate semi-pronounceable passwords which are easier to remember. +# Set this value to No to use more cryptographically secure, but harder to +# remember, passwords -- if your operating system and Python version support +# the necessary feature (specifically that /dev/urandom be available). +USER_FRIENDLY_PASSWORDS = Yes +# This value specifies the default lengths of member and list admin passwords +MEMBER_PASSWORD_LENGTH = 8 +ADMIN_PASSWORD_LENGTH = 10 + ##### @@ -806,6 +820,7 @@ DEFAULT_MAX_MESSAGE_SIZE = 40 # KB # These format strings will be expanded w.r.t. the dictionary for the # mailing list instance. DEFAULT_SUBJECT_PREFIX = "[%(real_name)s] " +# DEFAULT_SUBJECT_PREFIX = "[%(real_name)s %%d]" # for numbering DEFAULT_MSG_HEADER = "" DEFAULT_MSG_FOOTER = """_______________________________________________ %(real_name)s mailing list @@ -813,6 +828,20 @@ DEFAULT_MSG_FOOTER = """_______________________________________________ %(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s """ +# Where to put subject prefix for 'Re:' messages: +# +# old style: Re: [prefix] test +# new style: [prefix 123] Re: test ... (number is optional) +# +# Old style is default for backward compatibility. New style is forced if a +# list owner set %d (numbering) in prefix. If the site owner had applied new +# style patch (from SF patch area) before, he/she may want to set this No in +# mm_cfg.py. +OLD_STYLE_PREFIXING = Yes + +# Scrub regular delivery +DEFAULT_SCRUB_NONDIGEST = False + # Mail command processor will ignore mail command lines after designated max. DEFAULT_MAIL_COMMANDS_MAX_LINES = 25 @@ -823,6 +852,9 @@ DEFAULT_ADMIN_IMMED_NOTIFY = Yes # Is the list owner notified of subscribes/unsubscribes? DEFAULT_ADMIN_NOTIFY_MCHANGES = No +# Discard held messages after this days +DEFAULT_MAX_DAYS_TO_HOLD = 0 + # Should list members, by default, have their posts be moderated? DEFAULT_DEFAULT_MEMBER_MODERATION = No @@ -935,11 +967,27 @@ DEFAULT_FILTER_CONTENT = No # types regardless of subtype (jpeg, gif, etc.). DEFAULT_FILTER_MIME_TYPES = [] -# DEFAULT_PASS_MIME_TYPES is a list of MIME types to be passed through. Format is the same as DEFAULT_FILTER_MIME_TYPES +# DEFAULT_PASS_MIME_TYPES is a list of MIME types to be passed through. +# Format is the same as DEFAULT_FILTER_MIME_TYPES DEFAULT_PASS_MIME_TYPES = ['multipart/mixed', 'multipart/alternative', 'text/plain'] +# DEFAULT_FILTER_FILENAME_EXTENSIONS is a list of filename extensions to be +# removed. It is useful because many viruses fake their content-type as +# harmless ones while keep their extension as executable and expect to be +# executed when victims 'open' them. +DEFAULT_FILTER_FILENAME_EXTENSIONS = [ + 'exe', 'bat', 'cmd', 'com', 'pif', 'scr', 'vbs', 'cpl' + ] + +# DEFAULT_PASS_FILENAME_EXTENSIONS is a list of filename extensions to be +# passed through. Format is the same as DEFAULT_FILTER_FILENAME_EXTENSIONS. +DEFAULT_PASS_FILENAME_EXTENSIONS = [] + +# Replace multipart/alternative with its first alternative. +DEFAULT_COLLAPSE_ALTERNATIVES = Yes + # Whether text/html should be converted to text/plain after content filtering # is performed. Conversion is done according to HTML_TO_PLAIN_TEXT_COMMAND DEFAULT_CONVERT_HTML_TO_PLAINTEXT = Yes @@ -1001,6 +1049,9 @@ PLAIN_DIGEST_KEEP_HEADERS = [ # Should we do any bounced mail response at all? DEFAULT_BOUNCE_PROCESSING = Yes +# How often should the bounce qrunner process queued detected bounces? +REGISTER_BOUNCES_EVERY = minutes(15) + # Bounce processing works like this: when a bounce from a member is received, # we look up the `bounce info' for this member. If there is no bounce info, # this is the first bounce we've received from this member. In that case, we @@ -1057,8 +1108,8 @@ DEFAULT_BOUNCE_NOTIFY_OWNER_ON_REMOVAL = Yes # General time limits ##### -# How long should subscriptions requests await confirmation before being -# dropped? +# Default length of time a pending request is live before it is evicted from +# the pending database. PENDING_REQUEST_LIFE = days(3) # How long should messages which have delivery failures continue to be @@ -1066,6 +1117,9 @@ PENDING_REQUEST_LIFE = days(3) # will be dequeued and those recipients will never receive the message. DELIVERY_RETRY_PERIOD = days(5) +# How long should we wait before we retry a temporary delivery failure? +DELIVERY_RETRY_WAIT = hours(1) + ##### @@ -1102,19 +1156,6 @@ LIST_LOCK_TIMEOUT = seconds(10) # debugging. PENDINGDB_LOCK_DEBUGGING = Off -# This variable specifies how long an attempt will be made to acquire a -# pendingdb lock by the incoming qrunner process. If the lock acquisition -# times out, the message will be re-queued for later delivery. -PENDINGDB_LOCK_TIMEOUT = seconds(30) - -# The pendingdb is shared among all lists, and handles all list -# (un)subscriptions, admin approvals and otherwise held messages, so it is -# potentially locked a lot more often than single lists. Mailman deals with -# this by re-trying any attempts to alter the pendingdb that failed because of -# locking errors. This variable indicates how many attempt should be made -# before abandoning all hope. -PENDINGDB_LOCK_ATTEMPTS = 10 - ##### @@ -1158,9 +1199,10 @@ Checkbox = 12 # An "extended email list". Contents must be an email address or a ^-prefixed # regular expression. Used in the sender moderation text boxes. EmailListEx = 13 +# Extended spam filter widget +HeaderFilter = 14 -# Held message disposition actions, for use between admindb.py and -# ListAdmin.py. +# Actions DEFER = 0 APPROVE = 1 REJECT = 2 @@ -1270,31 +1312,35 @@ LC_DESCRIPTIONS = {} def add_language(code, description, charset): LC_DESCRIPTIONS[code] = (description, charset) -add_language('big5', _('Traditional Chinese'), 'big5') -add_language('cs', _('Czech'), 'iso-8859-2') -add_language('da', _('Danish'), 'iso-8859-1') -add_language('de', _('German'), 'iso-8859-1') -add_language('en', _('English (USA)'), 'us-ascii') -add_language('es', _('Spanish (Spain)'), 'iso-8859-1') -add_language('et', _('Estonian'), 'iso-8859-15') -add_language('eu', _('Euskara'), 'iso-8859-15') # Basque -add_language('fi', _('Finnish'), 'iso-8859-1') -add_language('fr', _('French'), 'iso-8859-1') -add_language('gb', _('Simplified Chinese'), 'gb2312') -add_language('hu', _('Hungarian'), 'iso-8859-2') -add_language('it', _('Italian'), 'iso-8859-1') -add_language('ja', _('Japanese'), 'euc-jp') -add_language('ko', _('Korean'), 'euc-kr') -add_language('lt', _('Lithuanian'), 'iso-8859-13') -add_language('nl', _('Dutch'), 'iso-8859-1') -add_language('no', _('Norwegian'), 'iso-8859-1') -add_language('pl', _('Polish'), 'iso-8859-2') -add_language('pt', _('Portuguese'), 'iso-8859-1') +add_language('ca', _('Catalan'), 'iso-8859-1') +add_language('cs', _('Czech'), 'iso-8859-2') +add_language('da', _('Danish'), 'iso-8859-1') +add_language('de', _('German'), 'iso-8859-1') +add_language('en', _('English (USA)'), 'us-ascii') +add_language('es', _('Spanish (Spain)'), 'iso-8859-1') +add_language('et', _('Estonian'), 'iso-8859-15') +add_language('eu', _('Euskara'), 'iso-8859-15') # Basque +add_language('fi', _('Finnish'), 'iso-8859-1') +add_language('fr', _('French'), 'iso-8859-1') +add_language('hr', _('Croatian'), 'iso-8859-2') +add_language('hu', _('Hungarian'), 'iso-8859-2') +add_language('it', _('Italian'), 'iso-8859-1') +add_language('ja', _('Japanese'), 'euc-jp') +add_language('ko', _('Korean'), 'euc-kr') +add_language('lt', _('Lithuanian'), 'iso-8859-13') +add_language('nl', _('Dutch'), 'iso-8859-1') +add_language('no', _('Norwegian'), 'iso-8859-1') +add_language('pl', _('Polish'), 'iso-8859-2') +add_language('pt', _('Portuguese'), 'iso-8859-1') add_language('pt_BR', _('Portuguese (Brazil)'), 'iso-8859-1') -add_language('ru', _('Russian'), 'koi8-r') -add_language('sr', _('Serbian'), 'utf-8') -add_language('sl', _('Slovenian'), 'iso-8859-2') -add_language('sv', _('Swedish'), 'iso-8859-1') -add_language('uk', _('Ukrainian'), 'utf-8') +add_language('ro', _('Romanian'), 'iso-8859-2') +add_language('ru', _('Russian'), 'koi8-r') +add_language('sr', _('Serbian'), 'utf-8') +add_language('sl', _('Slovenian'), 'iso-8859-2') +add_language('sv', _('Swedish'), 'iso-8859-1') +add_language('tr', _('Turkish'), 'iso-8859-9') +add_language('uk', _('Ukrainian'), 'utf-8') +add_language('zh_CN', _('Chinese (China)'), 'utf-8') +add_language('zh_TW', _('Chinese (Taiwan)'), 'utf-8') del _ diff --git a/Mailman/Deliverer.py b/Mailman/Deliverer.py index 5624473ed..7dfb5be71 100644 --- a/Mailman/Deliverer.py +++ b/Mailman/Deliverer.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2004 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -25,6 +25,7 @@ from Mailman import Errors from Mailman import Utils from Mailman import Message from Mailman import i18n +from Mailman import Pending from Mailman.Logging.Syslog import syslog _ = i18n._ @@ -111,19 +112,27 @@ your membership administrative address, %(addr)s.''')) cpuser = self.getMemberCPAddress(user) recipient = self.GetMemberAdminEmail(cpuser) subject = _('%(listfullname)s mailing list reminder') + # Get user's language and charset + lang = self.getMemberLanguage(user) + cset = Utils.GetCharSet(lang) + password = self.getMemberPassword(user) + # TK: Make unprintables to ? + # The list owner should allow users to set language options if they + # want to use non-us-ascii characters in password and send it back. + password = unicode(password, cset, 'replace').encode(cset, 'replace') # get the text from the template text = Utils.maketext( 'userpass.txt', {'user' : cpuser, 'listname' : self.real_name, 'fqdn_lname' : self.GetListEmail(), - 'password' : self.getMemberPassword(user), + 'password' : password, 'options_url': self.GetOptionsURL(user, absolute=True), 'requestaddr': requestaddr, 'owneraddr' : self.GetOwnerEmail(), - }, lang=self.getMemberLanguage(user), mlist=self) + }, lang=lang, mlist=self) msg = Message.UserNotification(recipient, adminaddr, subject, text, - self.getMemberLanguage(user)) + lang) msg['X-No-Archive'] = 'yes' msg.send(self, verp=mm_cfg.VERP_PERSONALIZED_DELIVERIES) @@ -181,3 +190,42 @@ is required."""))) msg.send(mlist) finally: i18n.set_translation(otrans) + + def sendProbe(self, member, msg): + listname = self.real_name + # Put together the substitution dictionary. + d = {'listname': listname, + 'address': member, + 'optionsurl': self.GetOptionsURL(member, absolute=True), + 'password': self.getMemberPassword(member), + 'owneraddr': self.GetOwnerEmail(), + } + text = Utils.maketext('probe.txt', d, + lang=self.getMemberLanguage(member), + mlist=self) + # Calculate the VERP'd sender address for bounce processing of the + # probe message. + token = self.pend_new(Pending.PROBE_BOUNCE, member, msg) + probedict = { + 'bounces': self.internal_name() + '-bounces', + 'token': token, + } + probeaddr = '%s@%s' % ((mm_cfg.VERP_PROBE_FORMAT % probedict), + self.host_name) + # Calculate the Subject header, in the member's preferred language + ulang = self.getMemberLanguage(member) + otrans = i18n.get_translation() + i18n.set_language(ulang) + try: + subject = _('%(listname)s mailing list probe message') + finally: + i18n.set_translation(otrans) + outer = Message.UserNotification(member, probeaddr, subject) + outer.set_type('multipart/mixed') + text = MIMEText(text, _charset=Utils.GetCharSet(ulang)) + outer.attach(text) + outer.attach(MIMEMessage(msg)) + # Turn off further VERP'ing in the final delivery step. We set + # probe_token for the OutgoingRunner to more easily handling local + # rejects of probe messages. + outer.send(self, envsender=probeaddr, verp=False, probe_token=token) diff --git a/Mailman/Errors.py b/Mailman/Errors.py index 96e34f1cb..2e80f21a5 100644 --- a/Mailman/Errors.py +++ b/Mailman/Errors.py @@ -141,6 +141,12 @@ class RejectMessage(HandlerError): def __init__(self, notice=None): if notice is None: notice = _('Your message was rejected') + if notice.endswith('\n\n'): + pass + elif notice.endswith('\n'): + notice += '\n' + else: + notice += '\n\n' self.__notice = notice def notice(self): diff --git a/Mailman/Gui/Bounce.py b/Mailman/Gui/Bounce.py index 8fb5bbc98..1dc837fc9 100644 --- a/Mailman/Gui/Bounce.py +++ b/Mailman/Gui/Bounce.py @@ -1,17 +1,17 @@ -# Copyright (C) 2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2004 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software +# along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. from Mailman import mm_cfg @@ -84,7 +84,19 @@ class Bounce(GUIBase): ('bounce_score_threshold', mm_cfg.Number, 5, 0, _("""The maximum member bounce score before the member's subscription is disabled. This value can be a floating point - number.""")), + number."""), + _("""Each subscriber is assigned a bounce score, as a floating + point number. Whenever Mailman receives a bounce from a list + member, that member's score is incremented. Hard bounces (fatal + errors) increase the score by 1, while soft bounces (temporary + errors) increase the score by 0.5. Only one bounce per day + counts against a member's score, so even if 10 bounces are + received for a member on the same day, their score will increase + by just 1. + + This variable describes the upper limit for a member's bounce + score, above which they are automatically disabled, but not + removed from the mailing list.""")), ('bounce_info_stale_after', mm_cfg.Number, 5, 0, _("""The number of days after which a member's bounce information diff --git a/Mailman/Gui/ContentFilter.py b/Mailman/Gui/ContentFilter.py index 0fadf8f3e..167f42e24 100644 --- a/Mailman/Gui/ContentFilter.py +++ b/Mailman/Gui/ContentFilter.py @@ -1,17 +1,17 @@ -# Copyright (C) 2002 by the Free Software Foundation, Inc. +# Copyright (C) 2002-2005 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software +# along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. """GUI component managing the content filtering options. @@ -57,9 +57,13 @@ class ContentFilter(GUIBase): <p>After this initial filtering, any <tt>multipart</tt> attachments that are empty are removed. If the outer message is left empty after this filtering, then the whole message is - discarded. Then, each <tt>multipart/alternative</tt> section will + discarded. + + <p> Then, each <tt>multipart/alternative</tt> section will be replaced by just the first alternative that is non-empty after - filtering. + filtering if + <a href="?VARHELP=contentfilter/collapse_alternatives" + >collapse_alternatives</a> is enabled. <p>Finally, any <tt>text/html</tt> parts that are left in the message may be converted to <tt>text/plain</tt> if @@ -74,7 +78,7 @@ class ContentFilter(GUIBase): ('filter_mime_types', mm_cfg.Text, (10, WIDTH), 0, _("""Remove message attachments that have a matching content type."""), - + _("""Use this option to remove each message attachment that matches one of these content types. Each line should contain a string naming a MIME <tt>type/subtype</tt>, @@ -100,6 +104,19 @@ class ContentFilter(GUIBase): <tt>multipart</tt> to this list, any messages with attachments will be rejected by the pass filter.""")), + ('filter_filename_extensions', mm_cfg.Text, (10, WIDTH), 0, + _("""Remove message attachments that have a matching filename + extension."""),), + + ('pass_filename_extensions', mm_cfg.Text, (10, WIDTH), 0, + _("""Remove message attachments that don't have a matching + filename extension. Leave this field blank to skip this filter + test."""),), + + ('collapse_alternatives', mm_cfg.Radio, (_('No'), _('Yes')), 0, + _("""Should Mailman collapse multipart/alternative to its + first part content?""")), + ('convert_html_to_plaintext', mm_cfg.Radio, (_('No'), _('Yes')), 0, _("""Should Mailman convert <tt>text/html</tt> parts to plain text? This conversion happens after MIME attachments have been @@ -154,10 +171,19 @@ class ContentFilter(GUIBase): doc.addError(_('Bad MIME type ignored: %(spectype)s')) else: types.append(spectype.strip().lower()) - if property == 'filter_mime_types': + if property == 'filter_mime_types': mlist.filter_mime_types = types - elif property == 'pass_mime_types': + elif property == 'pass_mime_types': mlist.pass_mime_types = types + elif property in ('filter_filename_extensions', + 'pass_filename_extensions'): + fexts = [] + for ext in [s.strip() for s in val.splitlines()]: + fexts.append(ext.lower()) + if property == 'filter_filename_extensions': + mlist.filter_filename_extensions = fexts + elif property == 'pass_filename_extensions': + mlist.pass_filename_extensions = fexts else: GUIBase._setValue(self, mlist, property, val, doc) @@ -166,4 +192,8 @@ class ContentFilter(GUIBase): return NL.join(mlist.filter_mime_types) if property == 'pass_mime_types': return NL.join(mlist.pass_mime_types) + if property == 'filter_filename_extensions': + return NL.join(mlist.filter_filename_extensions) + if property == 'pass_filename_extensions': + return NL.join(mlist.pass_filename_extensions) return None diff --git a/Mailman/Gui/GUIBase.py b/Mailman/Gui/GUIBase.py index 149038926..5f0f2c1e6 100644 --- a/Mailman/Gui/GUIBase.py +++ b/Mailman/Gui/GUIBase.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002 by the Free Software Foundation, Inc. +# Copyright (C) 2002-2004 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -108,6 +108,8 @@ class GUIBase: return val if wtype == mm_cfg.Topics: return val + if wtype == mm_cfg.HeaderFilter: + return val # Should never get here assert 0, 'Bad gui widget type: %s' % wtype @@ -120,6 +122,10 @@ class GUIBase: # Validate all the attributes for this category pass + def _escape(self, property, value): + value = value.replace('<', '<') + return value + def handleForm(self, mlist, category, subcat, cgidata, doc): for item in self.GetConfigInfo(mlist, category, subcat): # Skip descriptions and legacy non-attributes @@ -138,11 +144,12 @@ class GUIBase: elif not cgidata.has_key(property): continue elif isinstance(cgidata[property], ListType): - val = [x.value for x in cgidata[property]] + val = [self._escape(property, x.value) + for x in cgidata[property]] else: - val = cgidata[property].value + val = self._escape(property, cgidata[property].value) # Coerce the value to the expected type, raising exceptions if the - # value is invalid + # value is invalid. try: val = self._getValidValue(mlist, property, wtype, val) except ValueError: diff --git a/Mailman/Gui/General.py b/Mailman/Gui/General.py index f9fb6deca..26a55c24b 100644 --- a/Mailman/Gui/General.py +++ b/Mailman/Gui/General.py @@ -1,21 +1,22 @@ -# Copyright (C) 2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2005 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software +# along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -"""MailList mixin class managing the general options. -""" +"""MailList mixin class managing the general options.""" + +import re from Mailman import mm_cfg from Mailman import Utils @@ -143,7 +144,11 @@ class General(GUIBase): posted to the list, to distinguish mailing list messages in in mailbox summaries. Brevity is premium here, it's ok to shorten long mailing list names to something more concise, as long as it - still identifies the mailing list.""")), + still identifies the mailing list. + You can also add a sequencial number by %%d substitution + directive. eg.; [listname %%d] -> [listname 123] + (listname %%05d) -> (listname 00123) + """)), ('anonymous_list', mm_cfg.Radio, (_('No'), _('Yes')), 0, _("""Hide the sender of a message, replacing it with the list @@ -174,7 +179,7 @@ class General(GUIBase): messages, overriding the header in the original message if necessary (<em>Explicit address</em> inserts the value of <a href="?VARHELP=general/reply_to_address">reply_to_address</a>). - + <p>There are many reasons not to introduce or override the <tt>Reply-To:</tt> header. One is that some posters depend on their own <tt>Reply-To:</tt> settings to convey their valid @@ -183,7 +188,7 @@ class General(GUIBase): href="http://www.unicom.com/pw/reply-to-harmful.html">`Reply-To' Munging Considered Harmful</a> for a general discussion of this issue. See <a - href="http://www.metasystema.org/essays/reply-to-useful.mhtml">Reply-To + href="http://www.metasystema.net/essays/reply-to.mhtml">Reply-To Munging Considered Useful</a> for a dissenting opinion. <p>Some mailing lists have restricted posting privileges, with a @@ -283,7 +288,7 @@ class General(GUIBase): <li>A blank line separates paragraphs. </ul>""")), - ('send_welcome_msg', mm_cfg.Radio, (_('No'), _('Yes')), 0, + ('send_welcome_msg', mm_cfg.Radio, (_('No'), _('Yes')), 0, _('Send welcome message to newly subscribed members?'), _("""Turn this off only if you plan on subscribing people manually and don't want them to know that you did so. This option is most @@ -310,15 +315,11 @@ class General(GUIBase): ('admin_notify_mchanges', mm_cfg.Radio, (_('No'), _('Yes')), 0, _('''Should administrator get notices of subscribes and unsubscribes?''')), - + ('respond_to_post_requests', mm_cfg.Radio, (_('No'), _('Yes')), 0, - _('Send mail to poster when their posting is held for approval?'), - - _("""Approval notices are sent when mail triggers certain of the - limits <em>except</em> routine list moderation and spam filters, - for which notices are <em>not</em> sent. This option overrides - ever sending the notice.""")), + _('Send mail to poster when their posting is held for approval?') + ), _('Additional settings'), @@ -338,7 +339,7 @@ class General(GUIBase): # to tell if all were deselected! 0, _('''Default options for new members joining this list.<input type="hidden" name="new_member_options" value="ignore">'''), - + _("""When a new member is subscribed to this list, their initial set of options is taken from the this variable's setting.""")), @@ -407,6 +408,13 @@ class General(GUIBase): headers.)""")) ) + # Discard held messages after this number of days + rtn.append( + ('max_days_to_hold', mm_cfg.Number, 7, 0, + _("""Discard held messages older than this number of days. + Use 0 for no automatic discarding.""")) + ) + return rtn def _setValue(self, mlist, property, val, doc): @@ -430,6 +438,15 @@ class General(GUIBase): else: GUIBase._setValue(self, mlist, property, val, doc) + def _escape(self, property, value): + # The 'info' property allows HTML, but lets sanitize it to avoid XSS + # exploits. Everything else should be fully escaped. + if property <> 'info': + return GUIBase._escape(self, property, value) + # Sanitize <script> and </script> tags but nothing else. Not the best + # solution, but expedient. + return re.sub(r'<([/]?script.*?)>', r'<\1>', value) + def _postValidate(self, mlist, doc): if not mlist.reply_to_address.strip() and \ mlist.reply_goes_to_list == 2: diff --git a/Mailman/Gui/NonDigest.py b/Mailman/Gui/NonDigest.py index 05ed7361b..ca125a037 100644 --- a/Mailman/Gui/NonDigest.py +++ b/Mailman/Gui/NonDigest.py @@ -134,6 +134,15 @@ and footers: _('''Text appended to the bottom of every immediately-delivery message. ''') + headfoot + extra), ]) + + info.extend([ + ('scrub_nondigest', mm_cfg.Toggle, (_('No'), _('Yes')), 0, + _('Scrub attachments of regular delivery message?'), + _('''When you scrub attachments, they are stored in archive + area and links are made in the message so that the member can + access via web browser. If you want the attachments totally + disappear, you can use content filter options.''')), + ]) return info def _setValue(self, mlist, property, val, doc): diff --git a/Mailman/Gui/Privacy.py b/Mailman/Gui/Privacy.py index 85d5db58c..1ae2699a7 100644 --- a/Mailman/Gui/Privacy.py +++ b/Mailman/Gui/Privacy.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2003 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -17,10 +17,19 @@ """MailList mixin class managing the privacy options. """ +import re + from Mailman import mm_cfg +from Mailman import Utils from Mailman.i18n import _ from Mailman.Gui.GUIBase import GUIBase +try: + True, False +except NameError: + True = 1 + False = 0 + class Privacy(GUIBase): @@ -297,6 +306,12 @@ class Privacy(GUIBase): _("""Should messages from non-members, which are automatically discarded, be forwarded to the list moderator?""")), + ('nonmember_rejection_notice', mm_cfg.Text, (10, WIDTH), 1, + _("""Text to include in any rejection notice to be sent to + non-members who post to this list. This notice can include + the list's owner address by %%(listowner)s and replaces the + internally crafted default message.""")), + ] recip_rtn = [ @@ -361,7 +376,29 @@ class Privacy(GUIBase): your list members end up receiving. """), - _("Anti-Spam filters"), + _('Header filters'), + + ('header_filter_rules', mm_cfg.HeaderFilter, 0, 0, + _('Filter rules to match against the headers of a message.'), + + _("""Each header filter rule has two parts, a list of regular + expressions, one per line, and an action to take. Mailman + matches the message's headers against every regular expression in + the rule and if any match, the message is rejected, held, or + discarded based on the action you specify. Use <em>Defer</em> to + temporarily disable a rule. + + You can have more than one filter rule for your list. In that + case, each rule is matched in turn, with processing stopped after + the first match. + + Note that headers are collected from all the attachments + (except for the mailman administrivia message) and + matched against the regular expressions. With this feature, + you can effectively sort out messages with dangerous file + types or file name extensions.""")), + + _('Legacy anti-spam filters'), ('bounce_matching_headers', mm_cfg.Text, (6, WIDTH), 0, _('Hold posts with header value matching a specified regexp.'), @@ -390,9 +427,104 @@ class Privacy(GUIBase): return subscribing_rtn def _setValue(self, mlist, property, val, doc): + # Ignore any hdrfilter_* form variables + if property.startswith('hdrfilter_'): + return # For subscribe_policy when ALLOW_OPEN_SUBSCRIBE is true, we need to # add one to the value because the page didn't present an open list as # an option. if property == 'subscribe_policy' and not mm_cfg.ALLOW_OPEN_SUBSCRIBE: val += 1 setattr(mlist, property, val) + + # We need to handle the header_filter_rules widgets specially, but + # everything else can be done by the base class's handleForm() method. + # However, to do this we need an awful hack. _setValue() and + # _getValidValue() will essentially ignore any hdrfilter_* form variables. + # TK: we should call this function only in subcat == 'spam' + def _handleForm(self, mlist, category, subcat, cgidata, doc): + # First deal with + rules = [] + # We start i at 1 and keep going until we no longer find items keyed + # with the marked tags. + i = 1 + downi = None + while True: + deltag = 'hdrfilter_delete_%02d' % i + reboxtag = 'hdrfilter_rebox_%02d' % i + actiontag = 'hdrfilter_action_%02d' % i + wheretag = 'hdrfilter_where_%02d' % i + addtag = 'hdrfilter_add_%02d' % i + newtag = 'hdrfilter_new_%02d' % i + uptag = 'hdrfilter_up_%02d' % i + downtag = 'hdrfilter_down_%02d' % i + i += 1 + # Was this a delete? If so, we can just ignore this entry + if cgidata.has_key(deltag): + continue + # Get the data for the current box + pattern = cgidata.getvalue(reboxtag) + try: + action = int(cgidata.getvalue(actiontag)) + # We'll get a TypeError when the actiontag is missing and the + # .getvalue() call returns None. + except (ValueError, TypeError): + action = mm_cfg.DEFER + if pattern is None: + # We came to the end of the boxes + break + if cgidata.has_key(newtag) and not pattern: + # This new entry is incomplete. + if i == 2: + # OK it is the first. + continue + doc.addError(_("""Header filter rules require a pattern. + Incomplete filter rules will be ignored.""")) + continue + # Make sure the pattern was a legal regular expression + try: + re.compile(pattern) + except (re.error, TypeError): + safepattern = Utils.websafe(pattern) + doc.addError(_("""The header filter rule pattern + '%(safepattern)s' is not a legal regular expression. This + rule will be ignored.""")) + continue + # Was this an add item? + if cgidata.has_key(addtag): + # Where should the new one be added? + where = cgidata.getvalue(wheretag) + if where == 'before': + # Add a new empty rule box before the current one + rules.append(('', mm_cfg.DEFER, True)) + rules.append((pattern, action, False)) + # Default is to add it after... + else: + rules.append((pattern, action, False)) + rules.append(('', mm_cfg.DEFER, True)) + # Was this an up movement? + elif cgidata.has_key(uptag): + # As long as this one isn't the first rule, move it up + if rules: + rules.insert(-1, (pattern, action, False)) + else: + rules.append((pattern, action, False)) + # Was this the down movement? + elif cgidata.has_key(downtag): + downi = i - 2 + rules.append((pattern, action, False)) + # Otherwise, just retain this one in the list + else: + rules.append((pattern, action, False)) + # Move any down button filter rule + if downi is not None: + rule = rules[downi] + del rules[downi] + rules.insert(downi+1, rule) + mlist.header_filter_rules = rules + + def handleForm(self, mlist, category, subcat, cgidata, doc): + if subcat == 'spam': + self._handleForm(mlist, category, subcat, cgidata, doc) + # Everything else is dealt with by the base handler + GUIBase.handleForm(self, mlist, category, subcat, cgidata, doc) diff --git a/Mailman/Gui/Topics.py b/Mailman/Gui/Topics.py index 494c76517..e25ef65d3 100644 --- a/Mailman/Gui/Topics.py +++ b/Mailman/Gui/Topics.py @@ -1,26 +1,33 @@ -# Copyright (C) 2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2003 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software +# along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. import re from Mailman import mm_cfg +from Mailman import Utils from Mailman.i18n import _ from Mailman.Logging.Syslog import syslog from Mailman.Gui.GUIBase import GUIBase +try: + True, False +except NameError: + True = 1 + False = 0 + class Topics(GUIBase): @@ -88,7 +95,7 @@ class Topics(GUIBase): # We start i at 1 and keep going until we no longer find items keyed # with the marked tags. i = 1 - while 1: + while True: deltag = 'topic_delete_%02d' % i boxtag = 'topic_box_%02d' % i reboxtag = 'topic_rebox_%02d' % i @@ -96,51 +103,46 @@ class Topics(GUIBase): wheretag = 'topic_where_%02d' % i addtag = 'topic_add_%02d' % i newtag = 'topic_new_%02d' % i - i += 1 # Was this a delete? If so, we can just ignore this entry if cgidata.has_key(deltag): continue - # Get the data for the current box name = cgidata.getvalue(boxtag) pattern = cgidata.getvalue(reboxtag) desc = cgidata.getvalue(desctag) - if name is None: # We came to the end of the boxes break - if cgidata.has_key(newtag) and (not name or not pattern): # This new entry is incomplete. doc.addError(_("""Topic specifications require both a name and a pattern. Incomplete topics will be ignored.""")) continue - # Make sure the pattern was a legal regular expression + name = Utils.websafe(name) try: re.compile(pattern) except (re.error, TypeError): - doc.addError(_("""The topic pattern `%(pattern)s' is not a + safepattern = Utils.websafe(pattern) + doc.addError(_("""The topic pattern '%(safepattern)s' is not a legal regular expression. It will be discarded.""")) continue - # Was this an add item? if cgidata.has_key(addtag): # Where should the new one be added? where = cgidata.getvalue(wheretag) if where == 'before': # Add a new empty topics box before the current one - topics.append(('', '', '', 1)) - topics.append((name, pattern, desc, 0)) + topics.append(('', '', '', True)) + topics.append((name, pattern, desc, False)) # Default is to add it after... else: - topics.append((name, pattern, desc, 0)) - topics.append(('', '', '', 1)) + topics.append((name, pattern, desc, False)) + topics.append(('', '', '', True)) # Otherwise, just retain this one in the list else: - topics.append((name, pattern, desc, 0)) - + topics.append((name, pattern, desc, False)) # Add these topics to the mailing list object, and deal with other # options. mlist.topics = topics diff --git a/Mailman/Handlers/Acknowledge.py b/Mailman/Handlers/Acknowledge.py index d27ba3739..dc8175773 100644 --- a/Mailman/Handlers/Acknowledge.py +++ b/Mailman/Handlers/Acknowledge.py @@ -48,7 +48,7 @@ def process(mlist, msg, msgdata): realname = mlist.real_name text = Utils.maketext( 'postack.txt', - {'subject' : origsubj, + {'subject' : Utils.oneline(origsubj, Utils.GetCharSet(lang)), 'listname' : realname, 'listinfo_url': mlist.GetScriptURL('listinfo', absolute=1), 'optionsurl' : mlist.GetOptionsURL(sender, absolute=1), diff --git a/Mailman/Handlers/Approve.py b/Mailman/Handlers/Approve.py index c3db183b0..633b9169d 100644 --- a/Mailman/Handlers/Approve.py +++ b/Mailman/Handlers/Approve.py @@ -1,17 +1,17 @@ -# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2005 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software +# along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. """Determine whether the message is approved for delivery. @@ -20,7 +20,6 @@ This module only tests for definitive approvals. IOW, this module only determines whether the message is definitively approved or definitively denied. Situations that could hold a message for approval or confirmation are not tested by this module. - """ from email.Iterators import typed_subpart_iterator @@ -52,7 +51,9 @@ def process(mlist, msg, msgdata): part = None for part in typed_subpart_iterator(msg, 'text', 'plain'): break - if part is not None: + # XXX I'm not entirely sure why, but it is possible for the payload of + # the part to be None, and you can't splitlines() on None. + if part is not None and part.get_payload() is not None: lines = part.get_payload().splitlines() line = '' for lineno, line in zip(range(len(lines)), lines): diff --git a/Mailman/Handlers/AvoidDuplicates.py b/Mailman/Handlers/AvoidDuplicates.py index 09677206e..fdcc49ca1 100644 --- a/Mailman/Handlers/AvoidDuplicates.py +++ b/Mailman/Handlers/AvoidDuplicates.py @@ -92,4 +92,5 @@ def process(mlist, msg, msgdata): msgdata['recips'] = newrecips # RFC 2822 specifies zero or one CC header del msg['cc'] - msg['Cc'] = COMMASPACE.join([formataddr(i) for i in ccaddrs.values()]) + if ccaddrs: + msg['Cc'] = COMMASPACE.join([formataddr(i) for i in ccaddrs.values()]) diff --git a/Mailman/Handlers/Cleanse.py b/Mailman/Handlers/Cleanse.py index 059652568..65c5df69e 100644 --- a/Mailman/Handlers/Cleanse.py +++ b/Mailman/Handlers/Cleanse.py @@ -16,6 +16,11 @@ """Cleanse certain headers from all messages.""" +from email.Utils import formataddr + +from Mailman.Logging.Syslog import syslog +from Mailman.Handlers.CookHeaders import uheader + def process(mlist, msg, msgdata): # Always remove this header from any outgoing messages. Be sure to do @@ -26,12 +31,15 @@ def process(mlist, msg, msgdata): del msg['urgent'] # We remove other headers from anonymous lists if mlist.anonymous_list: + syslog('post', 'post to %s from %s anonymized', + mlist.internal_name(), msg.get('from')) del msg['from'] del msg['reply-to'] del msg['sender'] # Hotmail sets this one del msg['x-originating-email'] - msg['From'] = mlist.GetListEmail() + i18ndesc = str(uheader(mlist, mlist.description, 'From')) + msg['From'] = formataddr((i18ndesc, mlist.GetListEmail())) msg['Reply-To'] = mlist.GetListEmail() # Some headers can be used to fish for membership del msg['return-receipt-to'] diff --git a/Mailman/Handlers/CookHeaders.py b/Mailman/Handlers/CookHeaders.py index 43bb2ea38..4ebc4e577 100644 --- a/Mailman/Handlers/CookHeaders.py +++ b/Mailman/Handlers/CookHeaders.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2005 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -14,16 +14,16 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -"""Cook a message's Subject header. -""" +"""Cook a message's Subject header.""" from __future__ import nested_scopes import re from types import UnicodeType from email.Charset import Charset -from email.Header import Header, decode_header +from email.Header import Header, decode_header, make_header from email.Utils import parseaddr, formataddr, getaddresses +from email.Errors import HeaderParseError from Mailman import mm_cfg from Mailman import Utils @@ -34,28 +34,34 @@ CONTINUATION = ',\n\t' COMMASPACE = ', ' MAXLINELEN = 78 +# True/False +try: + True, False +except NameError: + True = 1 + False = 0 + def _isunicode(s): return isinstance(s, UnicodeType) -def uheader(mlist, s, header_name=None, continuation_ws='\t'): - # Get the charset to encode the string in. If this is us-ascii, we'll use - # iso-8859-1 instead, just to get a little extra coverage, and because the - # Header class tries us-ascii first anyway. +nonascii = re.compile('[^\s!-~]') + +def uheader(mlist, s, header_name=None, continuation_ws='\t', maxlinelen=None): + # Get the charset to encode the string in. Then search if there is any + # non-ascii character is in the string. If there is and the charset is + # us-ascii then we use iso-8859-1 instead. If the string is ascii only + # we use 'us-ascii' if another charset is specified. charset = Utils.GetCharSet(mlist.preferred_language) - if charset == 'us-ascii': - charset = 'iso-8859-1' - charset = Charset(charset) - # Convert the string to unicode so Header will do the 3-charset encoding. - # If s is a byte string and there are funky characters in it that don't - # match the charset, we might as well replace them now. - if not _isunicode(s): - codec = charset.input_codec or 'ascii' - s = unicode(s, codec, 'replace') - # We purposefully leave no space b/w prefix and subject! - return Header(s, charset, header_name=header_name, - continuation_ws=continuation_ws) + if nonascii.search(s): + # use list charset but ... + if charset == 'us-ascii': + charset = 'iso-8859-1' + else: + # there is no nonascii so ... + charset = 'us-ascii' + return Header(s, charset, maxlinelen, header_name, continuation_ws) @@ -72,7 +78,12 @@ def process(mlist, msg, msgdata): # VirginRunner sets _fasttrack for internally crafted messages. fasttrack = msgdata.get('_fasttrack') if not msgdata.get('isdigest') and not fasttrack: - prefix_subject(mlist, msg, msgdata) + try: + prefix_subject(mlist, msg, msgdata) + except (UnicodeError, ValueError): + # TK: Sometimes subject header is not MIME encoded for 8bit + # simply abort prefixing. + pass # Mark message so we know we've been here, but leave any existing # X-BeenThere's intact. msg['X-BeenThere'] = mlist.GetListEmail() @@ -128,7 +139,7 @@ def process(mlist, msg, msgdata): # because some folks think that some MUAs make it easier to delete # addresses from the right than from the left. if mlist.reply_goes_to_list == 1: - i18ndesc = uheader(mlist, mlist.description) + i18ndesc = uheader(mlist, mlist.description, 'Reply-To') add((str(i18ndesc), mlist.GetListEmail())) del msg['reply-to'] # Don't put Reply-To: back if there's nothing to add! @@ -144,14 +155,17 @@ def process(mlist, msg, msgdata): # was munged into the Reply-To header, but if not, we'll add it to a # Cc header. BAW: should we force it into a Reply-To header in the # above code? - if mlist.personalize == 2 and mlist.reply_goes_to_list <> 1: + # Also skip Cc if this is an anonymous list as list posting address + # is already in From and Reply-To in this case. + if mlist.personalize == 2 and mlist.reply_goes_to_list <> 1 \ + and not mlist.anonymous_list: # Watch out for existing Cc headers, merge, and remove dups. Note # that RFC 2822 says only zero or one Cc header is allowed. new = [] d = {} for pair in getaddresses(msg.get_all('cc', [])): add(pair) - i18ndesc = uheader(mlist, mlist.description) + i18ndesc = uheader(mlist, mlist.description, 'Cc') add((str(i18ndesc), mlist.GetListEmail())) del msg['Cc'] msg['Cc'] = COMMASPACE.join([formataddr(pair) for pair in new]) @@ -164,15 +178,17 @@ def process(mlist, msg, msgdata): # headers by default, pissing off their users. Too bad. Fix the MUAs. if msgdata.get('_nolist') or not mlist.include_rfc2369_headers: return - # Pre-calculate - listid = '<%s.%s>' % (mlist.internal_name(), mlist.host_name) + # This will act like an email address for purposes of formataddr() + listid = '%s.%s' % (mlist.internal_name(), mlist.host_name) + cset = Utils.GetCharSet(mlist.preferred_language) if mlist.description: - # Make sure description is properly i18n'd - listid_h = uheader(mlist, mlist.description, 'List-Id') - listid_h.append(listid, 'us-ascii') + # Don't wrap the header since here we just want to get it properly RFC + # 2047 encoded. + i18ndesc = uheader(mlist, mlist.description, 'List-Id', maxlinelen=998) + listid_h = formataddr((str(i18ndesc), listid)) else: - # For wrapping - listid_h = Header(listid, 'us-ascii', header_name='List-Id') + # without desc we need to ensure the MUST brackets + listid_h = '<%s>' % listid # We always add a List-ID: header. del msg['list-id'] msg['List-Id'] = listid_h @@ -218,7 +234,9 @@ def prefix_subject(mlist, msg, msgdata): # Add the subject prefix unless the message is a digest or is being fast # tracked (e.g. internally crafted, delivered to a single user such as the # list admin). - prefix = mlist.subject_prefix + prefix = mlist.subject_prefix.strip() + if not prefix: + return subject = msg.get('subject', '') # Try to figure out what the continuation_ws is for the header if isinstance(subject, Header): @@ -229,35 +247,116 @@ def prefix_subject(mlist, msg, msgdata): if len(lines) > 1 and lines[1] and lines[1][0] in ' \t': ws = lines[1][0] msgdata['origsubj'] = subject - if not subject: + # The subject may be multilingual but we take the first charset + # as major one and try to decode. If it is decodable, returned + # subject is in one line and cset is properly set. If fail, + # subject is mime-encoded and cset is set as us-ascii. See detail + # for ch_oneline() (CookHeaders one line function). + subject, cset = ch_oneline(subject) + # If the subject_prefix contains '%d', it is replaced with the + # mailing list sequential number. Sequential number format allows + # '%d' or '%05d' like pattern. + prefix_pattern = re.escape(prefix) + # unescape '%' :-< + prefix_pattern = '%'.join(prefix_pattern.split(r'\%')) + p = re.compile('%\d*d') + if p.search(prefix, 1): + # prefix have number, so we should search prefix w/number in subject. + # Also, force new style. + prefix_pattern = p.sub(r'\s*\d+\s*', prefix_pattern) + old_style = False + else: + old_style = mm_cfg.OLD_STYLE_PREFIXING + subject = re.sub(prefix_pattern, '', subject) + rematch = re.match('((RE|AW|SV)(\[\d+\])?:\s*)+', subject, re.I) + if rematch: + subject = subject[rematch.end():] + recolon = 'Re:' + else: + recolon = '' + # At this point, subject may become null if someone post mail with + # subject: [subject prefix] + if subject.strip() == '': subject = _('(no subject)') - # The header may be multilingual; decode it from base64/quopri and search - # each chunk for the prefix. BAW: Note that if the prefix contains spaces - # and each word of the prefix is encoded in a different chunk in the - # header, we won't find it. I think in practice that's unlikely though. - headerbits = decode_header(subject) - if prefix and subject: - pattern = re.escape(prefix.strip()) - for decodedsubj, charset in headerbits: - if re.search(pattern, decodedsubj, re.IGNORECASE): - # The subject's already got the prefix, so don't change it - return - del msg['subject'] + cset = Utils.GetCharSet(mlist.preferred_language) + # and substitute %d in prefix with post_id + try: + prefix = prefix % mlist.post_id + except TypeError: + pass + # If charset is 'us-ascii', try to concatnate as string because there + # is some weirdness in Header module (TK) + if cset == 'us-ascii': + try: + if old_style: + h = ' '.join([recolon, prefix, subject]) + else: + h = ' '.join([prefix, recolon, subject]) + if type(h) == UnicodeType: + h = h.encode('us-ascii') + else: + h = unicode(h, 'us-ascii').encode('us-ascii') + h = uheader(mlist, h, 'Subject', continuation_ws=ws) + del msg['subject'] + msg['Subject'] = h + ss = ' '.join([recolon, subject]) + if _isunicode(ss): + ss = ss.encode('us-ascii') + else: + ss = unicode(ss, 'us-ascii').encode('us-ascii') + ss = uheader(mlist, ss, 'Subject', continuation_ws=ws) + msgdata['stripped_subject'] = ss + return + except UnicodeError: + pass # Get the header as a Header instance, with proper unicode conversion - h = uheader(mlist, prefix, 'Subject', continuation_ws=ws) - for s, c in headerbits: - # Once again, convert the string to unicode. - if c is None: - c = Charset('iso-8859-1') - if not isinstance(c, Charset): - c = Charset(c) - if not _isunicode(s): - codec = c.input_codec or 'ascii' - try: - s = unicode(s, codec, 'replace') - except LookupError: - # Unknown codec, is this default reasonable? - s = unicode(s, Utils.GetCharSet(mlist.preferred_language), - 'replace') - h.append(s, c) + if old_style: + h = uheader(mlist, recolon, 'Subject', continuation_ws=ws) + h.append(prefix) + else: + h = uheader(mlist, prefix, 'Subject', continuation_ws=ws) + h.append(recolon) + # in seq version, subject header is already concatnated + if not _isunicode(subject): + try: + subject = unicode(subject, cset, 'replace') + except (LookupError, TypeError): + # unknown codec + cset = Utils.GetCharSet(mlist.preferred_language) + subject = unicode(subject, cset, 'replace') + subject = subject.encode(cset,'replace') + h.append(subject, cset) + del msg['subject'] msg['Subject'] = h + ss = uheader(mlist, recolon, 'Subject', continuation_ws=ws) + ss.append(subject, cset) + msgdata['stripped_subject'] = ss + + +def ch_oneline(s): + # Decode header string in one line and convert into single charset + # copied and modified from ToDigest.py and Utils.py + # return (string, cset) tuple as check for failure + try: + d = decode_header(s) + # at this point, we should rstrip() every string because some + # MUA deliberately add trailing spaces when composing return + # message. + i = 0 + for (s,c) in d: + s = s.rstrip() + d[i] = (s,c) + i += 1 + cset = 'us-ascii' + for x in d: + # search for no-None charset + if x[1]: + cset = x[1] + break + h = make_header(d) + ustr = h.__unicode__() + oneline = u''.join(ustr.splitlines()) + return oneline.encode(cset, 'replace'), cset + except (LookupError, UnicodeError, ValueError, HeaderParseError): + # possibly charset problem. return with undecoded string in one line. + return ''.join(s.splitlines()), 'us-ascii' diff --git a/Mailman/Handlers/Decorate.py b/Mailman/Handlers/Decorate.py index b515edb84..afb0a1c90 100644 --- a/Mailman/Handlers/Decorate.py +++ b/Mailman/Handlers/Decorate.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2005 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -14,8 +14,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -"""Decorate a message by sticking the header and footer around it. -""" +"""Decorate a message by sticking the header and footer around it.""" from types import ListType from email.MIMEText import MIMEText @@ -28,6 +27,12 @@ from Mailman.i18n import _ from Mailman.SafeDict import SafeDict from Mailman.Logging.Syslog import syslog +try: + True, False +except: + True = 1 + False = 0 + def process(mlist, msg, msgdata): @@ -47,7 +52,12 @@ def process(mlist, msg, msgdata): # BAW: Hmm, should we allow this? d['user_password'] = mlist.getMemberPassword(member) d['user_language'] = mlist.getMemberLanguage(member) - d['user_name'] = mlist.getMemberName(member) or _('not available') + username = mlist.getMemberName(member) or None + try: + username = username.encode(Utils.GetCharSet(d['user_language'])) + except (AttributeError, UnicodeError): + username = member + d['user_name'] = username d['user_optionsurl'] = mlist.GetOptionsURL(member) except Errors.NotAMemberError: pass @@ -71,24 +81,45 @@ def process(mlist, msg, msgdata): # safely add the header/footer to a plain text message since all # charsets Mailman supports are strict supersets of us-ascii -- # no, UTF-16 emails are not supported yet. - mcset = msg.get_param('charset', 'us-ascii').lower() + # + # TK: Message with 'charset=' cause trouble. So, instead of + # mgs.get_content_charset('us-ascii') ... + mcset = msg.get_content_charset() or 'us-ascii' lcset = Utils.GetCharSet(mlist.preferred_language) - msgtype = msg.get_type('text/plain') + msgtype = msg.get_content_type() # BAW: If the charsets don't match, should we add the header and footer by # MIME multipart chroming the message? - wrap = 1 - if not msg.is_multipart() and msgtype == 'text/plain' and \ - msg.get('content-transfer-encoding', '').lower() <> 'base64' and \ - (lcset == 'us-ascii' or mcset == lcset): - oldpayload = msg.get_payload() - frontsep = endsep = '' - if header and not header.endswith('\n'): - frontsep = '\n' - if footer and not oldpayload.endswith('\n'): - endsep = '\n' - payload = header + frontsep + oldpayload + endsep + footer - msg.set_payload(payload) - wrap = 0 + wrap = True + if not msg.is_multipart() and msgtype == 'text/plain': + # TK: Try to keep the message plain by converting the header/ + # footer/oldpayload into unicode and encode with mcset/lcset. + # Try to decode qp/base64 also. + uheader = unicode(header, lcset) + ufooter = unicode(footer, lcset) + try: + oldpayload = unicode(msg.get_payload(decode=1), mcset) + frontsep = endsep = u'' + if header and not header.endswith('\n'): + frontsep = u'\n' + if footer and not oldpayload.endswith('\n'): + endsep = u'\n' + payload = uheader + frontsep + oldpayload + endsep + ufooter + try: + # first, try encode with list charset + payload = payload.encode(lcset) + newcset = lcset + except UnicodeError: + if lcset != mcset: + # if fail, encode with message charset (if different) + payload = payload.encode(mcset) + newcset = mcset + # if this fails, fallback to outer try and wrap=true + del msg['content-transfer-encoding'] + del msg['content-type'] + msg.set_payload(payload, newcset) + wrap = False + except (LookupError, UnicodeError): + pass elif msg.get_type() == 'multipart/mixed': # The next easiest thing to do is just prepend the header and append # the footer as additional subparts @@ -104,7 +135,7 @@ def process(mlist, msg, msgdata): mimehdr['Content-Disposition'] = 'inline' payload.insert(0, mimehdr) msg.set_payload(payload) - wrap = 0 + wrap = False # If we couldn't add the header or footer in a less intrusive way, we can # at least do it by MIME encapsulation. We want to keep as much of the # outer chrome as possible. diff --git a/Mailman/Handlers/Hold.py b/Mailman/Handlers/Hold.py index f0c02fbc9..ffc0839bc 100644 --- a/Mailman/Handlers/Hold.py +++ b/Mailman/Handlers/Hold.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2004 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -197,6 +197,12 @@ def hold_for_approval(mlist, msg, msgdata, exc): exc = exc() listname = mlist.real_name sender = msgdata.get('sender', msg.get_sender()) + usersubject = msg.get('subject') + charset = Utils.GetCharSet(mlist.preferred_language) + if usersubject: + usersubject = Utils.oneline(usersubject, charset) + else: + usersubject = _('(no subject)') message_id = msg.get('message-id', 'n/a') owneraddr = mlist.GetOwnerEmail() adminaddr = mlist.GetBouncesEmail() @@ -212,7 +218,7 @@ def hold_for_approval(mlist, msg, msgdata, exc): 'hostname' : mlist.host_name, 'reason' : _(reason), 'sender' : sender, - 'subject' : msg.get('subject', _('(no subject)')), + 'subject' : usersubject, 'admindb_url': mlist.GetScriptURL('admindb', absolute=1), } # We may want to send a notification to the original sender too @@ -224,9 +230,9 @@ def hold_for_approval(mlist, msg, msgdata, exc): # # This message should appear to come from <list>-admin so as to handle any # bounce processing that might be needed. - cookie = Pending.new(Pending.HELD_MESSAGE, id) + cookie = mlist.pend_new(Pending.HELD_MESSAGE, id) if not fromusenet and ackp(msg) and mlist.respond_to_post_requests and \ - mlist.autorespondToSender(sender): + mlist.autorespondToSender(sender, mlist.getMemberLanguage(sender)): # Get a confirmation cookie d['confirmurl'] = '%s/%s' % (mlist.GetScriptURL('confirm', absolute=1), cookie) @@ -247,7 +253,6 @@ def hold_for_approval(mlist, msg, msgdata, exc): lang = mlist.preferred_language charset = Utils.GetCharSet(lang) # We need to regenerate or re-translate a few values in d - usersubject = msg.get('subject', _('(no subject)')) d['reason'] = _(reason) d['subject'] = usersubject # craft the admin notification message and deliver it diff --git a/Mailman/Handlers/MimeDel.py b/Mailman/Handlers/MimeDel.py index 9c21f11d8..33cfe1420 100644 --- a/Mailman/Handlers/MimeDel.py +++ b/Mailman/Handlers/MimeDel.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002 by the Free Software Foundation, Inc. +# Copyright (C) 2002-2005 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -26,6 +26,7 @@ contents. import os import errno import tempfile +from os.path import splitext from email.Iterators import typed_subpart_iterator @@ -36,6 +37,7 @@ from Mailman.Queue.sbcache import get_switchboard from Mailman.Logging.Syslog import syslog from Mailman.Version import VERSION from Mailman.i18n import _ +from Mailman.Utils import oneline @@ -59,12 +61,23 @@ def process(mlist, msg, msgdata): if passtypes and not (ctype in passtypes or mtype in passtypes): dispose(mlist, msg, msgdata, _("The message's content type was not explicitly allowed")) + # Filter by file extensions + filterexts = mlist.filter_filename_extensions + passexts = mlist.pass_filename_extensions + fext = get_file_ext(msg) + if fext: + if fext in filterexts: + dispose(mlist, msg, msgdata, + _("The message's file extension was explicitly disallowed")) + if passexts and not (fext in passexts): + dispose(mlist, msg, msgdata, + _("The message's file extension was not explicitly allowed")) numparts = len([subpart for subpart in msg.walk()]) # If the message is a multipart, filter out matching subparts if msg.is_multipart(): # Recursively filter out any subparts that match the filter list prelen = len(msg.get_payload()) - filter_parts(msg, filtertypes, passtypes) + filter_parts(msg, filtertypes, passtypes, filterexts, passexts) # If the outer message is now an empty multipart (and it wasn't # before!) then, again it gets discarded. postlen = len(msg.get_payload()) @@ -77,10 +90,12 @@ def process(mlist, msg, msgdata): # headers. For now we'll move the subpart's payload into the outer part, # and then copy over its Content-Type: and Content-Transfer-Encoding: # headers (any others?). - collapse_multipart_alternatives(msg) - if ctype == 'multipart/alternative': - firstalt = msg.get_payload(0) - reset_payload(msg, firstalt) + # TK: Make this configurable from Gui/ContentFilter.py. + if mlist.collapse_alternatives: + collapse_multipart_alternatives(msg) + if ctype == 'multipart/alternative': + firstalt = msg.get_payload(0) + reset_payload(msg, firstalt) # If we removed some parts, make note of this changedp = 0 if numparts <> len([subpart for subpart in msg.walk()]): @@ -121,7 +136,7 @@ def reset_payload(msg, subpart): -def filter_parts(msg, filtertypes, passtypes): +def filter_parts(msg, filtertypes, passtypes, filterexts, passexts): # Look at all the message's subparts, and recursively filter if not msg.is_multipart(): return 1 @@ -129,7 +144,8 @@ def filter_parts(msg, filtertypes, passtypes): prelen = len(payload) newpayload = [] for subpart in payload: - keep = filter_parts(subpart, filtertypes, passtypes) + keep = filter_parts(subpart, filtertypes, passtypes, + filterexts, passexts) if not keep: continue ctype = subpart.get_content_type() @@ -140,6 +156,13 @@ def filter_parts(msg, filtertypes, passtypes): if passtypes and not (ctype in passtypes or mtype in passtypes): # Throw this subpart away continue + # check file extension + fext = get_file_ext(subpart) + if fext: + if fext in filterexts: + continue + if passexts and not (fext in passexts): + continue newpayload.append(subpart) # Check to see if we discarded all the subparts postlen = len(newpayload) @@ -218,3 +241,18 @@ are receiving the only remaining copy of the discarded message. badq.enqueue(msg, msgdata) # Most cases also discard the message raise Errors.DiscardMessage + +def get_file_ext(m): + """ + Get filename extension. Caution: some virus don't put filename + in 'Content-Disposition' header. +""" + fext = '' + filename = m.get_filename('') or m.get_param('name', '') + if filename: + fext = splitext(oneline(filename,'utf-8'))[1] + if len(fext) > 1: + fext = fext[1:] + else: + fext = '' + return fext diff --git a/Mailman/Handlers/Moderate.py b/Mailman/Handlers/Moderate.py index eec4b72c7..97d998b24 100644 --- a/Mailman/Handlers/Moderate.py +++ b/Mailman/Handlers/Moderate.py @@ -135,7 +135,11 @@ def matches_p(sender, nonmembers): def do_reject(mlist): listowner = mlist.GetOwnerEmail() - raise Errors.RejectMessage, Utils.wrap(_("""\ + if mlist.nonmember_rejection_notice: + raise Errors.RejectMessage, \ + Utils.wrap(_(mlist.nonmember_rejection_notice)) + else: + raise Errors.RejectMessage, Utils.wrap(_("""\ You are not allowed to post to this mailing list, and your message has been automatically rejected. If you think that your messages are being rejected in error, contact the mailing list owner at %(listowner)s.""")) diff --git a/Mailman/Handlers/SMTPDirect.py b/Mailman/Handlers/SMTPDirect.py index 854511e21..44209ebb0 100644 --- a/Mailman/Handlers/SMTPDirect.py +++ b/Mailman/Handlers/SMTPDirect.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2004 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -45,6 +45,12 @@ from email.Charset import Charset DOT = '.' +try: + True, False +except NameError: + True = 1 + False = 0 + # Manage a connection to the SMTP server @@ -123,7 +129,7 @@ def process(mlist, msg, msgdata): # Be sure never to decorate the message more than once! if not msgdata.get('decorated'): Decorate.process(mlist, msg, msgdata) - msgdata['decorated'] = 1 + msgdata['decorated'] = True deliveryfunc = bulkdeliver refused = {} t0 = time.time() @@ -337,6 +343,11 @@ def bulkdeliver(mlist, msg, msgdata, envsender, failures, conn): # Do some final cleanup of the message header. Start by blowing away # any the Sender: and Errors-To: headers so remote MTAs won't be # tempted to delivery bounces there instead of our envelope sender + # + # BAW An interpretation of RFCs 2822 and 2076 could argue for not touching + # the Sender header at all. Brad Knowles points out that MTAs tend to + # wipe existing Return-Path headers, and old MTAs may still honor + # Errors-To while new ones will at worst ignore the header. del msg['sender'] del msg['errors-to'] msg['Sender'] = envsender @@ -345,20 +356,33 @@ def bulkdeliver(mlist, msg, msgdata, envsender, failures, conn): msgtext = msg.as_string() refused = {} recips = msgdata['recips'] + msgid = msg['message-id'] try: # Send the message refused = conn.sendmail(envsender, recips, msgtext) except smtplib.SMTPRecipientsRefused, e: + syslog('smtp-failure', 'All recipients refused: %s, msgid: %s', + e, msgid) refused = e.recipients - # MTA not responding, or other socket problems, or any other kind of - # SMTPException. In that case, nothing got delivered - except (socket.error, smtplib.SMTPException, IOError), e: - # BAW: should this be configurable? - syslog('smtp', 'All recipients refused: %s', e) - # If the exception had an associated error code, use it, otherwise, - # fake it with a non-triggering exception code - errcode = getattr(e, 'smtp_code', -1) - errmsg = getattr(e, 'smtp_error', 'ignore') + except smtplib.SMTPResponseException, e: + syslog('smtp-failure', 'SMTP session failure: %s, %s, msgid: %s', + e.smtp_code, e.smtp_error, msgid) + # If this was a permanent failure, don't add the recipients to the + # refused, because we don't want them to be added to failures. + # Otherwise, if the MTA rejects the message because of the message + # content (e.g. it's spam, virii, or has syntactic problems), then + # this will end up registering a bounce score for every recipient. + # Definitely /not/ what we want. + if e.smtp_code < 500 or e.smtp_code == 552: + # It's a temporary failure + for r in recips: + refused[r] = (e.smtp_code, e.smtp_error) + except (socket.error, IOError, smtplib.SMTPException), e: + # MTA not responding, or other socket problems, or any other kind of + # SMTPException. In that case, nothing got delivered, so treat this + # as a temporary failure. + syslog('smtp-failure', 'Low level smtp error: %s, msgid: %s', e, msgid) + error = str(e) for r in recips: - refused[r] = (errcode, errmsg) + refused[r] = (-1, error) failures.update(refused) diff --git a/Mailman/Handlers/Scrubber.py b/Mailman/Handlers/Scrubber.py index d4af9f059..bb0d2f4e8 100644 --- a/Mailman/Handlers/Scrubber.py +++ b/Mailman/Handlers/Scrubber.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2003 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2005 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -14,8 +14,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -"""Cleanse a message for archiving. -""" +"""Cleanse a message for archiving.""" from __future__ import nested_scopes @@ -27,11 +26,12 @@ import errno import binascii import tempfile from cStringIO import StringIO -from types import IntType +from types import IntType, StringType from email.Utils import parsedate from email.Parser import HeaderParser from email.Generator import Generator +from email.Charset import Charset from Mailman import mm_cfg from Mailman import Utils @@ -53,10 +53,17 @@ BR = '<br>\n' SPACE = ' ' try: + True, False +except NameError: + True = 1 + False = 0 + + +try: from mimetypes import guess_all_extensions except ImportError: import mimetypes - def guess_all_extensions(ctype, strict=1): + def guess_all_extensions(ctype, strict=True): # BAW: sigh, guess_all_extensions() is new in Python 2.3 all = [] def check(map): @@ -76,7 +83,7 @@ def guess_extension(ctype, ext): # and .wiz are all mapped to application/msword. This sucks for finding # the best reverse mapping. If the extension is one of the giving # mappings, we'll trust that, otherwise we'll just guess. :/ - all = guess_all_extensions(ctype, strict=0) + all = guess_all_extensions(ctype, strict=False) if ext in all: return ext return all and all[0] @@ -93,8 +100,9 @@ def guess_extension(ctype, ext): # This isn't perfect because we still get stuff like the multipart boundaries, # but see below for how we corrupt that to our nefarious goals. class ScrubberGenerator(Generator): - def __init__(self, outfp, mangle_from_=1, maxheaderlen=78, skipheaders=1): - Generator.__init__(self, outfp, mangle_from_=0) + def __init__(self, outfp, mangle_from_=True, + maxheaderlen=78, skipheaders=True): + Generator.__init__(self, outfp, mangle_from_=False) self.__skipheaders = skipheaders def _write_headers(self, msg): @@ -156,12 +164,19 @@ def calculate_attachments_dir(mlist, msg, msgdata): def process(mlist, msg, msgdata=None): sanitize = mm_cfg.ARCHIVE_HTML_SANITIZER - outer = 1 + outer = True if msgdata is None: msgdata = {} + if msgdata: + # msgdata is available if it is in GLOBAL_PIPELINE + # ie. not in digest or archiver + # check if the list owner want to scrub regular delivery + if not mlist.scrub_nondigest: + return dir = calculate_attachments_dir(mlist, msg, msgdata) charset = None lcset = Utils.GetCharSet(mlist.preferred_language) + lcset_out = Charset(lcset).output_charset or lcset # Now walk over all subparts of this message and scrub out various types for part in msg.walk(): ctype = part.get_type(part.get_default_type()) @@ -172,11 +187,29 @@ def process(mlist, msg, msgdata=None): # message. if charset is None: charset = part.get_content_charset(lcset) + # TK: if part is attached then check charset and scrub if none + if part.get('content-disposition') and \ + not part.get_content_charset(): + omask = os.umask(002) + try: + url = save_attachment(mlist, part, dir) + finally: + os.umask(omask) + filename = part.get_filename(_('not available')) + filename = Utils.oneline(filename, lcset) + del part['content-type'] + del part['content-transfer-encoding'] + part.set_payload(_("""\ +An embedded and charset-unspecified text was scrubbed... +Name: %(filename)s +Url: %(url)s +"""), lcset) elif ctype == 'text/html' and isinstance(sanitize, IntType): if sanitize == 0: if outer: raise DiscardMessage del part['content-type'] + del part['content-transfer-encoding'] part.set_payload(_('HTML attachment scrubbed and removed'), # Adding charset arg and removing content-tpe # sets content-type to text/plain @@ -190,10 +223,11 @@ def process(mlist, msg, msgdata=None): # lists. omask = os.umask(002) try: - url = save_attachment(mlist, part, dir, filter_html=0) + url = save_attachment(mlist, part, dir, filter_html=False) finally: os.umask(omask) del part['content-type'] + del part['content-transfer-encoding'] part.set_payload(_("""\ An HTML attachment was scrubbed... URL: %(url)s @@ -201,7 +235,7 @@ URL: %(url)s else: # HTML-escape it and store it as an attachment, but make it # look a /little/ bit prettier. :( - payload = Utils.websafe(part.get_payload(decode=1)) + payload = Utils.websafe(part.get_payload(decode=True)) # For whitespace in the margin, change spaces into # non-breaking spaces, and tabs into 8 of those. Then use a # mono-space font. Still looks hideous to me, but then I'd @@ -216,7 +250,7 @@ URL: %(url)s del part['content-transfer-encoding'] omask = os.umask(002) try: - url = save_attachment(mlist, part, dir, filter_html=0) + url = save_attachment(mlist, part, dir, filter_html=False) finally: os.umask(omask) del part['content-type'] @@ -248,9 +282,17 @@ Url: %(url)s # If the message isn't a multipart, then we'll strip it out as an # attachment that would have to be separately downloaded. Pipermail # will transform the url into a hyperlink. - elif not part.is_multipart(): - payload = part.get_payload(decode=1) + elif part and not part.is_multipart(): + payload = part.get_payload(decode=True) ctype = part.get_type() + # XXX Under email 2.5, it is possible that payload will be None. + # This can happen when you have a Content-Type: multipart/* with + # only one part and that part has two blank lines between the + # first boundary and the end boundary. In email 3.0 you end up + # with a string in the payload. I think in this case it's safe to + # ignore the part. + if payload is None: + continue size = len(payload) omask = os.umask(002) try: @@ -259,6 +301,7 @@ Url: %(url)s os.umask(omask) desc = part.get('content-description', _('not available')) filename = part.get_filename(_('not available')) + filename = Utils.oneline(filename, lcset) del part['content-type'] del part['content-transfer-encoding'] part.set_payload(_("""\ @@ -269,16 +312,19 @@ Size: %(size)d bytes Desc: %(desc)s Url : %(url)s """), lcset) - outer = 0 + outer = False # We still have to sanitize multipart messages to flat text because # Pipermail can't handle messages with list payloads. This is a kludge; # def (n) clever hack ;). - if msg.is_multipart(): + if msg.is_multipart() and sanitize <> 2: # By default we take the charset of the first text/plain part in the # message, but if there was none, we'll use the list's preferred # language's charset. - if charset is None or charset == 'us-ascii': - charset = lcset + if not charset or charset == 'us-ascii': + charset = lcset_out + else: + # normalize to the output charset if input/output are different + charset = Charset(charset).output_charset or charset # We now want to concatenate all the parts which have been scrubbed to # text/plain, into a single text/plain payload. We need to make sure # all the characters in the concatenated string are in the same @@ -286,21 +332,32 @@ Url : %(url)s # BAW: Martin's original patch suggested we might want to try # generalizing to utf-8, and that's probably a good idea (eventually). text = [] - for part in msg.get_payload(): + for part in msg.walk(): + # TK: bug-id 1099138 and multipart + if not part or part.is_multipart(): + continue # All parts should be scrubbed to text/plain by now. partctype = part.get_content_type() if partctype <> 'text/plain': - text.append(_('Skipped content of type %(partctype)s')) + text.append(_('Skipped content of type %(partctype)s\n')) continue try: - t = part.get_payload(decode=1) + t = part.get_payload(decode=True) except binascii.Error: t = part.get_payload() - partcharset = part.get_content_charset() + # TK: get_content_charset() returns 'iso-2022-jp' for internally + # crafted (scrubbed) 'euc-jp' text part. So, first try + # get_charset(), then get_content_charset() for the parts + # which are already embeded in the incoming message. + partcharset = part.get_charset() + if partcharset: + partcharset = str(partcharset) + else: + partcharset = part.get_content_charset() if partcharset and partcharset <> charset: try: t = unicode(t, partcharset, 'replace') - except (UnicodeError, LookupError): + except (UnicodeError, LookupError, ValueError): # Replace funny characters. We use errors='replace' for # both calls since the first replace will leave U+FFFD, # which isn't ASCII encodeable. @@ -309,12 +366,13 @@ Url : %(url)s try: # Should use HTML-Escape, or try generalizing to UTF-8 t = t.encode(charset, 'replace') - except (UnicodeError, LookupError): + except (UnicodeError, LookupError, ValueError): t = t.encode(lcset, 'replace') # Separation is useful - if not t.endswith('\n'): - t += '\n' - text.append(t) + if isinstance(t, StringType): + if not t.endswith('\n'): + t += '\n' + text.append(t) # Now join the text and set the payload sep = _('-------------- next part --------------\n') del msg['content-type'] @@ -339,17 +397,26 @@ def makedirs(dir): -def save_attachment(mlist, msg, dir, filter_html=1): +def save_attachment(mlist, msg, dir, filter_html=True): fsdir = os.path.join(mlist.archive_dir(), dir) makedirs(fsdir) # Figure out the attachment type and get the decoded data - decodedpayload = msg.get_payload(decode=1) + decodedpayload = msg.get_payload(decode=True) # BAW: mimetypes ought to handle non-standard, but commonly found types, # e.g. image/jpg (should be image/jpeg). For now we just store such # things as application/octet-streams since that seems the safest. ctype = msg.get_content_type() - fnext = os.path.splitext(msg.get_filename(''))[1] - ext = guess_extension(ctype, fnext) + # i18n file name is encoded + lcset = Utils.GetCharSet(mlist.preferred_language) + filename = Utils.oneline(msg.get_filename(''), lcset) + fnext = os.path.splitext(filename)[1] + # For safety, we should confirm this is valid ext for content-type + # but we can use fnext if we introduce fnext filtering + if mm_cfg.SCRUBBER_USE_ATTACHMENT_FILENAME_EXTENSION: + # HTML message doesn't have filename :-( + ext = fnext or guess_extension(ctype, fnext) + else: + ext = guess_extension(ctype, fnext) if not ext: # We don't know what it is, so assume it's just a shapeless # application/octet-stream, unless the Content-Type: is @@ -368,7 +435,7 @@ def save_attachment(mlist, msg, dir, filter_html=1): # Now base the filename on what's in the attachment, uniquifying it if # necessary. filename = msg.get_filename() - if not filename: + if not filename or mm_cfg.SCRUBBER_DONT_USE_ATTACHMENT_FILENAME: filebase = 'attachment' else: # Sanitize the filename given in the message headers @@ -388,7 +455,7 @@ def save_attachment(mlist, msg, dir, filter_html=1): # after filebase, e.g. msgdir/filebase-cnt.ext counter = 0 extra = '' - while 1: + while True: path = os.path.join(fsdir, filebase + extra + ext) # Generally it is not a good idea to test for file existance # before just trying to create it, but the alternatives aren't diff --git a/Mailman/Handlers/SpamDetect.py b/Mailman/Handlers/SpamDetect.py index 6a67f3410..b5f9d0b6a 100644 --- a/Mailman/Handlers/SpamDetect.py +++ b/Mailman/Handlers/SpamDetect.py @@ -1,17 +1,17 @@ -# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2004 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software +# along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. """Do more detailed spam detection. @@ -25,26 +25,116 @@ TBD: This needs to be made more configurable and robust. """ import re +from cStringIO import StringIO + +from email.Generator import Generator from Mailman import mm_cfg from Mailman import Errors +from Mailman import i18n +from Mailman.Handlers.Hold import hold_for_approval + +try: + True, False +except NameError: + True = 1 + False = 0 + +# First, play footsie with _ so that the following are marked as translated, +# but aren't actually translated until we need the text later on. +def _(s): + return s class SpamDetected(Errors.DiscardMessage): """The message contains known spam""" +class HeaderMatchHold(Errors.HoldMessage): + reason = _('The message headers matched a filter rule') + + +# And reset the translator +_ = i18n._ + + + +class Tee: + def __init__(self, outfp_a, outfp_b): + self._outfp_a = outfp_a + self._outfp_b = outfp_b + + def write(self, s): + self._outfp_a.write(s) + self._outfp_b.write(s) + + +# Class to capture the headers separate from the message body +class HeaderGenerator(Generator): + def __init__(self, outfp, mangle_from_=True, maxheaderlen=78): + Generator.__init__(self, outfp, mangle_from_, maxheaderlen) + self._headertxt = '' + + def _write_headers(self, msg): + sfp = StringIO() + oldfp = self._fp + self._fp = Tee(oldfp, sfp) + try: + Generator._write_headers(self, msg) + finally: + self._fp = oldfp + self._headertxt = sfp.getvalue() + + def header_text(self): + return self._headertxt + def process(mlist, msg, msgdata): - if msgdata.get('approved'): + # Don't check if the message has been approved OR it is generated + # internally for administration because holding '-owner' notification + # may cause infinite loop of checking. (Actually, it is stopped + # elsewhere.) + if msgdata.get('approved') or msg.get('x-list-administrivia'): return + # First do site hard coded header spam checks for header, regex in mm_cfg.KNOWN_SPAMMERS: cre = re.compile(regex, re.IGNORECASE) - value = msg[header] - if not value: + for value in msg.get_all(header, []): + mo = cre.search(value) + if mo: + # we've detected spam, so throw the message away + raise SpamDetected + # Now do header_filter_rules + # TK: Collect headers in sub-parts because attachment filename + # extension may be a clue to possible virus/spam. + if msg.is_multipart(): + headers = '' + for p in msg.walk(): + g = HeaderGenerator(StringIO()) + g.flatten(p) + headers += g.header_text() + else: + # Only the top level header should be checked. + g = HeaderGenerator(StringIO()) + g.flatten(msg) + headers = g.header_text() + # Now reshape headers (remove extra CR and connect multiline). + headers = re.sub('\n+', '\n', headers) + headers = re.sub('\n\s', ' ', headers) + for patterns, action, empty in mlist.header_filter_rules: + if action == mm_cfg.DEFER: continue - mo = cre.search(value) - if mo: - # we've detected spam, so throw the message away - raise SpamDetected + for pattern in patterns.splitlines(): + if pattern.startswith('#'): + continue + if re.search(pattern, headers, re.IGNORECASE|re.MULTILINE): + if action == mm_cfg.DISCARD: + raise Errors.DiscardMessage + if action == mm_cfg.REJECT: + raise Errors.RejectMessage( + _('Message rejected by filter rule match')) + if action == mm_cfg.HOLD: + hold_for_approval(mlist, msg, msgdata, HeaderMatchHold) + if action == mm_cfg.ACCEPT: + return diff --git a/Mailman/Handlers/ToDigest.py b/Mailman/Handlers/ToDigest.py index 1fc8928b2..0e4bb94d9 100644 --- a/Mailman/Handlers/ToDigest.py +++ b/Mailman/Handlers/ToDigest.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2005 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -37,13 +37,15 @@ from email.Generator import Generator from email.MIMEBase import MIMEBase from email.MIMEText import MIMEText from email.MIMEMessage import MIMEMessage -from email.Utils import getaddresses +from email.Utils import getaddresses, formatdate from email.Header import decode_header, make_header, Header +from email.Charset import Charset from Mailman import mm_cfg from Mailman import Utils from Mailman import Message from Mailman import i18n +from Mailman import Errors from Mailman.Mailbox import Mailbox from Mailman.MemberAdaptor import ENABLED from Mailman.Handlers.Decorate import decorate @@ -86,9 +88,20 @@ def process(mlist, msg, msgdata): if size / 1024.0 >= mlist.digest_size_threshhold: # This is a bit of a kludge to get the mbox file moved to the digest # queue directory. - mboxfp.seek(0) - send_digests(mlist, mboxfp) - os.unlink(mboxfile) + try: + # Let's close in try - except here because a error in send_digest + # can stop regular delivery silently. Unsuccessful digest + # delivery should be tried again by cron and the site + # administrator will be notified of any error explicitly by the + # cron error message. + mboxfp.seek(0) + send_digests(mlist, mboxfp) + os.unlink(mboxfile) + except Exception, errmsg: + # I know bare except is prohibited in mailman coding but we can't + # forcast what new exception can occur here. + syslog('error', 'send_digests() failed: %s', errmsg) + pass mboxfp.close() @@ -136,9 +149,11 @@ def send_digests(mlist, mboxfp): def send_i18n_digests(mlist, mboxfp): mbox = Mailbox(mboxfp) - # Prepare common information + # Prepare common information (first lang/charset) lang = mlist.preferred_language lcset = Utils.GetCharSet(lang) + lcset_out = Charset(lcset).output_charset or lcset + # Common Information (contd) realname = mlist.real_name volume = mlist.volume issue = mlist.next_digest_number @@ -146,6 +161,7 @@ def send_i18n_digests(mlist, mboxfp): digestsubj = Header(digestid, lcset, header_name='Subject') # Set things up for the MIME digest. Only headers not added by # CookHeaders need be added here. + # Date/Message-ID should be added here also. mimemsg = Message.Message() mimemsg['Content-Type'] = 'multipart/mixed' mimemsg['MIME-Version'] = '1.0' @@ -153,6 +169,8 @@ def send_i18n_digests(mlist, mboxfp): mimemsg['Subject'] = digestsubj mimemsg['To'] = mlist.GetListEmail() mimemsg['Reply-To'] = mlist.GetListEmail() + mimemsg['Date'] = formatdate(localtime=1) + mimemsg['Message-ID'] = Utils.unique_message_id(mlist) # Set things up for the rfc1153 digest plainmsg = StringIO() rfc1153msg = Message.Message() @@ -160,6 +178,8 @@ def send_i18n_digests(mlist, mboxfp): rfc1153msg['Subject'] = digestsubj rfc1153msg['To'] = mlist.GetListEmail() rfc1153msg['Reply-To'] = mlist.GetListEmail() + rfc1153msg['Date'] = formatdate(localtime=1) + rfc1153msg['Message-ID'] = Utils.unique_message_id(mlist) separator70 = '-' * 70 separator30 = '-' * 30 # In the rfc1153 digest, the masthead contains the digest boilerplate plus @@ -210,18 +230,19 @@ def send_i18n_digests(mlist, mboxfp): if msg == '': # It was an unparseable message msg = mbox.next() + continue msgcount += 1 messages.append(msg) # Get the Subject header msgsubj = msg.get('subject', _('(no subject)')) - subject = oneline(msgsubj, lcset) + subject = Utils.oneline(msgsubj, lcset) # Don't include the redundant subject prefix in the toc mo = re.match('(re:? *)?(%s)' % re.escape(mlist.subject_prefix), subject, re.IGNORECASE) if mo: subject = subject[:mo.start(2)] + subject[mo.end(2):] username = '' - addresses = getaddresses([oneline(msg.get('from', ''), lcset)]) + addresses = getaddresses([Utils.oneline(msg.get('from', ''), lcset)]) # Take only the first author we find if isinstance(addresses, ListType) and addresses: username = addresses[0][0] @@ -301,15 +322,30 @@ def send_i18n_digests(mlist, mboxfp): print >> plainmsg, separator30 print >> plainmsg # Use Mailman.Handlers.Scrubber.process() to get plain text - msg = scrubber(mlist, msg) + try: + msg = scrubber(mlist, msg) + except Errors.DiscardMessage: + print >> plainmsg, _('[Message discarded by content filter]') + continue # Honor the default setting for h in mm_cfg.PLAIN_DIGEST_KEEP_HEADERS: if msg[h]: - uh = Utils.wrap('%s: %s' % (h, oneline(msg[h], lcset))) + uh = Utils.wrap('%s: %s' % (h, Utils.oneline(msg[h], lcset))) uh = '\n\t'.join(uh.split('\n')) print >> plainmsg, uh print >> plainmsg - payload = msg.get_payload(decode=True) + payload = msg.get_payload(decode=True)\ + or msg.as_string().split('\n\n',1)[1] + mcset = msg.get_content_charset('') + if mcset and mcset <> lcset and mcset <> lcset_out: + try: + payload = unicode(payload, mcset, 'replace' + ).encode(lcset, 'replace') + except LookupError: + # TK: Message has something unknown charset. + # _out means charset in 'outer world'. + payload = unicode(payload, lcset_out, 'replace' + ).encode(lcset, 'replace') print >> plainmsg, payload if not payload.endswith('\n'): print >> plainmsg @@ -375,16 +411,3 @@ def send_i18n_digests(mlist, mboxfp): recips=plainrecips, listname=mlist.internal_name(), isdigest=True) - - - -def oneline(s, cset): - # Decode header string in one line and convert into specified charset - try: - h = make_header(decode_header(s)) - ustr = h.__unicode__() - oneline = UEMPTYSTRING.join(ustr.splitlines()) - return oneline.encode(cset, 'replace') - except (LookupError, UnicodeError): - # possibly charset problem. return with undecoded string in one line. - return EMPTYSTRING.join(s.splitlines()) diff --git a/Mailman/ListAdmin.py b/Mailman/ListAdmin.py index 282b2823b..ba486e094 100644 --- a/Mailman/ListAdmin.py +++ b/Mailman/ListAdmin.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2004 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -25,9 +25,9 @@ elsewhere. import os import time -import marshal import errno import cPickle +import marshal from cStringIO import StringIO import email @@ -60,6 +60,12 @@ LOST = 2 DASH = '-' NL = '\n' +try: + True, False +except NameError: + True = 1 + False = 0 + class ListAdmin: @@ -69,51 +75,22 @@ class ListAdmin: def InitTempVars(self): self.__db = None - - def __filename(self): - return os.path.join(self.fullpath(), 'request.db') + self.__filename = os.path.join(self.fullpath(), 'request.pck') def __opendb(self): - filename = self.__filename() if self.__db is None: assert self.Locked() try: - fp = open(filename) - self.__db = marshal.load(fp) - fp.close() + fp = open(self.__filename) + try: + self.__db = cPickle.load(fp) + finally: + fp.close() except IOError, e: if e.errno <> errno.ENOENT: raise self.__db = {} - except EOFError, e: - # The unmarshalling failed, which means the file is corrupt. - # Sigh. Start over. - syslog('error', - 'request.db file corrupt for list %s, blowing it away.', - self.internal_name()) - self.__db = {} - # Migrate pre-2.1a3 held subscription records to include the - # fullname data field. - type, version = self.__db.get('version', (IGN, None)) - if version is None: - # No previous revision number, must be upgrading to 2.1a3 or - # beyond from some unknown earlier version. - for id, (type, data) in self.__db.items(): - if type == IGN: - pass - elif type == HELDMSG and len(data) == 5: - # tack on a msgdata dictionary - self.__db[id] = data + ({},) - elif type == SUBSCRIPTION: - if len(data) == 4: - # fullname and lang was added - stime, addr, password, digest = data - lang = self.preferred_language - data = stime, addr, '', password, digest, lang - elif len(data) == 5: - # a fullname field was added - stime, addr, password, digest, lang = data - data = stime, addr, '', password, digest, lang - self.__db[id] = type, data + # put version number in new database + self.__db['version'] = IGN, mm_cfg.REQUESTS_FILE_SCHEMA_VERSION def __closedb(self): if self.__db is not None: @@ -123,36 +100,42 @@ class ListAdmin: # Now save a temp file and do the tmpfile->real file dance. BAW: # should we be as paranoid as for the config.pck file? Should we # use pickle? - tmpfile = self.__filename() + '.tmp' + tmpfile = self.__filename + '.tmp' omask = os.umask(002) try: fp = open(tmpfile, 'w') - marshal.dump(self.__db, fp) - fp.close() - self.__db = None + try: + cPickle.dump(self.__db, fp, 1) + fp.flush() + os.fsync(fp.fileno()) + finally: + fp.close() finally: os.umask(omask) + self.__db = None # Do the dance - os.rename(tmpfile, self.__filename()) + os.rename(tmpfile, self.__filename) - def __request_id(self): - id = self.next_request_id - self.next_request_id += 1 - return id + def __nextid(self): + assert self.Locked() + while True: + next = self.next_request_id + self.next_request_id += 1 + if not self.__db.has_key(next): + break + return next def SaveRequestsDb(self): self.__closedb() def NumRequestsPending(self): self.__opendb() - # Subtrace one for the version pseudo-entry - if self.__db.has_key('version'): - return len(self.__db) - 1 - return len(self.__db) + # Subtract one for the version pseudo-entry + return len(self.__db) - 1 def __getmsgids(self, rtype): self.__opendb() - ids = [k for k, (type, data) in self.__db.items() if type == rtype] + ids = [k for k, (op, data) in self.__db.items() if op == rtype] ids.sort() return ids @@ -198,16 +181,11 @@ class ListAdmin: # Make a copy of msgdata so that subsequent changes won't corrupt the # request database. TBD: remove the `filebase' key since this will # not be relevant when the message is resurrected. - newmsgdata = {} - newmsgdata.update(msgdata) - msgdata = newmsgdata + msgdata = msgdata.copy() # assure that the database is open for writing self.__opendb() # get the next unique id - id = self.__request_id() - while self.__db.has_key(id): - # Shouldn't happen unless the db has gone odd, but let's cope. - id = self.__request_id() + id = self.__nextid() # get the message sender sender = msg.get_sender() # calculate the file name for the message text and write it to disk @@ -217,17 +195,19 @@ class ListAdmin: ext = 'txt' filename = 'heldmsg-%s-%d.%s' % (self.internal_name(), id, ext) omask = os.umask(002) - fp = None try: fp = open(os.path.join(mm_cfg.DATA_DIR, filename), 'w') - if mm_cfg.HOLD_MESSAGES_AS_PICKLES: - cPickle.dump(msg, fp, 1) - else: - g = Generator(fp) - g(msg, 1) - finally: - if fp: + try: + if mm_cfg.HOLD_MESSAGES_AS_PICKLES: + cPickle.dump(msg, fp, 1) + else: + g = Generator(fp) + g(msg, 1) + fp.flush() + os.fsync(fp.fileno()) + finally: fp.close() + finally: os.umask(omask) # save the information to the request database. for held message # entries, each record in the database will be of the following @@ -365,8 +345,8 @@ class ListAdmin: \tSubject: %(subject)s''' % { 'listname' : self.internal_name(), 'rejection': rejection, - 'sender' : sender.replace('%', '%%'), - 'subject' : subject.replace('%', '%%'), + 'sender' : str(sender).replace('%', '%%'), + 'subject' : str(subject).replace('%', '%%'), } if comment: note += '\n\tReason: ' + comment.replace('%', '%%') @@ -387,9 +367,7 @@ class ListAdmin: # Assure that the database is open for writing self.__opendb() # Get the next unique id - id = self.__request_id() - assert not self.__db.has_key(id) - # + id = self.__nextid() # Save the information to the request database. for held subscription # entries, each record in the database will be one of the following # format: @@ -399,7 +377,6 @@ class ListAdmin: # the subscriber's selected password (TBD: is this safe???) # the digest flag # the user's preferred language - # data = time.time(), addr, fullname, password, digest, lang self.__db[id] = (SUBSCRIPTION, data) # @@ -441,7 +418,7 @@ class ListAdmin: assert value == mm_cfg.SUBSCRIBE try: userdesc = UserDesc(addr, fullname, password, digest, lang) - self.ApprovedAddMember(userdesc) + self.ApprovedAddMember(userdesc, whence='via admin approval') except Errors.MMAlreadyAMember: # User has already been subscribed, after sending the request pass @@ -454,8 +431,7 @@ class ListAdmin: # Assure the database is open for writing self.__opendb() # Get the next unique id - id = self.__request_id() - assert not self.__db.has_key(id) + id = self.__nextid() # All we need to do is save the unsubscribing address self.__db[id] = (UNSUBSCRIPTION, addr) syslog('vette', '%s: held unsubscription request from %s', @@ -540,9 +516,30 @@ class ListAdmin: # These always include the requests time, the sender, subject, default # rejection reason, and message text. When of length 6, it also # includes the message metadata dictionary on the end of the tuple. - self.__opendb() - for id, (type, info) in self.__db.items(): - if type == SUBSCRIPTION: + # + # In Mailman 2.1.5 we converted these files to pickles. + filename = os.path.join(self.fullpath(), 'request.db') + try: + fp = open(filename) + try: + self.__db = marshal.load(fp) + finally: + fp.close() + os.unlink(filename) + except IOError, e: + if e.errno <> errno.ENOENT: raise + filename = os.path.join(self.fullpath(), 'request.pck') + try: + fp = open(filename) + try: + self.__db = cPickle.load(fp) + finally: + fp.close() + except IOError, e: + if e.errno <> errno.ENOENT: raise + self.__db = {} + for id, (op, info) in self.__db.items(): + if op == SUBSCRIPTION: if len(info) == 4: # pre-2.1a2 compatibility when, addr, passwd, digest = info @@ -557,7 +554,7 @@ class ListAdmin: continue # Here's the new layout self.__db[id] = when, addr, fullname, passwd, digest, lang - elif type == HELDMSG: + elif op == HELDMSG: if len(info) == 5: when, sender, subject, reason, text = info msgdata = {} diff --git a/Mailman/Logging/Logger.py b/Mailman/Logging/Logger.py index 8a416b3f6..6e72843f0 100644 --- a/Mailman/Logging/Logger.py +++ b/Mailman/Logging/Logger.py @@ -1,17 +1,17 @@ -# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2005 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software +# along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. """File-based logger, writes to named category files in mm_cfg.LOG_DIR.""" @@ -85,7 +85,7 @@ class Logger: def write(self, msg): if isinstance(msg, StringType): - msg = unicode(msg, self.__encoding) + msg = unicode(msg, self.__encoding, 'replace') f = self.__get_f() try: f.write(msg) diff --git a/Mailman/MTA/Manual.py b/Mailman/MTA/Manual.py index dd73f43cf..fae3889c2 100644 --- a/Mailman/MTA/Manual.py +++ b/Mailman/MTA/Manual.py @@ -97,7 +97,7 @@ equivalent) file by adding the following lines, and possibly running the _('Mailing list creation request for list %(listname)s'), sfp.getvalue(), mm_cfg.DEFAULT_SERVER_LANGUAGE) outq = get_switchboard(mm_cfg.OUTQUEUE_DIR) - outq.enqueue(msg, recips=[siteowner]) + outq.enqueue(msg, recips=[siteowner], nodecorate=1) @@ -141,4 +141,4 @@ equivalent) file by removing the following lines, and possibly running the _('Mailing list removal request for list %(listname)s'), sfp.getvalue(), mm_cfg.DEFAULT_SERVER_LANGUAGE) outq = get_switchboard(mm_cfg.OUTQUEUE_DIR) - outq.enqueue(msg, recips=[siteowner]) + outq.enqueue(msg, recips=[siteowner], nodecorate=1) diff --git a/Mailman/MailList.py b/Mailman/MailList.py index fb85584d8..0a52567d9 100644 --- a/Mailman/MailList.py +++ b/Mailman/MailList.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2005 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -38,6 +38,7 @@ from types import * import email.Iterators from email.Utils import getaddresses, formataddr, parseaddr +from email.Header import Header from Mailman import mm_cfg from Mailman import Utils @@ -56,6 +57,7 @@ from Mailman.HTMLFormatter import HTMLFormatter from Mailman.ListAdmin import ListAdmin from Mailman.SecurityManager import SecurityManager from Mailman.TopicMgr import TopicMgr +from Mailman import Pending # gui components package from Mailman import Gui @@ -64,7 +66,6 @@ from Mailman import Gui from Mailman import MemberAdaptor from Mailman.OldStyleMemberships import OldStyleMemberships from Mailman import Message -from Mailman import Pending from Mailman import Site from Mailman import i18n from Mailman.Logging.Syslog import syslog @@ -84,7 +85,7 @@ except NameError: # Use mixins here just to avoid having any one chunk be too large. class MailList(HTMLFormatter, Deliverer, ListAdmin, Archiver, Digester, SecurityManager, Bouncer, GatewayManager, - Autoresponder, TopicMgr): + Autoresponder, TopicMgr, Pending.Pending): # # A MailList object's basic Python object model support @@ -194,8 +195,11 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, def GetOwnerEmail(self): return self.getListAddress('owner') - def GetRequestEmail(self): - return self.getListAddress('request') + def GetRequestEmail(self, cookie=''): + if mm_cfg.VERP_CONFIRMATIONS and cookie: + return self.GetConfirmEmail(cookie) + else: + return self.getListAddress('request') def GetConfirmEmail(self, cookie): return mm_cfg.VERP_CONFIRM_FORMAT % { @@ -203,6 +207,26 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, 'cookie': cookie, } + '@' + self.host_name + def GetConfirmJoinSubject(self, listname, cookie): + if mm_cfg.VERP_CONFIRMATIONS and cookie: + cset = Utils.GetCharSet(self.preferred_language) + subj = Header( + _('Your confirmation is required to join the %(listname)s mailing list'), + cset, header_name='subject') + return subj + else: + return 'confirm ' + cookie + + def GetConfirmLeaveSubject(self, listname, cookie): + if mm_cfg.VERP_CONFIRMATIONS and cookie: + cset = Utils.GetCharSet(self.preferred_language) + subj = Header( + _('Your confirmation is required to leave the %(listname)s mailing list'), + cset, header_name='subject') + return subj + else: + return 'confirm ' + cookie + def GetListEmail(self): return self.getListAddress() @@ -251,7 +275,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, if name: self._full_path = Site.get_listpath(name) else: - self._full_path = None + self._full_path = '' # Only one level of mixin inheritance allowed for baseclass in self.__class__.__bases__: if hasattr(baseclass, 'InitTempVars'): @@ -315,6 +339,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, self.send_goodbye_msg = mm_cfg.DEFAULT_SEND_GOODBYE_MSG self.bounce_matching_headers = \ mm_cfg.DEFAULT_BOUNCE_MATCHING_HEADERS + self.header_filter_rules = [] self.anonymous_list = mm_cfg.DEFAULT_ANONYMOUS_LIST internalname = self.internal_name() self.real_name = internalname[0].upper() + internalname[1:] @@ -334,7 +359,11 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, self.include_list_post_header = 1 self.filter_mime_types = mm_cfg.DEFAULT_FILTER_MIME_TYPES self.pass_mime_types = mm_cfg.DEFAULT_PASS_MIME_TYPES + self.filter_filename_extensions = \ + mm_cfg.DEFAULT_FILTER_FILENAME_EXTENSIONS + self.pass_filename_extensions = mm_cfg.DEFAULT_PASS_FILENAME_EXTENSIONS self.filter_content = mm_cfg.DEFAULT_FILTER_CONTENT + self.collapse_alternatives = mm_cfg.DEFAULT_COLLAPSE_ALTERNATIVES self.convert_html_to_plaintext = \ mm_cfg.DEFAULT_CONVERT_HTML_TO_PLAINTEXT self.filter_action = mm_cfg.DEFAULT_FILTER_ACTION @@ -357,6 +386,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, self.discard_these_nonmembers = [] self.forward_auto_discards = mm_cfg.DEFAULT_FORWARD_AUTO_DISCARDS self.generic_nonmember_action = mm_cfg.DEFAULT_GENERIC_NONMEMBER_ACTION + self.nonmember_rejection_notice = '' # Ban lists self.ban_list = [] # BAW: This should really be set in SecurityManager.InitVars() @@ -365,7 +395,6 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # 2-tuple of the date of the last autoresponse and the number of # autoresponses sent on that date. self.hold_and_cmd_autoresponses = {} - # Only one level of mixin inheritance allowed for baseclass in self.__class__.__bases__: if hasattr(baseclass, 'InitVars'): @@ -382,6 +411,10 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, self.encode_ascii_prefixes = 0 else: self.encode_ascii_prefixes = 2 + # scrub regular delivery + self.scrub_nondigest = mm_cfg.DEFAULT_SCRUB_NONDIGEST + # automatic discarding + self.max_days_to_hold = mm_cfg.DEFAULT_MAX_DAYS_TO_HOLD # @@ -432,7 +465,8 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # # List creation # - def Create(self, name, admin, crypted_password, langs=None): + def Create(self, name, admin, crypted_password, + langs=None, emailhost=None): if Utils.list_exists(name): raise Errors.MMListAlreadyExistsError, name # Validate what will be the list's posting address. If that's @@ -440,7 +474,9 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # part doesn't really matter, since that better already be valid. # However, most scripts already catch MMBadEmailError as exceptions on # the admin's email address, so transform the exception. - postingaddr = '%s@%s' % (name, mm_cfg.DEFAULT_EMAIL_HOST) + if emailhost is None: + emailhost = mm_cfg.DEFAULT_EMAIL_HOST + postingaddr = '%s@%s' % (name, emailhost) try: Utils.ValidateEmail(postingaddr) except Errors.MMBadEmailError: @@ -564,7 +600,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, if type(dict) <> DictType: return None, 'Load() expected to return a dictionary' except (EOFError, ValueError, TypeError, MemoryError, - cPickle.PicklingError), e: + cPickle.PicklingError, cPickle.UnpicklingError), e: return None, e finally: fp.close() @@ -572,7 +608,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, self.__timestamp = mtime return dict, None - def Load(self, check_version=1): + def Load(self, check_version=True): if not Utils.list_exists(self.internal_name()): raise Errors.MMUnknownListError # We first try to load config.pck, which contains the up-to-date @@ -605,13 +641,20 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, raise Errors.MMCorruptListDatabaseError, e # Now, if we didn't end up using the primary database file, we want to # copy the fallback into the primary so that the logic in Save() will - # still work. For giggles, we'll copy it to a safety backup. - if file == plast: - shutil.copy(file, pfile) - shutil.copy(file, pfile + '.safety') - elif file == dlast: - shutil.copy(file, dfile) - shutil.copy(file, pfile + '.safety') + # still work. For giggles, we'll copy it to a safety backup. Note we + # MUST do this with the underlying list lock acquired. + if file == plast or file == dlast: + syslog('error', 'fixing corrupt config file, using: %s', file) + unlock = True + try: + try: + self.__lock.lock() + except LockFile.AlreadyLockedError: + unlock = False + self.__fix_corrupt_pckfile(file, pfile, plast, dfile, dlast) + finally: + if unlock: + self.__lock.unlock() # Copy the loaded dictionary into the attributes of the current # mailing list object, then run sanity check on the data. self.__dict__.update(dict) @@ -619,6 +662,36 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, self.CheckVersion(dict) self.CheckValues() + def __fix_corrupt_pckfile(self, file, pfile, plast, dfile, dlast): + if file == plast: + # Move aside any existing pickle file and delete any existing + # safety file. This avoids EPERM errors inside the shutil.copy() + # calls if those files exist with different ownership. + try: + os.rename(pfile, pfile + '.corrupt') + except OSError, e: + if e.errno <> errno.ENOENT: raise + try: + os.remove(pfile + '.safety') + except OSError, e: + if e.errno <> errno.ENOENT: raise + shutil.copy(file, pfile) + shutil.copy(file, pfile + '.safety') + elif file == dlast: + # Move aside any existing marshal file and delete any existing + # safety file. This avoids EPERM errors inside the shutil.copy() + # calls if those files exist with different ownership. + try: + os.rename(dfile, dfile + '.corrupt') + except OSError, e: + if e.errno <> errno.ENOENT: raise + try: + os.remove(dfile + '.safety') + except OSError, e: + if e.errno <> errno.ENOENT: raise + shutil.copy(file, dfile) + shutil.copy(file, dfile + '.safety') + # # Sanity checks @@ -691,14 +764,14 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, """ invitee = userdesc.address Utils.ValidateEmail(invitee) - requestaddr = self.GetRequestEmail() # Hack alert! Squirrel away a flag that only invitations have, so # that we can do something slightly different when an invitation # subscription is confirmed. In those cases, we don't need further # admin approval, even if the list is so configured. The flag is the # list name to prevent invitees from cross-subscribing. userdesc.invitation = self.internal_name() - cookie = Pending.new(Pending.SUBSCRIPTION, userdesc) + cookie = self.pend_new(Pending.SUBSCRIPTION, userdesc) + requestaddr = self.getListAddress('request') confirmurl = '%s/%s' % (self.GetScriptURL('confirm', absolute=1), cookie) listname = self.real_name @@ -712,17 +785,13 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, 'cookie' : cookie, 'listowner' : self.GetOwnerEmail(), }, mlist=self) - if mm_cfg.VERP_CONFIRMATIONS: - subj = _( - 'You have been invited to join the %(listname)s mailing list') - sender = self.GetConfirmEmail(cookie) - else: - # Do it the old fashioned way - subj = 'confirm ' + cookie - sender = requestaddr + sender = self.GetRequestEmail(cookie) msg = Message.UserNotification( - invitee, sender, subj, - text, lang=self.preferred_language) + invitee, sender, + text=text, lang=self.preferred_language) + subj = self.GetConfirmJoinSubject(listname, cookie) + del msg['subject'] + msg['Subject'] = subj msg.send(self) def AddMember(self, userdesc, remote=None): @@ -807,11 +876,11 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # means the user must confirm; 2 means the admin must approve; 3 means # the user must confirm and then the admin must approve if self.subscribe_policy == 0: - self.ApprovedAddMember(userdesc) + self.ApprovedAddMember(userdesc, whence=remote or '') elif self.subscribe_policy == 1 or self.subscribe_policy == 3: # User confirmation required. BAW: this should probably just # accept a userdesc instance. - cookie = Pending.new(Pending.SUBSCRIPTION, userdesc) + cookie = self.pend_new(Pending.SUBSCRIPTION, userdesc) # Send the user the confirmation mailback if remote is None: by = remote = '' @@ -829,18 +898,18 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, 'listaddr' : self.GetListEmail(), 'listname' : realname, 'cookie' : cookie, - 'requestaddr' : self.GetRequestEmail(), + 'requestaddr' : self.getListAddress('request'), 'remote' : remote, 'listadmin' : self.GetOwnerEmail(), 'confirmurl' : confirmurl, }, lang=lang, mlist=self) msg = Message.UserNotification( - recipient, self.GetRequestEmail(), + recipient, self.GetRequestEmail(cookie), text=text, lang=lang) # BAW: See ChangeMemberAddress() for why we do it this way... del msg['subject'] - msg['Subject'] = 'confirm ' + cookie - msg['Reply-To'] = self.GetRequestEmail() + msg['Subject'] = self.GetConfirmJoinSubject(realname, cookie) + msg['Reply-To'] = self.GetRequestEmail(cookie) msg.send(self) who = formataddr((name, email)) syslog('subscribe', '%s: pending %s %s', @@ -854,7 +923,8 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, raise Errors.MMNeedApproval, _( 'subscriptions to %(realname)s require moderator approval') - def ApprovedAddMember(self, userdesc, ack=None, admin_notif=None, text=''): + def ApprovedAddMember(self, userdesc, ack=None, admin_notif=None, text='', + whence=''): """Add a member right now. The member's subscription must be approved by what ever policy the @@ -903,8 +973,8 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, kind = ' (digest)' else: kind = '' - syslog('subscribe', '%s: new%s %s', self.internal_name(), - kind, formataddr((email, name))) + syslog('subscribe', '%s: new%s %s, %s', self.internal_name(), + kind, formataddr((email, name)), whence) if ack: self.SendSubscribeAck(email, self.getMemberPassword(email), digest, text) @@ -1004,8 +1074,8 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, if newaddr == self.GetListEmail().lower(): raise Errors.MMBadEmailError # Pend the subscription change - cookie = Pending.new(Pending.CHANGE_OF_ADDRESS, - oldaddr, newaddr, globally) + cookie = self.pend_new(Pending.CHANGE_OF_ADDRESS, + oldaddr, newaddr, globally) confirmurl = '%s/%s' % (self.GetScriptURL('confirm', absolute=1), cookie) realname = self.real_name @@ -1016,7 +1086,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, 'listaddr' : self.GetListEmail(), 'listname' : realname, 'cookie' : cookie, - 'requestaddr': self.GetRequestEmail(), + 'requestaddr': self.getListAddress('request'), 'remote' : '', 'listadmin' : self.GetOwnerEmail(), 'confirmurl' : confirmurl, @@ -1029,15 +1099,21 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # Subject: in a separate step, although we have to delete the one # UserNotification adds. msg = Message.UserNotification( - newaddr, self.GetRequestEmail(), + newaddr, self.GetRequestEmail(cookie), text=text, lang=lang) del msg['subject'] - msg['Subject'] = 'confirm ' + cookie - msg['Reply-To'] = self.GetRequestEmail() + msg['Subject'] = self.GetConfirmJoinSubject(realname, cookie) + msg['Reply-To'] = self.GetRequestEmail(cookie) msg.send(self) def ApprovedChangeMemberAddress(self, oldaddr, newaddr, globally): - self.changeMemberAddress(oldaddr, newaddr) + # It's possible they were a member of this list, but choose to change + # their membership globally. In that case, we simply remove the old + # address. + if self.getMemberCPAddress(oldaddr) == newaddr: + self.removeMember(oldaddr) + else: + self.changeMemberAddress(oldaddr, newaddr) # If globally is true, then we also include every list for which # oldaddr is a member. if not globally: @@ -1053,7 +1129,11 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, continue mlist.Lock() try: - mlist.changeMemberAddress(oldaddr, newaddr) + # Same logic as above, re newaddr is already a member + if mlist.getMemberCPAddress(oldaddr) == newaddr: + mlist.removeMember(oldaddr) + else: + mlist.changeMemberAddress(oldaddr, newaddr) mlist.Save() finally: mlist.Unlock() @@ -1063,15 +1143,16 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # Confirmation processing # def ProcessConfirmation(self, cookie, context=None): - data = Pending.confirm(cookie) - if data is None: - raise Errors.MMBadConfirmation, 'data is None' + rec = self.pend_confirm(cookie) + if rec is None: + raise Errors.MMBadConfirmation, 'No cookie record for %s' % cookie try: - op = data[0] - data = data[1:] + op = rec[0] + data = rec[1:] except ValueError: - raise Errors.MMBadConfirmation, 'op-less data %s' % (data,) + raise Errors.MMBadConfirmation, 'op-less data %s' % (rec,) if op == Pending.SUBSCRIPTION: + whence = 'via email confirmation' try: userdesc = data[0] # If confirmation comes from the web, context should be a @@ -1080,6 +1161,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # context is a Message and isn't relevant, so ignore it. if isinstance(context, UserDesc): userdesc += context + whence = 'via web confirmation' addr = userdesc.address fullname = userdesc.fullname password = userdesc.password @@ -1104,7 +1186,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, name = self.real_name raise Errors.MMNeedApproval, _( 'subscriptions to %(name)s require administrator approval') - self.ApprovedAddMember(userdesc) + self.ApprovedAddMember(userdesc, whence=whence) return op, addr, password, digest, lang elif op == Pending.UNSUBSCRIPTION: addr = data[0] @@ -1125,8 +1207,10 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, approved = None # Confirmation should be coming from email, where context should # be the confirming message. If the message does not have an - # Approved: header, this is a discard, otherwise it's an approval - # (if the passwords match). + # Approved: header, this is a discard. If it has an Approved: + # header that does not match the list password, then we'll notify + # the list administrator that they used the wrong password. + # Otherwise it's an approval. if isinstance(context, Message.Message): # See if it's got an Approved: header, either in the headers, # or in the first text/plain section of the response. For @@ -1140,7 +1224,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, subpart = None if subpart: s = StringIO(subpart.get_payload()) - while 1: + while True: line = s.readline() if not line: break @@ -1153,11 +1237,19 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # then approved = line[i+1:].strip() break - # Okay, does the approved header match the list password? - if approved and self.Authenticate([mm_cfg.AuthListAdmin, - mm_cfg.AuthListModerator], - approved) <> mm_cfg.UnAuthorized: - action = mm_cfg.APPROVE + # Is there an approved header? + if approved is not None: + # Does it match the list password? Note that we purposefully + # do not allow the site password here. + if self.Authenticate([mm_cfg.AuthListAdmin, + mm_cfg.AuthListModerator], + approved) <> mm_cfg.UnAuthorized: + action = mm_cfg.APPROVE + else: + # The password didn't match. Re-pend the message and + # inform the list moderators about the problem. + self.pend_repend(cookie, rec) + raise Errors.MMBadPasswordError else: action = mm_cfg.DISCARD try: @@ -1171,11 +1263,13 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, member = data[1] self.setDeliveryStatus(member, MemberAdaptor.ENABLED) return op, member + else: + assert 0, 'Bad op: %s' % op def ConfirmUnsubscription(self, addr, lang=None, remote=None): if lang is None: lang = self.getMemberLanguage(addr) - cookie = Pending.new(Pending.UNSUBSCRIPTION, addr) + cookie = self.pend_new(Pending.UNSUBSCRIPTION, addr) confirmurl = '%s/%s' % (self.GetScriptURL('confirm', absolute=1), cookie) realname = self.real_name @@ -1191,18 +1285,18 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, 'listaddr' : self.GetListEmail(), 'listname' : realname, 'cookie' : cookie, - 'requestaddr' : self.GetRequestEmail(), + 'requestaddr' : self.getListAddress('request'), 'remote' : remote, 'listadmin' : self.GetOwnerEmail(), 'confirmurl' : confirmurl, }, lang=lang, mlist=self) msg = Message.UserNotification( - addr, self.GetRequestEmail(), + addr, self.GetRequestEmail(cookie), text=text, lang=lang) # BAW: See ChangeMemberAddress() for why we do it this way... del msg['subject'] - msg['Subject'] = 'confirm ' + cookie - msg['Reply-To'] = self.GetRequestEmail() + msg['Subject'] = self.GetConfirmLeaveSubject(realname, cookie) + msg['Reply-To'] = self.GetRequestEmail(cookie) msg.send(self) @@ -1211,20 +1305,19 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, # def HasExplicitDest(self, msg): """True if list name or any acceptable_alias is included among the - to or cc addrs.""" - # BAW: fall back to Utils.ParseAddr if the first test fails. - # this is the list's full address + addresses in the recipient headers. + """ + # This is the list's full address. listfullname = '%s@%s' % (self.internal_name(), self.host_name) recips = [] - # check all recipient addresses against the list's explicit addresses, + # Check all recipient addresses against the list's explicit addresses, # specifically To: Cc: and Resent-to: to = [] for header in ('to', 'cc', 'resent-to', 'resent-cc'): to.extend(getaddresses(msg.get_all(header, []))) for fullname, addr in to: - # It's possible that if the header doesn't have a valid - # (i.e. RFC822) value, we'll get None for the address. So skip - # it. + # It's possible that if the header doesn't have a valid RFC 2822 + # value, we'll get None for the address. So skip it. if addr is None: continue addr = addr.lower() @@ -1233,40 +1326,39 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin, localpart == self.internal_name() or # exact match against the complete list address addr == listfullname): - return 1 + return True recips.append((addr, localpart)) - # - # helper function used to match a pattern against an address. Do it + # Helper function used to match a pattern against an address. def domatch(pattern, addr): try: - if re.match(pattern, addr): - return 1 + if re.match(pattern, addr, re.IGNORECASE): + return True except re.error: # The pattern is a malformed regexp -- try matching safely, # with all non-alphanumerics backslashed: - if re.match(re.escape(pattern), addr): - return 1 - # + if re.match(re.escape(pattern), addr, re.IGNORECASE): + return True + return False # Here's the current algorithm for matching acceptable_aliases: # # 1. If the pattern does not have an `@' in it, we first try matching # it against just the localpart. This was the behavior prior to - # 2.0beta3, and is kept for backwards compatibility. - # (deprecated). + # 2.0beta3, and is kept for backwards compatibility. (deprecated). # # 2. If that match fails, or the pattern does have an `@' in it, we # try matching against the entire recip address. + aliases = self.acceptable_aliases.splitlines() for addr, localpart in recips: - for alias in self.acceptable_aliases.split('\n'): + for alias in aliases: stripped = alias.strip() if not stripped: - # ignore blank or empty lines + # Ignore blank or empty lines continue if '@' not in stripped and domatch(stripped, localpart): - return 1 + return True if domatch(stripped, addr): - return 1 - return 0 + return True + return False def parse_matching_header_opt(self): """Return a list of triples [(field name, regex, line), ...].""" @@ -1313,13 +1405,17 @@ bad regexp in bounce_matching_header line: %s return line return 0 - def autorespondToSender(self, sender): + def autorespondToSender(self, sender, lang=None): """Return true if Mailman should auto-respond to this sender. This is only consulted for messages sent to the -request address, or for posting hold notifications, and serves only as a safety value for mail loops with email 'bots. """ + # language setting + if lang == None: + lang = self.preferred_language + i18n.set_language(lang) # No limit if mm_cfg.MAX_AUTORESPONSES_PER_DAY == 0: return 1 @@ -1347,11 +1443,12 @@ bad regexp in bounce_matching_header line: %s 'listname': '%s@%s' % (self.real_name, self.host_name), 'num' : count, 'owneremail': self.GetOwnerEmail(), - }) + }, + lang=lang) msg = Message.UserNotification( sender, self.GetOwnerEmail(), _('Last autoresponse notification for today'), - text) + text, lang=lang) msg.send(self) return 0 self.hold_and_cmd_autoresponses[sender] = (today, count+1) @@ -1370,4 +1467,6 @@ bad regexp in bounce_matching_header line: %s # preferred language. if mm_cfg.DEFAULT_SERVER_LANGUAGE not in langs: langs.append(mm_cfg.DEFAULT_SERVER_LANGUAGE) - return langs + # When testing, it's possible we've disabled a language, so just + # filter things out so we don't get tracebacks. + return [lang for lang in langs if mm_cfg.LC_DESCRIPTIONS.has_key(lang)] diff --git a/Mailman/Message.py b/Mailman/Message.py index 1058c64e0..d5fd30e74 100644 --- a/Mailman/Message.py +++ b/Mailman/Message.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2005 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License diff --git a/Mailman/Pending.py b/Mailman/Pending.py index 5ae520cb9..224565b57 100644 --- a/Mailman/Pending.py +++ b/Mailman/Pending.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2004 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -14,28 +14,16 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -""" Track pending confirmation of subscriptions. - -new(stuff...) places an item's data in the db, returning its cookie. - -confirmed(cookie) returns a tuple for the data, removing the item -from the db. It returns None if the cookie is not registered. -""" +"""Track pending actions which require confirmation.""" import os -import time import sha -import marshal -import cPickle -import random +import time import errno +import random +import cPickle from Mailman import mm_cfg -from Mailman import LockFile - -DBFILE = os.path.join(mm_cfg.DATA_DIR, 'pending.db') -PCKFILE = os.path.join(mm_cfg.DATA_DIR, 'pending.pck') -LOCKFILE = os.path.join(mm_cfg.LOCK_DIR, 'pending.lock') # Types of pending records SUBSCRIPTION = 'S' @@ -43,11 +31,12 @@ UNSUBSCRIPTION = 'U' CHANGE_OF_ADDRESS = 'C' HELD_MESSAGE = 'H' RE_ENABLE = 'E' +PROBE_BOUNCE = 'P' -_ALLKEYS = [(x,) for x in (SUBSCRIPTION, UNSUBSCRIPTION, - CHANGE_OF_ADDRESS, HELD_MESSAGE, - RE_ENABLE, - )] +_ALLKEYS = (SUBSCRIPTION, UNSUBSCRIPTION, + CHANGE_OF_ADDRESS, HELD_MESSAGE, + RE_ENABLE, PROBE_BOUNCE, + ) try: True, False @@ -56,213 +45,140 @@ except NameError: False = 0 - -def new(*content): - """Create a new entry in the pending database, returning cookie for it.""" - # It's a programming error if this assertion fails! We do it this way so - # the assert test won't fail if the sequence is empty. - assert content[:1] in _ALLKEYS - - # Get a lock handle now, but only lock inside the loop. - lock = LockFile.LockFile(LOCKFILE, - withlogging=mm_cfg.PENDINGDB_LOCK_DEBUGGING) - # We try the main loop several times. If we get a lock error somewhere - # (for instance because someone broke the lock) we simply try again. - retries = mm_cfg.PENDINGDB_LOCK_ATTEMPTS - try: - while retries: - retries -= 1 - if not lock.locked(): - try: - lock.lock(timeout=mm_cfg.PENDINGDB_LOCK_TIMEOUT) - except LockFile.TimeOutError: - continue - # Load the current database - db = _load() - # Calculate a unique cookie. Algorithm vetted by the Timbot. - # time() has high resolution on Linux, clock() on Windows. random - # gives us about 45 bits in Python 2.2, 53 bits on Python 2.3. - # The time and clock values basically help obscure the random - # number generator, as does the hash calculation. The integral - # parts of the time values are discarded because they're the most - # predictable bits. - while True: - now = time.time() - x = random.random() + now % 1.0 + time.clock() % 1.0 - hashfood = repr(x) - cookie = sha.new(hashfood).hexdigest() - # We'll never get a duplicate, but we'll be anal about - # checking anyway. - if not db.has_key(cookie): - break - # Store the content, plus the time in the future when this entry - # will be evicted from the database, due to staleness. - db[cookie] = content - evictions = db.setdefault('evictions', {}) - evictions[cookie] = now + mm_cfg.PENDING_REQUEST_LIFE - try: - _save(db, lock) - except LockFile.NotLockedError: - continue - return cookie - else: - # We failed to get the lock or keep it long enough to save the - # data! - raise LockFile.TimeOutError - finally: - if lock.locked(): - lock.unlock() +_missing = [] -def confirm(cookie, expunge=True): - """Return data for cookie, or None if not found. - - If optional expunge is True (the default), the record is also removed from - the database. - """ - if not expunge: - db = _load() - missing = [] - content = db.get(cookie, missing) - if content is missing: - return None - return content - - # Get a lock handle now, but only lock inside the loop. - lock = LockFile.LockFile(LOCKFILE, - withlogging=mm_cfg.PENDINGDB_LOCK_DEBUGGING) - # We try the main loop several times. If we get a lock error somewhere - # (for instance because someone broke the lock) we simply try again. - retries = mm_cfg.PENDINGDB_LOCK_ATTEMPTS - try: - while retries: - retries -= 1 - if not lock.locked(): - try: - lock.lock(timeout=mm_cfg.PENDINGDB_LOCK_TIMEOUT) - except LockFile.TimeOutError: - continue - # Load the database - db = _load() - missing = [] - content = db.get(cookie, missing) - if content is missing: - return None - del db[cookie] - del db['evictions'][cookie] - try: - _save(db, lock) - except LockFile.NotLockedError: - continue - return content - else: - # We failed to get the lock and keep it long enough to save the - # data! - raise LockFile.TimeOutError - finally: - if lock.locked(): - lock.unlock() +class Pending: + def InitTempVars(self): + self.__pendfile = os.path.join(self.fullpath(), 'pending.pck') + def pend_new(self, op, *content, **kws): + """Create a new entry in the pending database, returning cookie for it. + """ + assert op in _ALLKEYS, 'op: %s' % op + lifetime = kws.get('lifetime', mm_cfg.PENDING_REQUEST_LIFE) + # We try the main loop several times. If we get a lock error somewhere + # (for instance because someone broke the lock) we simply try again. + assert self.Locked() + # Load the database + db = self.__load() + # Calculate a unique cookie. Algorithm vetted by the Timbot. time() + # has high resolution on Linux, clock() on Windows. random gives us + # about 45 bits in Python 2.2, 53 bits on Python 2.3. The time and + # clock values basically help obscure the random number generator, as + # does the hash calculation. The integral parts of the time values + # are discarded because they're the most predictable bits. + while True: + now = time.time() + x = random.random() + now % 1.0 + time.clock() % 1.0 + cookie = sha.new(repr(x)).hexdigest() + # We'll never get a duplicate, but we'll be anal about checking + # anyway. + if not db.has_key(cookie): + break + # Store the content, plus the time in the future when this entry will + # be evicted from the database, due to staleness. + db[cookie] = (op,) + content + evictions = db.setdefault('evictions', {}) + evictions[cookie] = now + lifetime + self.__save(db) + return cookie - -def _load(): - # The list's lock must be acquired if you wish to alter data and save. - # - # First try to load the pickle file - fp = None - try: + def __load(self): try: - fp = open(PCKFILE) - return cPickle.load(fp) + fp = open(self.__pendfile) except IOError, e: if e.errno <> errno.ENOENT: raise - try: - # Try to load the old DBFILE - fp = open(DBFILE) - return marshal.load(fp) - except IOError, e: - if e.errno <> errno.ENOENT: raise - # Fresh pendings database - return {'evictions': {}} - finally: - if fp: + return {'evictions': {}} + try: + return cPickle.load(fp) + finally: fp.close() + def __save(self, db): + evictions = db['evictions'] + now = time.time() + for cookie, data in db.items(): + if cookie in ('evictions', 'version'): + continue + timestamp = evictions[cookie] + if now > timestamp: + # The entry is stale, so remove it. + del db[cookie] + del evictions[cookie] + # Clean out any bogus eviction entries. + for cookie in evictions.keys(): + if not db.has_key(cookie): + del evictions[cookie] + db['version'] = mm_cfg.PENDING_FILE_SCHEMA_VERSION + tmpfile = '%s.tmp.%d.%d' % (self.__pendfile, os.getpid(), now) + omask = os.umask(007) + try: + fp = open(tmpfile, 'w') + try: + cPickle.dump(db, fp) + fp.flush() + os.fsync(fp.fileno()) + finally: + fp.close() + os.rename(tmpfile, self.__pendfile) + finally: + os.umask(omask) -def _save(db, lock): - # Lock must be acquired before loading the data that is now being saved. - if not lock.locked(): - raise LockFile.NotLockedError - evictions = db['evictions'] - now = time.time() - for cookie, data in db.items(): - if cookie in ('evictions', 'version'): - continue - timestamp = evictions[cookie] - if now > timestamp: - # The entry is stale, so remove it. - del db[cookie] - del evictions[cookie] - # Clean out any bogus eviction entries. - for cookie in evictions.keys(): - if not db.has_key(cookie): - del evictions[cookie] - db['version'] = mm_cfg.PENDING_FILE_SCHEMA_VERSION - omask = os.umask(007) - # Always save this as a pickle (safely), and after that succeeds, blow - # away any old marshal file. - tmpfile = '%s.tmp.%d.%d' % (PCKFILE, os.getpid(), now) - fp = None - try: - fp = open(tmpfile, 'w') - cPickle.dump(db, fp) - fp.close() - fp = None - if not lock.locked(): - # Our lock was broken? - os.remove(tmpfile) - raise LockFile.NotLockedError - os.rename(tmpfile, PCKFILE) - if os.path.exists(DBFILE): - os.remove(DBFILE) - finally: - if fp: - fp.close() - os.umask(omask) + def pend_confirm(self, cookie, expunge=True): + """Return data for cookie, or None if not found. + + If optional expunge is True (the default), the record is also removed + from the database. + """ + db = self.__load() + # If we're not expunging, the database is read-only. + if not expunge: + return db.get(cookie) + # Since we're going to modify the database, we must make sure the list + # is locked, since it's the list lock that protects pending.pck. + assert self.Locked() + content = db.get(cookie, _missing) + if content is _missing: + return None + # Do the expunge + del db[cookie] + del db['evictions'][cookie] + self.__save(db) + return content + + def pend_repend(self, cookie, data, lifetime=mm_cfg.PENDING_REQUEST_LIFE): + assert self.Locked() + db = self.__load() + db[cookie] = data + db['evictions'][cookie] = time.time() + lifetime + self.__save(db) def _update(olddb): - # Update an old pending_subscriptions.db database to the new format - lock = LockFile.LockFile(LOCKFILE, - withlogging=mm_cfg.PENDINGDB_LOCK_DEBUGGING) - lock.lock(timeout=mm_cfg.PENDINGDB_LOCK_TIMEOUT) - try: - # We don't need this entry anymore - if olddb.has_key('lastculltime'): - del olddb['lastculltime'] - db = _load() - evictions = db.setdefault('evictions', {}) - for cookie, data in olddb.items(): - # The cookies used to be kept as a 6 digit integer. We now keep - # the cookies as a string (sha in our case, but it doesn't matter - # for cookie matching). - cookie = str(cookie) - # The old format kept the content as a tuple and tacked the - # timestamp on as the last element of the tuple. We keep the - # timestamps separate, but require the prepending of a record type - # indicator. We know that the only things that were kept in the - # old format were subscription requests. Also, the old request - # format didn't have the subscription language. Best we can do - # here is use the server default. - db[cookie] = (SUBSCRIPTION,) + data[:-1] + \ - (mm_cfg.DEFAULT_SERVER_LANGUAGE,) - # The old database format kept the timestamp as the time the - # request was made. The new format keeps it as the time the - # request should be evicted. - evictions[cookie] = data[-1] + mm_cfg.PENDING_REQUEST_LIFE - _save(db, lock) - finally: - if lock.locked(): - lock.unlock() + db = {} + # We don't need this entry anymore + if olddb.has_key('lastculltime'): + del olddb['lastculltime'] + evictions = db.setdefault('evictions', {}) + for cookie, data in olddb.items(): + # The cookies used to be kept as a 6 digit integer. We now keep the + # cookies as a string (sha in our case, but it doesn't matter for + # cookie matching). + cookie = str(cookie) + # The old format kept the content as a tuple and tacked the timestamp + # on as the last element of the tuple. We keep the timestamps + # separate, but require the prepending of a record type indicator. We + # know that the only things that were kept in the old format were + # subscription requests. Also, the old request format didn't have the + # subscription language. Best we can do here is use the server + # default. + db[cookie] = (SUBSCRIPTION,) + data[:-1] + \ + (mm_cfg.DEFAULT_SERVER_LANGUAGE,) + # The old database format kept the timestamp as the time the request + # was made. The new format keeps it as the time the request should be + # evicted. + evictions[cookie] = data[-1] + mm_cfg.PENDING_REQUEST_LIFE + return db diff --git a/Mailman/Queue/ArchRunner.py b/Mailman/Queue/ArchRunner.py index cbb49736b..0abb1d1b2 100644 --- a/Mailman/Queue/ArchRunner.py +++ b/Mailman/Queue/ArchRunner.py @@ -14,7 +14,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -"""Outgoing queue runner.""" +"""Archive queue runner.""" import time from email.Utils import parsedate_tz, mktime_tz, formatdate diff --git a/Mailman/Queue/BounceRunner.py b/Mailman/Queue/BounceRunner.py index 02c06e680..76a304708 100644 --- a/Mailman/Queue/BounceRunner.py +++ b/Mailman/Queue/BounceRunner.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2004 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -16,8 +16,10 @@ """Bounce queue runner.""" +import os import re import time +import cPickle from email.MIMEText import MIMEText from email.MIMEMessage import MIMEMessage @@ -35,22 +37,131 @@ from Mailman.i18n import _ COMMASPACE = ', ' -REGISTER_BOUNCES_EVERY = mm_cfg.minutes(15) +try: + True, False +except NameError: + True = 1 + False = 0 -class BounceRunner(Runner): +class BounceMixin: + def __init__(self): + # Registering a bounce means acquiring the list lock, and it would be + # too expensive to do this for each message. Instead, each bounce + # runner maintains an event log which is essentially a file with + # multiple pickles. Each bounce we receive gets appended to this file + # as a 4-tuple record: (listname, addr, today, msg) + # + # today is itself a 3-tuple of (year, month, day) + # + # Every once in a while (see _doperiodic()), the bounce runner cracks + # open the file, reads all the records and registers all the bounces. + # Then it truncates the file and continues on. We don't need to lock + # the bounce event file because bounce qrunners are single threaded + # and each creates a uniquely named file to contain the events. + # + # XXX When Python 2.3 is minimal require, we can use the new + # tempfile.TemporaryFile() function. + # + # XXX We used to classify bounces to the site list as bounce events + # for every list, but this caused severe problems. Here's the + # scenario: aperson@example.com is a member of 4 lists, and a list + # owner of the foo list. example.com has an aggressive spam filter + # which rejects any message that is spam or contains spam as an + # attachment. Now, a spambot sends a piece of spam to the foo list, + # but since that spambot is not a member, the list holds the message + # for approval, and sends a notification to aperson@example.com as + # list owner. That notification contains a copy of the spam. Now + # example.com rejects the message, causing a bounce to be sent to the + # site list's bounce address. The bounce runner would then dutifully + # register a bounce for all 4 lists that aperson@example.com was a + # member of, and eventually that person would get disabled on all + # their lists. So now we ignore site list bounces. Ce La Vie for + # password reminder bounces. + self._bounce_events_file = os.path.join( + mm_cfg.DATA_DIR, 'bounce-events-%05d.pck' % os.getpid()) + self._bounce_events_fp = None + self._bouncecnt = 0 + self._nextaction = time.time() + mm_cfg.REGISTER_BOUNCES_EVERY + + def _queue_bounces(self, listname, addrs, msg): + today = time.localtime()[:3] + if self._bounce_events_fp is None: + self._bounce_events_fp = open(self._bounce_events_file, 'a+b') + for addr in addrs: + cPickle.dump((listname, addr, today, msg), + self._bounce_events_fp, 1) + self._bounce_events_fp.flush() + os.fsync(self._bounce_events_fp.fileno()) + self._bouncecnt += len(addrs) + + def _register_bounces(self): + syslog('bounce', '%s processing %s queued bounces', + self, self._bouncecnt) + # Read all the records from the bounce file, then unlink it. Sort the + # records by listname for more efficient processing. + events = {} + self._bounce_events_fp.seek(0) + while True: + try: + listname, addr, day, msg = cPickle.load(self._bounce_events_fp) + except ValueError, e: + syslog('bounce', 'Error reading bounce events: %s', e) + except EOFError: + break + events.setdefault(listname, []).append((addr, day, msg)) + # Now register all events sorted by list + for listname in events.keys(): + mlist = self._open_list(listname) + mlist.Lock() + try: + for addr, day, msg in events[listname]: + mlist.registerBounce(addr, msg, day=day) + mlist.Save() + finally: + mlist.Unlock() + # Reset and free all the cached memory + self._bounce_events_fp.close() + self._bounce_events_fp = None + os.unlink(self._bounce_events_file) + self._bouncecnt = 0 + + def _cleanup(self): + if self._bouncecnt > 0: + self._register_bounces() + + def _doperiodic(self): + now = time.time() + if self._nextaction > now or self._bouncecnt == 0: + return + # Let's go ahead and register the bounces we've got stored up + self._nextaction = now + mm_cfg.REGISTER_BOUNCES_EVERY + self._register_bounces() + + def _probe_bounce(self, mlist, token): + locked = mlist.Locked() + if not locked: + mlist.Lock() + try: + op, addr, bmsg = mlist.pend_confirm(token) + info = mlist.getBounceInfo(addr) + mlist.disableBouncingMember(addr, info, bmsg) + # Only save the list if we're unlocking it + if not locked: + mlist.Save() + finally: + if not locked: + mlist.Unlock() + + + +class BounceRunner(Runner, BounceMixin): QDIR = mm_cfg.BOUNCEQUEUE_DIR def __init__(self, slice=None, numslices=1): Runner.__init__(self, slice, numslices) - # This is a simple sequence of bounce score events. Each entry in the - # list is a tuple of (address, day, msg) where day is a tuple of - # (YYYY, MM, DD). We'll sort and collate all this information in - # _register_bounces() below. - self._bounces = {} - self._bouncecnt = 0 - self._next_registration = time.time() + REGISTER_BOUNCES_EVERY + BounceMixin.__init__(self) def _dispose(self, mlist, msg, msgdata): # Make sure we have the most up-to-date state @@ -68,7 +179,7 @@ class BounceRunner(Runner): # All messages to list-owner@vdom.ain have their envelope sender set # to site-owner@dom.ain (no virtual domain). Is this a bounce for a # message to a list owner, coming to the site owner? - if msg.get('to', '') == Utils.get_site_email(extra='-owner'): + if msg.get('to', '') == Utils.get_site_email(extra='owner'): # Send it on to the site owners, but craft the envelope sender to # be the -loop detection address, so if /they/ bounce, we won't # get stuck in a bounce loop. @@ -82,8 +193,13 @@ class BounceRunner(Runner): # Try VERP detection first, since it's quick and easy addrs = verp_bounce(mlist, msg) if not addrs: + # See if this was a probe message. + token = verp_probe(mlist, msg) + if token: + self._probe_bounce(mlist, token) + return # That didn't give us anything useful, so try the old fashion - # bounce matching modules + # bounce matching modules. addrs = BouncerAPI.ScanMessages(mlist, msg) # If that still didn't return us any useful addresses, then send it on # or discard it. @@ -96,47 +212,12 @@ class BounceRunner(Runner): # although I'm unsure how that could happen. Possibly ScanMessages() # can let None's sneak through. In any event, this will kill them. addrs = filter(None, addrs) - # Store the bounce score events so we can register them periodically - today = time.localtime()[:3] - events = [(addr, today, msg) for addr in addrs] - self._bounces.setdefault(mlist.internal_name(), []).extend(events) - self._bouncecnt += len(addrs) + self._queue_bounces(mlist.internal_name(), addrs, msg) - def _doperiodic(self): - now = time.time() - if self._next_registration > now or not self._bounces: - return - # Let's go ahead and register the bounces we've got stored up - self._next_registration = now + REGISTER_BOUNCES_EVERY - self._register_bounces() - - def _register_bounces(self): - syslog('bounce', 'Processing %s queued bounces', self._bouncecnt) - # First, get the list of bounces register against the site list. For - # these addresses, we want to register a bounce on every list the - # address is a member of -- which we don't know yet. - sitebounces = self._bounces.get(mm_cfg.MAILMAN_SITE_LIST, []) - if sitebounces: - listnames = Utils.list_names() - else: - listnames = self._bounces.keys() - for listname in listnames: - mlist = self._open_list(listname) - mlist.Lock() - try: - events = self._bounces.get(listname, []) + sitebounces - for addr, day, msg in events: - mlist.registerBounce(addr, msg, day=day) - mlist.Save() - finally: - mlist.Unlock() - # Reset and free all the cached memory - self._bounces = {} - self._bouncecnt = 0 + _doperiodic = BounceMixin._doperiodic def _cleanup(self): - if self._bounces: - self._register_bounces() + BounceMixin._cleanup(self) Runner._cleanup(self) @@ -173,6 +254,41 @@ def verp_bounce(mlist, msg): +def verp_probe(mlist, msg): + bmailbox, bdomain = Utils.ParseEmail(mlist.GetBouncesEmail()) + # Sadly not every MTA bounces VERP messages correctly, or consistently. + # Fall back to Delivered-To: (Postfix), Envelope-To: (Exim) and + # Apparently-To:, and then short-circuit if we still don't have anything + # to work with. Note that there can be multiple Delivered-To: headers so + # we need to search them all (and we don't worry about false positives for + # forwarded email, because only one should match VERP_REGEXP). + vals = [] + for header in ('to', 'delivered-to', 'envelope-to', 'apparently-to'): + vals.extend(msg.get_all(header, [])) + for field in vals: + to = parseaddr(field)[1] + if not to: + continue # empty header + mo = re.search(mm_cfg.VERP_PROBE_REGEXP, to) + if not mo: + continue # no match of regexp + try: + if bmailbox <> mo.group('bounces'): + continue # not a bounce to our list + # Extract the token and see if there's an entry + token = mo.group('token') + data = mlist.pend_confirm(token, expunge=False) + if data is not None: + return token + except IndexError: + syslog( + 'error', + "VERP_PROBE_REGEXP doesn't yield the right match groups: %s", + mm_cfg.VERP_PROBE_REGEXP) + return None + + + def maybe_forward(mlist, msg): # Does the list owner want to get non-matching bounce messages? # If not, simply discard it. diff --git a/Mailman/Queue/CommandRunner.py b/Mailman/Queue/CommandRunner.py index 8967e2f65..e08d02eb5 100644 --- a/Mailman/Queue/CommandRunner.py +++ b/Mailman/Queue/CommandRunner.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2004 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -40,9 +40,10 @@ from Mailman.Logging.Syslog import syslog from Mailman import LockFile from email.Header import decode_header, make_header, Header +from email.Errors import HeaderParseError +from email.Iterators import typed_subpart_iterator from email.MIMEText import MIMEText from email.MIMEMessage import MIMEMessage -from email.Iterators import typed_subpart_iterator NL = '\n' @@ -72,9 +73,15 @@ class Results: # Extract the subject header and do RFC 2047 decoding. Note that # Python 2.1's unicode() builtin doesn't call obj.__unicode__(). subj = msg.get('subject', '') - subj = make_header(decode_header(subj)).__unicode__() - # Always process the Subject: header first - self.commands.append(subj) + try: + subj = make_header(decode_header(subj)).__unicode__() + # TK: Currently we don't allow 8bit or multibyte in mail command. + subj = subj.encode('us-ascii') + # Always process the Subject: header first + self.commands.append(subj) + except (HeaderParseError, UnicodeError, LookupError): + # We couldn't parse it so ignore the Subject header + pass # Find the first text/plain part part = None for part in typed_subpart_iterator(msg, 'text', 'plain'): @@ -163,7 +170,7 @@ To obtain instructions, send a message containing just the word "help". resp.append(_('\n- Done.\n\n')) # Encode any unicode strings into the list charset, so we don't try to # join unicode strings and invalid ASCII. - charset = Utils.GetCharSet(self.mlist.preferred_language) + charset = Utils.GetCharSet(self.msgdata['lang']) encoded_resp = [] for item in resp: if isinstance(item, UnicodeType): @@ -179,13 +186,13 @@ To obtain instructions, send a message containing just the word "help". # BAW: We wait until now to make this decision since our sender may # not be self.msg.get_sender(), but I'm not sure this is right. recip = self.returnaddr or self.msg.get_sender() - if not self.mlist.autorespondToSender(recip): + if not self.mlist.autorespondToSender(recip, self.msgdata['lang']): return msg = Message.UserNotification( recip, self.mlist.GetBouncesEmail(), _('The results of your email commands'), - lang=self.mlist.preferred_language) + lang=self.msgdata['lang']) msg.set_type('multipart/mixed') msg.attach(results) orig = MIMEMessage(self.msg) diff --git a/Mailman/Queue/NewsRunner.py b/Mailman/Queue/NewsRunner.py index cdfd5fead..448500633 100644 --- a/Mailman/Queue/NewsRunner.py +++ b/Mailman/Queue/NewsRunner.py @@ -1,4 +1,4 @@ -# Copyright (C) 2000-2003 by the Free Software Foundation, Inc. +# Copyright (C) 2000-2005 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -102,11 +102,14 @@ def prepare_message(mlist, msg, msgdata): del msg['approved'] msg['Approved'] = mlist.GetListEmail() # Should we restore the original, non-prefixed subject for gatewayed - # messages? - origsubj = msgdata.get('origsubj') - if not mlist.news_prefix_subject_too and origsubj is not None: + # messages? TK: We use stripped_subject (prefix stripped) which was + # crafted in CookHeaders.py to ensure prefix was stripped from the subject + # came from mailing list user. + stripped_subject = msgdata.get('stripped_subject') \ + or msgdata.get('origsubj') + if not mlist.news_prefix_subject_too and stripped_subject is not None: del msg['subject'] - msg['subject'] = origsubj + msg['subject'] = stripped_subject # Add the appropriate Newsgroups: header ngheader = msg['newsgroups'] if ngheader is not None: diff --git a/Mailman/Queue/OutgoingRunner.py b/Mailman/Queue/OutgoingRunner.py index 13dc2014a..001b68645 100644 --- a/Mailman/Queue/OutgoingRunner.py +++ b/Mailman/Queue/OutgoingRunner.py @@ -1,4 +1,4 @@ -# Copyright (C) 2000-2003 by the Free Software Foundation, Inc. +# Copyright (C) 2000-2004 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -30,6 +30,7 @@ from Mailman import Errors from Mailman import LockFile from Mailman.Queue.Runner import Runner from Mailman.Queue.Switchboard import Switchboard +from Mailman.Queue.BounceRunner import BounceMixin from Mailman.Logging.Syslog import syslog # This controls how often _doperiodic() will try to deal with deferred @@ -44,14 +45,12 @@ except NameError: -class OutgoingRunner(Runner): +class OutgoingRunner(Runner, BounceMixin): QDIR = mm_cfg.OUTQUEUE_DIR def __init__(self, slice=None, numslices=1): Runner.__init__(self, slice, numslices) - # Maps mailing lists to (recip, msg) tuples - self._permfailures = {} - self._permfail_counter = 0 + BounceMixin.__init__(self) # We look this function up only at startup time modname = 'Mailman.Handlers.' + mm_cfg.DELIVERY_MODULE mod = __import__(modname) @@ -63,6 +62,10 @@ class OutgoingRunner(Runner): self.__retryq = Switchboard(mm_cfg.RETRYQUEUE_DIR) def _dispose(self, mlist, msg, msgdata): + # See if we should retry delivery of this message again. + deliver_after = msgdata.get('deliver_after', 0) + if time.time() < deliver_after: + return True # Make sure we have the most up-to-date state mlist.Load() try: @@ -87,67 +90,46 @@ class OutgoingRunner(Runner): self.__logged = True return True except Errors.SomeRecipientsFailed, e: - # The delivery module being used (SMTPDirect or Sendmail) failed - # to deliver the message to one or all of the recipients. - # Permanent failures should be registered (but registration - # requires the list lock), and temporary failures should be - # retried later. - # - # For permanent failures, make a copy of the message for bounce - # handling. I'm not sure this is necessary, or the right thing to - # do. - if e.permfailures: - pcnt = len(e.permfailures) - msgcopy = copy.deepcopy(msg) - self._permfailures.setdefault(mlist, []).extend( - zip(e.permfailures, [msgcopy] * pcnt)) - # Move temporary failures to the qfiles/retry queue which will - # occasionally move them back here for another shot at delivery. - if e.tempfailures: - now = time.time() - recips = e.tempfailures - last_recip_count = msgdata.get('last_recip_count', 0) - deliver_until = msgdata.get('deliver_until', now) - if len(recips) == last_recip_count: - # We didn't make any progress, so don't attempt delivery - # any longer. BAW: is this the best disposition? - if now > deliver_until: - return False - else: - # Keep trying to delivery this message for a while - deliver_until = now + mm_cfg.DELIVERY_RETRY_PERIOD - msgdata['last_recip_count'] = len(recips) - msgdata['deliver_until'] = deliver_until - msgdata['recips'] = recips - self.__retryq.enqueue(msg, msgdata) + # Handle local rejects of probe messages differently. + if msgdata.get('probe_token'): + self._probe_bounce(mlist, msgdata['probe_token']) + else: + # Delivery failed at SMTP time for some or all of the + # recipients. Permanent failures are registered as bounces, + # but temporary failures are retried for later. + # + # BAW: msg is going to be the original message that failed + # delivery, not a bounce message. This may be confusing if + # this is what's sent to the user in the probe message. Maybe + # we should craft a bounce-like message containing information + # about the permanent SMTP failure? + self._queue_bounces(mlist.internal_name(), e.permfailures, msg) + # Move temporary failures to the qfiles/retry queue which will + # occasionally move them back here for another shot at + # delivery. + if e.tempfailures: + now = time.time() + recips = e.tempfailures + last_recip_count = msgdata.get('last_recip_count', 0) + deliver_until = msgdata.get('deliver_until', now) + if len(recips) == last_recip_count: + # We didn't make any progress, so don't attempt + # delivery any longer. BAW: is this the best + # disposition? + if now > deliver_until: + return False + else: + # Keep trying to delivery this message for a while + deliver_until = now + mm_cfg.DELIVERY_RETRY_PERIOD + msgdata['last_recip_count'] = len(recips) + msgdata['deliver_until'] = deliver_until + msgdata['recips'] = recips + self.__retryq.enqueue(msg, msgdata) # We've successfully completed handling of this message return False - def _doperiodic(self): - # Periodically try to acquire the list lock and clear out the - # permanent failures. - self._permfail_counter += 1 - if self._permfail_counter < DEAL_WITH_PERMFAILURES_EVERY: - return - self._handle_permfailures() - - def _handle_permfailures(self): - # Reset the counter - self._permfail_counter = 0 - # And deal with the deferred permanent failures. - for mlist in self._permfailures.keys(): - try: - mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT) - except LockFile.TimeOutError: - return - try: - for recip, msg in self._permfailures[mlist]: - mlist.registerBounce(recip, msg) - del self._permfailures[mlist] - mlist.Save() - finally: - mlist.Unlock() + _doperiodic = BounceMixin._doperiodic def _cleanup(self): - self._handle_permfailures() + BounceMixin._cleanup(self) Runner._cleanup(self) diff --git a/Mailman/Queue/Runner.py b/Mailman/Queue/Runner.py index d46ec8a1e..1e7854d7d 100644 --- a/Mailman/Queue/Runner.py +++ b/Mailman/Queue/Runner.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2004 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -31,6 +31,8 @@ from Mailman import i18n from Mailman.Queue.Switchboard import Switchboard from Mailman.Logging.Syslog import syslog +import email.Errors + try: True, False except NameError: @@ -52,6 +54,9 @@ class Runner: self._shunt = Switchboard(mm_cfg.SHUNTQUEUE_DIR) self._stop = False + def __repr__(self): + return '<%s at %s>' % (self.__class__.__name__, id(self)) + def stop(self): self._stop = True @@ -88,32 +93,34 @@ class Runner: # available for this qrunner to process. files = self._switchboard.files() for filebase in files: - # Ask the switchboard for the message and metadata objects - # associated with this filebase. - msg, msgdata = self._switchboard.dequeue(filebase) - # It's possible one or both files got lost. If so, just ignore - # this filebase entry. dequeue() will automatically unlink the - # other file, but we should log an error message for diagnostics. - if msg is None or msgdata is None: - syslog('error', 'lost data files for filebase: %s', filebase) - else: - # Now that we've dequeued the message, we want to be - # incredibly anal about making sure that no uncaught exception - # could cause us to lose the message. All runners that - # implement _dispose() must guarantee that exceptions are - # caught and dealt with properly. Still, there may be a bug - # in the infrastructure, and we do not want those to cause - # messages to be lost. Any uncaught exceptions will cause the - # message to be stored in the shunt queue for human + try: + # Ask the switchboard for the message and metadata objects + # associated with this filebase. + msg, msgdata = self._switchboard.dequeue(filebase) + except email.Errors.MessageParseError, e: + # It's possible to get here if the message was stored in the + # pickle in plain text, and the metadata had a _parsemsg key + # that was true, /and/ if the message had some bogosity in + # it. It's almost always going to be spam or bounced spam. + # There's not much we can do (and we didn't even get the + # metadata, so just log the exception and continue. + self._log(e) + syslog('error', 'Ignoring unparseable message: %s', filebase) + continue + try: + self._onefile(msg, msgdata) + except Exception, e: + # All runners that implement _dispose() must guarantee that + # exceptions are caught and dealt with properly. Still, there + # may be a bug in the infrastructure, and we do not want those + # to cause messages to be lost. Any uncaught exceptions will + # cause the message to be stored in the shunt queue for human # intervention. - try: - self._onefile(msg, msgdata) - except Exception, e: - self._log(e) - # Put a marker in the metadata for unshunting - msgdata['whichq'] = self._switchboard.whichq() - filebase = self._shunt.enqueue(msg, msgdata) - syslog('error', 'SHUNTING: %s', filebase) + self._log(e) + # Put a marker in the metadata for unshunting + msgdata['whichq'] = self._switchboard.whichq() + filebase = self._shunt.enqueue(msg, msgdata) + syslog('error', 'SHUNTING: %s', filebase) # Other work we want to do each time through the loop Utils.reap(self._kids, once=True) self._doperiodic() diff --git a/Mailman/Queue/Switchboard.py b/Mailman/Queue/Switchboard.py index bf0d8d409..52c430b0a 100644 --- a/Mailman/Queue/Switchboard.py +++ b/Mailman/Queue/Switchboard.py @@ -1,4 +1,4 @@ -# Copyright (C) 2001-2003 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2004 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -34,13 +34,12 @@ # needs. import os -import time import sha -import marshal +import time +import email import errno import cPickle - -import email +import marshal from Mailman import mm_cfg from Mailman import Utils @@ -63,7 +62,7 @@ SAVE_MSGS_AS_PICKLES = True -class _Switchboard: +class Switchboard: def __init__(self, whichq, slice=None, numslices=1): self.__whichq = whichq # Create the directory if it doesn't yet exist. @@ -97,11 +96,11 @@ class _Switchboard: # Get some data for the input to the sha hash now = time.time() if SAVE_MSGS_AS_PICKLES and not data.get('_plaintext'): - msgsave = cPickle.dumps(_msg, 1) - ext = '.pck' + protocol = 1 + msgsave = cPickle.dumps(_msg, protocol) else: - msgsave = str(_msg) - ext = '.msg' + protocol = 0 + msgsave = cPickle.dumps(str(_msg), protocol) hashfood = msgsave + listname + `now` # Encode the current time into the file name for FIFO sorting in # files(). The file name consists of two parts separated by a `+': @@ -110,92 +109,46 @@ class _Switchboard: #rcvtime = data.setdefault('received_time', now) rcvtime = data.setdefault('received_time', now) filebase = `rcvtime` + '+' + sha.new(hashfood).hexdigest() - # Figure out which queue files the message is to be written to. - msgfile = os.path.join(self.__whichq, filebase + ext) - dbfile = os.path.join(self.__whichq, filebase + '.db') + filename = os.path.join(self.__whichq, filebase + '.pck') + tmpfile = filename + '.tmp' # Always add the metadata schema version number data['version'] = mm_cfg.QFILE_SCHEMA_VERSION # Filter out volatile entries for k in data.keys(): - if k[0] == '_': + if k.startswith('_'): del data[k] - # Now write the message text to one file and the metadata to another - # file. The metadata is always written second to avoid race - # conditions with the various queue runners (which key off of the .db - # filename). + # We have to tell the dequeue() method whether to parse the message + # object or not. + data['_parsemsg'] = (protocol == 0) + # Write to the pickle file the message object and metadata. omask = os.umask(007) # -rw-rw---- try: - msgfp = open(msgfile, 'w') + fp = open(tmpfile, 'w') + try: + fp.write(msgsave) + cPickle.dump(data, fp, protocol) + fp.flush() + os.fsync(fp.fileno()) + finally: + fp.close() finally: os.umask(omask) - msgfp.write(msgsave) - msgfp.close() - # Now write the metadata using the appropriate external metadata - # format. We play rename-switcheroo here to further plug the race - # condition holes. - tmpfile = dbfile + '.tmp' - self._ext_write(tmpfile, data) - os.rename(tmpfile, dbfile) + os.rename(tmpfile, filename) return filebase def dequeue(self, filebase): - # Calculate the .db and .msg filenames from the given filebase. - msgfile = os.path.join(self.__whichq, filebase + '.msg') - pckfile = os.path.join(self.__whichq, filebase + '.pck') - dbfile = os.path.join(self.__whichq, filebase + '.db') - # Now we are going to read the message and metadata for the given - # filebase. We want to read things in this order: first, the metadata - # file to find out whether the message is stored as a pickle or as - # plain text. Second, the actual message file. However, we want to - # first unlink the message file and then the .db file, because the - # qrunner only cues off of the .db file - msg = None - try: - data = self._ext_read(dbfile) - os.unlink(dbfile) - except EnvironmentError, e: - if e.errno <> errno.ENOENT: raise - data = {} - # Between 2.1b4 and 2.1b5, the `rejection-notice' key in the metadata - # was renamed to `rejection_notice', since dashes in the keys are not - # supported in METAFMT_ASCII. - if data.has_key('rejection-notice'): - data['rejection_notice'] = data['rejection-notice'] - del data['rejection-notice'] - msgfp = None + # Calculate the filename from the given filebase. + filename = os.path.join(self.__whichq, filebase + '.pck') + # Read the message object and metadata. + fp = open(filename) + os.unlink(filename) try: - try: - msgfp = open(pckfile) - msg = cPickle.load(msgfp) - os.unlink(pckfile) - except EnvironmentError, e: - if e.errno <> errno.ENOENT: raise - msgfp = None - try: - msgfp = open(msgfile) - msg = email.message_from_file(msgfp, Message.Message) - os.unlink(msgfile) - except EnvironmentError, e: - if e.errno <> errno.ENOENT: raise - except email.Errors.MessageParseError, e: - # This message was unparsable, most likely because its - # MIME encapsulation was broken. For now, there's not - # much we can do about it. - syslog('error', 'message is unparsable: %s', filebase) - msgfp.close() - msgfp = None - if mm_cfg.QRUNNER_SAVE_BAD_MESSAGES: - # Cheapo way to ensure the directory exists w/ the - # proper permissions. - sb = Switchboard(mm_cfg.BADQUEUE_DIR) - os.rename(msgfile, os.path.join( - mm_cfg.BADQUEUE_DIR, filebase + '.txt')) - else: - os.unlink(msgfile) - msg = data = None + msg = cPickle.load(fp) + data = cPickle.load(fp) finally: - if msgfp: - msgfp.close() + fp.close() + if data.get('_parsemsg'): + msg = email.message_from_string(msg, Message.Message) return msg, data def files(self): @@ -203,157 +156,17 @@ class _Switchboard: lower = self.__lower upper = self.__upper for f in os.listdir(self.__whichq): - # We only care about the file's base name (i.e. no extension). - # Thus we'll ignore anything that doesn't end in .db. - if not f.endswith('.db'): + # By ignoring anything that doesn't end in .pck, we ignore + # tempfiles and avoid a race condition. + if not f.endswith('.pck'): continue filebase = os.path.splitext(f)[0] when, digest = filebase.split('+') # Throw out any files which don't match our bitrange. BAW: test # performance and end-cases of this algorithm. - if not lower or (lower <= long(digest, 16) < upper): + if lower is None or (lower <= long(digest, 16) < upper): times[float(when)] = filebase # FIFO sort keys = times.keys() keys.sort() return [times[k] for k in keys] - - def _ext_write(self, tmpfile, data): - raise NotImplementedError - - def _ext_read(self, dbfile): - raise NotImplementedError - - - -class MarshalSwitchboard(_Switchboard): - """Python marshal format.""" - FLOAT_ATTRIBUTES = ['received_time'] - - def _ext_write(self, filename, dict): - omask = os.umask(007) # -rw-rw---- - try: - fp = open(filename, 'w') - finally: - os.umask(omask) - # Python's marshal, up to and including in Python 2.1, has a bug where - # the full precision of floats was not stored. We work around this - # bug by hardcoding a list of float values we know about, repr()-izing - # them ourselves, and doing the reverse conversion on _ext_read(). - for attr in self.FLOAT_ATTRIBUTES: - # We use try/except because we expect a hitrate of nearly 100% - try: - fval = dict[attr] - except KeyError: - pass - else: - dict[attr] = repr(fval) - marshal.dump(dict, fp) - # Make damn sure that the data we just wrote gets flushed to disk - fp.flush() - if mm_cfg.SYNC_AFTER_WRITE: - os.fsync(fp.fileno()) - fp.close() - - def _ext_read(self, filename): - fp = open(filename) - dict = marshal.load(fp) - # Update from version 2 files - if dict.get('version', 0) == 2: - del dict['filebase'] - # Do the reverse conversion (repr -> float) - for attr in self.FLOAT_ATTRIBUTES: - try: - sval = dict[attr] - except KeyError: - pass - else: - # Do a safe eval by setting up a restricted execution - # environment. This may not be strictly necessary since we - # know they are floats, but it can't hurt. - dict[attr] = eval(sval, {'__builtins__': {}}) - fp.close() - return dict - - - -class BSDDBSwitchboard(_Switchboard): - """Native (i.e. compiled-in) Berkeley db format.""" - def _ext_write(self, filename, dict): - import bsddb - omask = os.umask(0) - try: - hashfile = bsddb.hashopen(filename, 'n', 0660) - finally: - os.umask(omask) - # values must be strings - for k, v in dict.items(): - hashfile[k] = marshal.dumps(v) - hashfile.sync() - hashfile.close() - - def _ext_read(self, filename): - import bsddb - dict = {} - hashfile = bsddb.hashopen(filename, 'r') - for k in hashfile.keys(): - dict[k] = marshal.loads(hashfile[k]) - hashfile.close() - return dict - - - -class ASCIISwitchboard(_Switchboard): - """Human readable .db file format. - - key/value pairs are written as - - key = value - - as real Python code which can be execfile'd. - """ - - def _ext_write(self, filename, dict): - omask = os.umask(007) # -rw-rw---- - try: - fp = open(filename, 'w') - finally: - os.umask(omask) - for k, v in dict.items(): - print >> fp, '%s = %s' % (k, repr(v)) - # Make damn sure that the data we just wrote gets flushed to disk - fp.flush() - if mm_cfg.SYNC_AFTER_WRITE: - os.fsync(fp.fileno()) - fp.close() - - def _ext_read(self, filename): - dict = {'__builtins__': {}} - execfile(filename, dict) - del dict['__builtins__'] - return dict - - - -# Here are the various types of external file formats available. The format -# chosen is given defined in the mm_cfg.py configuration file. -if mm_cfg.METADATA_FORMAT == mm_cfg.METAFMT_MARSHAL: - Switchboard = MarshalSwitchboard -elif mm_cfg.METADATA_FORMAT == mm_cfg.METAFMT_BSDDB_NATIVE: - Switchboard = BSDDBSwitchboard -elif mm_cfg.METADATA_FORMAT == mm_cfg.METAFMT_ASCII: - Switchboard = ASCIISwitchboard -else: - syslog('error', 'Undefined metadata format: %d (using marshals)', - mm_cfg.METADATA_FORMAT) - Switchboard = MarshalSwitchboard - - - -# For bin/dumpdb -class DumperSwitchboard(Switchboard): - def __init__(self): - pass - - def read(self, filename): - return self._ext_read(filename) diff --git a/Mailman/SecurityManager.py b/Mailman/SecurityManager.py index 9454ce610..ba65fec28 100644 --- a/Mailman/SecurityManager.py +++ b/Mailman/SecurityManager.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2004 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -53,6 +53,7 @@ import time import Cookie import marshal import binascii +import urllib from types import StringType, TupleType from urlparse import urlparse @@ -67,6 +68,12 @@ from Mailman import Utils from Mailman import Errors from Mailman.Logging.Syslog import syslog +try: + True, False +except NameError: + True = 1 + False = 0 + class SecurityManager: @@ -97,7 +104,8 @@ class SecurityManager: # A bad system error raise TypeError, 'No user supplied for AuthUser context' secret = self.getMemberPassword(user) - key += 'user+%s' % Utils.ObscureEmail(user) + userdata = urllib.quote(Utils.ObscureEmail(user), safe='') + key += 'user+%s' % userdata elif authcontext == mm_cfg.AuthListModerator: secret = self.mod_password key += 'moderator' @@ -143,15 +151,15 @@ class SecurityManager: try: salt = secret[:2] if crypt and crypt.crypt(response, salt) == secret: - return 1 - return 0 + return True + return False except TypeError: # BAW: Hard to say why we can get a TypeError here. # SF bug report #585776 says crypt.crypt() can raise # this if salt contains null bytes, although I don't # know how that can happen (perhaps if a MM2.0 list # with USE_CRYPT = 0 has been updated? Doubtful. - return 0 + return False # The password for the list admin and list moderator are not # kept as plain text, but instead as an sha hexdigest. The # response being passed in is plain text, so we need to @@ -163,20 +171,18 @@ class SecurityManager: if secret is None: continue sharesponse = sha.new(response).hexdigest() - upgrade = ok = 0 + upgrade = ok = False if sharesponse == secret: - ok = 1 + ok = True elif md5.new(response).digest() == secret: - ok = 1 - upgrade = 1 + ok = upgrade = True elif cryptmatchp(response, secret): - ok = 1 - upgrade = 1 + ok = upgrade = True if upgrade: - save_and_unlock = 0 + save_and_unlock = False if not self.Locked(): self.Lock() - save_and_unlock = 1 + save_and_unlock = True try: self.password = sharesponse if save_and_unlock: @@ -192,8 +198,12 @@ class SecurityManager: if secret and sha.new(response).hexdigest() == secret: return ac elif ac == mm_cfg.AuthUser: - if self.authenticateMember(user, response): - return ac + if user is not None: + try: + if self.authenticateMember(user, response): + return ac + except Errors.NotAMemberError: + pass else: # What is this context??? syslog('error', 'Bad authcontext: %s', ac) @@ -205,22 +215,19 @@ class SecurityManager: # contains a matching authorization, falling back to checking whether # the response matches one of the passwords. authcontexts must be a # sequence, and if it contains the context AuthUser, then the user - # argument must not be None. + # argument should not be None. # # Returns a flag indicating whether authentication succeeded or not. - try: - for ac in authcontexts: - ok = self.CheckCookie(ac, user) - if ok: - return 1 - # Check passwords - ac = self.Authenticate(authcontexts, response, user) - if ac: - print self.MakeCookie(ac, user) - return 1 - except Errors.NotAMemberError: - pass - return 0 + for ac in authcontexts: + ok = self.CheckCookie(ac, user) + if ok: + return True + # Check passwords + ac = self.Authenticate(authcontexts, response, user) + if ac: + print self.MakeCookie(ac, user) + return True + return False def MakeCookie(self, authcontext, user=None): key, secret = self.AuthContextInfo(authcontext, user) @@ -269,7 +276,7 @@ class SecurityManager: # authentication. cookiedata = os.environ.get('HTTP_COOKIE') if not cookiedata: - return 0 + return False # We can't use the Cookie module here because it isn't liberal in what # it accepts. Feed it a MM2.0 cookie along with a MM2.1 cookie and # you get a CookieError. :(. All we care about is accessing the @@ -294,17 +301,20 @@ class SecurityManager: for user in [Utils.UnobscureEmail(u) for u in usernames]: ok = self.__checkone(c, authcontext, user) if ok: - return 1 - return 0 + return True + return False else: return self.__checkone(c, authcontext, user) def __checkone(self, c, authcontext, user): # Do the guts of the cookie check, for one authcontext/user # combination. - key, secret = self.AuthContextInfo(authcontext, user) + try: + key, secret = self.AuthContextInfo(authcontext, user) + except Errors.NotAMemberError: + return False if not c.has_key(key) or not isinstance(secret, StringType): - return 0 + return False # Undo the encoding we performed in MakeCookie() above. BAW: I # believe this is safe from exploit because marshal can't be forced to # load recursive data structures, and it can't be forced to execute @@ -318,18 +328,18 @@ class SecurityManager: data = marshal.loads(binascii.unhexlify(c[key])) issued, received_mac = data except (EOFError, ValueError, TypeError, KeyError): - return 0 + return False # Make sure the issued timestamp makes sense now = time.time() if now < issued: - return 0 + return False # Calculate what the mac ought to be based on the cookie's timestamp # and the shared secret. mac = sha.new(secret + `issued`).hexdigest() if mac <> received_mac: - return 0 + return False # Authenticated! - return 1 + return True diff --git a/Mailman/Site.py b/Mailman/Site.py index 33e1fdefe..2a1025aef 100644 --- a/Mailman/Site.py +++ b/Mailman/Site.py @@ -1,4 +1,4 @@ -# Copyright (C) 2002 by the Free Software Foundation, Inc. +# Copyright (C) 2002-2003 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -25,6 +25,12 @@ import errno from Mailman import mm_cfg +try: + True, False +except NameError: + True = 1 + False = 0 + def _makedir(path): @@ -63,7 +69,7 @@ def get_listpath(listname, domain=None, create=0): # BAW: We don't really support domain<>None yet. This will be added in a # future version. By default, Mailman will never pass in a domain argument. -def get_archpath(listname, domain=None, create=0, public=0): +def get_archpath(listname, domain=None, create=False, public=False): """Return the file system path to the list's archive directory for the named list in the named virtual domain. diff --git a/Mailman/UserDesc.py b/Mailman/UserDesc.py index 118cf4da3..ec7db748e 100644 --- a/Mailman/UserDesc.py +++ b/Mailman/UserDesc.py @@ -1,21 +1,26 @@ -# Copyright (C) 2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 2001-2004 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software +# along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. """User description class/structure, for ApprovedAddMember and friends.""" + +from types import UnicodeType + + + class UserDesc: def __init__(self, address=None, fullname=None, password=None, digest=None, lang=None): @@ -53,5 +58,10 @@ class UserDesc: elif digest == 1: digest = 'yes' language = getattr(self, 'language', 'n/a') + # Make sure fullname and password are encoded if they're strings + if isinstance(fullname, UnicodeType): + fullname = fullname.encode('ascii', 'replace') + if isinstance(password, UnicodeType): + password = password.encode('ascii', 'replace') return '<UserDesc %s (%s) [%s] [digest? %s] [%s]>' % ( address, fullname, password, digest, language) diff --git a/Mailman/Utils.py b/Mailman/Utils.py index 0e550c13f..3092db41a 100644 --- a/Mailman/Utils.py +++ b/Mailman/Utils.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2005 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -27,14 +27,17 @@ from __future__ import nested_scopes import os import re -import random -import urlparse +import cgi import sha -import errno import time -import cgi +import errno +import base64 +import random +import urlparse import htmlentitydefs +import email.Header import email.Iterators +from email.Errors import HeaderParseError from types import UnicodeType from string import whitespace, digits try: @@ -57,6 +60,7 @@ except NameError: False = 0 EMPTYSTRING = '' +UEMPTYSTRING = u'' NL = '\n' DOT = '.' IDENTCHARS = ascii_letters + digits + '_' @@ -196,7 +200,7 @@ def LCDomain(addr): # TBD: what other characters should be disallowed? -_badchars = re.compile(r'[][()<>|;^,/\200-\377]') +_badchars = re.compile(r'[][()<>|;^,\000-\037\177-\377]') def ValidateEmail(s): """Verify that the an email address isn't grossly evil.""" @@ -295,12 +299,53 @@ for v in _vowels: _syllables.append(v+c) del c, v -def MakeRandomPassword(length=6): +def UserFriendly_MakeRandomPassword(length): syls = [] while len(syls) * 2 < length: syls.append(random.choice(_syllables)) return EMPTYSTRING.join(syls)[:length] + +def Secure_MakeRandomPassword(length): + bytesread = 0 + bytes = [] + fd = None + try: + while bytesread < length: + try: + # Python 2.4 has this on available systems. + newbytes = os.urandom(length - bytesread) + except (AttributeError, NotImplementedError): + if fd is None: + try: + fd = os.open('/dev/urandom', os.O_RDONLY) + except OSError, e: + if e.errno <> errno.ENOENT: + raise + # We have no available source of cryptographically + # secure random characters. Log an error and fallback + # to the user friendly passwords. + from Mailman.Logging.Syslog import syslog + syslog('error', + 'urandom not available, passwords not secure') + return UserFriendly_MakeRandomPassword(length) + newbytes = os.read(fd, length - bytesread) + bytes.append(newbytes) + bytesread += len(newbytes) + s = base64.encodestring(EMPTYSTRING.join(bytes)) + # base64 will expand the string by 4/3rds + return s.replace('\n', '')[:length] + finally: + if fd is not None: + os.close(fd) + + +def MakeRandomPassword(length=mm_cfg.MEMBER_PASSWORD_LENGTH): + if mm_cfg.USER_FRIENDLY_PASSWORDS: + return UserFriendly_MakeRandomPassword(length) + return Secure_MakeRandomPassword(length) + + def GetRandomSeed(): chr1 = int(random.random() * 52) chr2 = int(random.random() * 52) @@ -355,7 +400,7 @@ def check_global_password(response, siteadmin=True): def websafe(s): - return cgi.escape(s, quote=1) + return cgi.escape(s, quote=True) def nntpsplit(s): @@ -697,7 +742,7 @@ def to_dollar(s): def to_percent(s): """Convert from $-strings to %-strings.""" - s = s.replace('%', '%%') + s = s.replace('%', '%%').replace('$$', '$') parts = dre.split(s) for i in range(1, len(parts), 4): if parts[i] is not None: @@ -794,6 +839,7 @@ def uncanonstr(s, lang=None): # Nope, it contains funny characters, so html-ref it return uquote(s) + def uquote(s): a = [] for c in s: @@ -804,3 +850,15 @@ def uquote(s): a.append(c) # Join characters together and coerce to byte string return str(EMPTYSTRING.join(a)) + + +def oneline(s, cset): + # Decode header string in one line and convert into specified charset + try: + h = email.Header.make_header(email.Header.decode_header(s)) + ustr = h.__unicode__() + line = UEMPTYSTRING.join(ustr.splitlines()) + return line.encode(cset, 'replace') + except (LookupError, UnicodeError, ValueError, HeaderParseError): + # possibly charset problem. return with undecoded string in one line. + return EMPTYSTRING.join(s.splitlines()) diff --git a/Mailman/Version.py b/Mailman/Version.py index 86e300f03..5ea601df5 100644 --- a/Mailman/Version.py +++ b/Mailman/Version.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998-2003 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2005 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -15,7 +15,7 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # Mailman version -VERSION = "2.2a0" +VERSION = "2.2.0a0" # And as a hex number in the manner of PY_VERSION_HEX ALPHA = 0xa @@ -36,13 +36,13 @@ HEX_VERSION = ((MAJOR_REV << 24) | (MINOR_REV << 16) | (MICRO_REV << 8) | (REL_LEVEL << 4) | (REL_SERIAL << 0)) # config.pck schema version number -DATA_FILE_VERSION = 88 +DATA_FILE_VERSION = 96 # qfile/*.db schema version number QFILE_SCHEMA_VERSION = 3 -# version number for the data/pending.db file schema -PENDING_FILE_SCHEMA_VERSION = 1 +# version number for the lists/<listname>/pending.db file schema +PENDING_FILE_SCHEMA_VERSION = 2 # version number for the lists/<listname>/request.db file schema REQUESTS_FILE_SCHEMA_VERSION = 1 diff --git a/Mailman/htmlformat.py b/Mailman/htmlformat.py index 7e21b43d3..1fe44d885 100644 --- a/Mailman/htmlformat.py +++ b/Mailman/htmlformat.py @@ -1,17 +1,17 @@ -# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2005 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software +# along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. @@ -84,7 +84,7 @@ class Table: # Add a new blank cell at the end def NewCell(self): self.cells[-1].append('') - + def AddRow(self, row): self.cells.append(row) @@ -99,7 +99,7 @@ class Table: DictMerge(self.cell_info[row], kws) else: self.cell_info[row][col] = kws - + def AddRowInfo(self, row, **kws): kws = CaseInsensitiveKeyedDict(kws) if not self.row_info.has_key(row): @@ -110,7 +110,7 @@ class Table: # What's the index for the row we just put in? def GetCurrentRowIndex(self): return len(self.cells)-1 - + # What's the index for the col we just put in? def GetCurrentCellIndex(self): return len(self.cells[-1])-1 @@ -154,7 +154,7 @@ class Table: if key == 'border' and val == None: output = output + ' BORDER' continue - else: + else: output = output + ' %s="%s"' % (key.upper(), val) return output @@ -202,14 +202,14 @@ class Table: output = output + '\n' + ' '*indent + '</table>\n' return output - + class Link: def __init__(self, href, text, target=None): self.href = href self.text = text self.target = target - + def Format(self, indent=0): texpr = "" if self.target != None: @@ -223,7 +223,7 @@ class FontSize: def __init__(self, size, *items): self.items = list(items) self.size = size - + def Format(self, indent=0): output = '<font size="%s">' % self.size for item in self.items: @@ -236,7 +236,7 @@ class FontAttr: def __init__(self, *items, **kw): self.items = list(items) self.attrs = kw - + def Format(self, indent=0): seq = [] for k, v in self.attrs.items(): @@ -334,18 +334,18 @@ class Document(Container): output.append('%s</HTML>' % tab) return NL.join(output) - def addError(self, errmsg, tag=None, *args): + def addError(self, errmsg, tag=None): if tag is None: tag = _('Error: ') self.AddItem(Header(3, Bold(FontAttr( _(tag), color=mm_cfg.WEB_ERROR_COLOR, size='+2')).Format() + - Italic(errmsg % args).Format())) + Italic(errmsg).Format())) class HeadlessDocument(Document): """Document without head section, for templates that provide their own.""" suppress_head = 1 - + class StdContainer(Container): def Format(self, indent=0): @@ -353,7 +353,7 @@ class StdContainer(Container): output = '<%s>' % self.tag output = output + Container.Format(self, indent) output = '%s</%s>' % (output, self.tag) - return output + return output class QuotedContainer(Container): @@ -375,9 +375,9 @@ class Address(StdContainer): class Underline(StdContainer): tag = 'u' - + class Bold(StdContainer): - tag = 'strong' + tag = 'strong' class Italic(StdContainer): tag = 'em' @@ -494,7 +494,7 @@ class VerticalSpacer: self.size = size def Format(self, indent=0): output = '<spacer type="vertical" height="%d">' % self.size - return output + return output class WidgetArray: Widget = None @@ -578,7 +578,7 @@ class OrderedList(Container): (spaces, HTMLFormatObject(item, indent + 2)) output = output + '%s</ol>\n' % spaces return output - + class DefinitionList(Container): def Format(self, indent=0): spaces = ' ' * indent @@ -632,14 +632,14 @@ def MailmanLogo(): class SelectOptions: - def __init__(self, varname, values, legend, + def __init__(self, varname, values, legend, selected=0, size=1, multiple=None): self.varname = varname self.values = values self.legend = legend self.size = size self.multiple = multiple - # we convert any type to tuple, commas are needed + # we convert any type to tuple, commas are needed if not multiple: if type(selected) == types.IntType: self.selected = (selected,) @@ -649,7 +649,7 @@ class SelectOptions: self.selected = (selected[0],) else: self.selected = (0,) - + def Format(self, indent=0): spaces = " " * indent items = min( len(self.values), len(self.legend) ) @@ -657,22 +657,22 @@ class SelectOptions: # jcrey: If there is no argument, we return nothing to avoid errors if items == 0: return "" - + text = "\n" + spaces + "<Select name=\"%s\"" % self.varname if self.size > 1: text = text + " size=%d" % self.size if self.multiple: text = text + " multiple" text = text + ">\n" - + for i in range(items): if i in self.selected: checked = " Selected" else: checked = "" - + opt = " <option value=\"%s\"%s> %s </option>" % ( self.values[i], checked, self.legend[i]) text = text + spaces + opt + "\n" - + return text + spaces + '</Select>' diff --git a/Mailman/i18n.py b/Mailman/i18n.py index c102b68cb..810f15663 100644 --- a/Mailman/i18n.py +++ b/Mailman/i18n.py @@ -108,7 +108,7 @@ def ctime(date): try: year, mon, day, hh, mm, ss, wday, ydat, dst = time.strptime(date) tzname = time.tzname[dst and 1 or 0] - except ValueError: + except (ValueError, AttributeError): try: wday, mon, day, hms, year = date.split() hh, mm, ss = hms.split(':') diff --git a/Mailman/versions.py b/Mailman/versions.py index 594a783fe..9d19e8df5 100644 --- a/Mailman/versions.py +++ b/Mailman/versions.py @@ -1,4 +1,4 @@ -# Copyright (C) 1998,1999,2000,2001,2002 by the Free Software Foundation, Inc. +# Copyright (C) 1998-2005 by the Free Software Foundation, Inc. # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License @@ -393,6 +393,19 @@ def NewVars(l): encode = 2 add_only_if_missing('encode_ascii_prefixes', encode) add_only_if_missing('news_moderation', 0) + add_only_if_missing('header_filter_rules', []) + # Scrubber in regular delivery + add_only_if_missing('scrub_nondigest', 0) + # ContentFilter by file extensions + add_only_if_missing('filter_filename_extensions', + mm_cfg.DEFAULT_FILTER_FILENAME_EXTENSIONS) + add_only_if_missing('pass_filename_extensions', []) + # automatic discard + add_only_if_missing('max_days_to_hold', 0) + add_only_if_missing('nonmember_rejection_notice', '') + # multipart/alternative collapse + add_only_if_missing('collapse_alternatives', + mm_cfg.DEFAULT_COLLAPSE_ALTERNATIVES) @@ -487,7 +500,8 @@ def NewRequestsDatabase(l): # # See the note above; the same holds true. for ign, ign, digest, addr, password in v: - l.HoldSubscription(addr, password, digest) + l.HoldSubscription(addr, '', password, digest, + mm_cfg.DEFAULT_SERVER_LANGUAGE) del r[k] else: syslog('error', """\ |