diff options
Diffstat (limited to 'Mailman/Handlers')
| -rw-r--r-- | Mailman/Handlers/Acknowledge.py | 65 | ||||
| -rw-r--r-- | Mailman/Handlers/AfterDelivery.py | 29 | ||||
| -rw-r--r-- | Mailman/Handlers/AvoidDuplicates.py | 93 | ||||
| -rw-r--r-- | Mailman/Handlers/CalcRecips.py | 128 | ||||
| -rw-r--r-- | Mailman/Handlers/Cleanse.py | 58 | ||||
| -rw-r--r-- | Mailman/Handlers/CleanseDKIM.py | 36 | ||||
| -rw-r--r-- | Mailman/Handlers/CookHeaders.py | 338 | ||||
| -rw-r--r-- | Mailman/Handlers/Decorate.py | 207 | ||||
| -rw-r--r-- | Mailman/Handlers/FileRecips.py | 46 | ||||
| -rw-r--r-- | Mailman/Handlers/MimeDel.py | 261 | ||||
| -rw-r--r-- | Mailman/Handlers/Moderate.py | 168 | ||||
| -rw-r--r-- | Mailman/Handlers/OwnerRecips.py | 27 | ||||
| -rw-r--r-- | Mailman/Handlers/Replybot.py | 111 | ||||
| -rw-r--r-- | Mailman/Handlers/SMTPDirect.py | 389 | ||||
| -rw-r--r-- | Mailman/Handlers/Scrubber.py | 500 | ||||
| -rw-r--r-- | Mailman/Handlers/Tagger.py | 157 | ||||
| -rw-r--r-- | Mailman/Handlers/ToArchive.py | 37 | ||||
| -rw-r--r-- | Mailman/Handlers/ToDigest.py | 420 | ||||
| -rw-r--r-- | Mailman/Handlers/ToOutgoing.py | 57 | ||||
| -rw-r--r-- | Mailman/Handlers/ToUsenet.py | 49 | ||||
| -rw-r--r-- | Mailman/Handlers/__init__.py | 0 |
21 files changed, 0 insertions, 3176 deletions
diff --git a/Mailman/Handlers/Acknowledge.py b/Mailman/Handlers/Acknowledge.py deleted file mode 100644 index 078c3ac92..000000000 --- a/Mailman/Handlers/Acknowledge.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright (C) 1998-2007 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""Send an acknowledgement of the successful post to the sender. - -This only happens if the sender has set their AcknowledgePosts attribute. -This module must appear after the deliverer in the message pipeline in order -to send acks only after successful delivery. - -""" - -from Mailman import Errors -from Mailman import Message -from Mailman import Utils -from Mailman.configuration import config -from Mailman.i18n import _ - -__i18n_templates__ = True - - - -def process(mlist, msg, msgdata): - # Extract the sender's address and find them in the user database - sender = msgdata.get('original_sender', msg.get_sender()) - member = mlist.members.get_member(sender) - if member is None: - return - ack = member.acknowledge_posts - if not ack: - return - # Okay, they want acknowledgement of their post. Give them their original - # subject. BAW: do we want to use the decoded header? - origsubj = msgdata.get('origsubj', msg.get('subject', _('(no subject)'))) - # Get the user's preferred language - lang = msgdata.get('lang', member.preferred_language) - # Now get the acknowledgement template - realname = mlist.real_name - text = Utils.maketext( - 'postack.txt', - {'subject' : Utils.oneline(origsubj, Utils.GetCharSet(lang)), - 'listname' : realname, - 'listinfo_url': mlist.script_url('listinfo'), - 'optionsurl' : member.options_url, - }, lang=lang, mlist=mlist, raw=True) - # Craft the outgoing message, with all headers and attributes - # necessary for general delivery. Then enqueue it to the outgoing - # queue. - subject = _('$realname post acknowledgment') - usermsg = Message.UserNotification(sender, mlist.bounces_address, - subject, text, lang) - usermsg.send(mlist) diff --git a/Mailman/Handlers/AfterDelivery.py b/Mailman/Handlers/AfterDelivery.py deleted file mode 100644 index 16caf5773..000000000 --- a/Mailman/Handlers/AfterDelivery.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (C) 1998-2007 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""Perform some bookkeeping after a successful post. - -This module must appear after the delivery module in the message pipeline. -""" - -import datetime - - - -def process(mlist, msg, msgdata): - mlist.last_post_time = datetime.datetime.now() - mlist.post_id += 1 diff --git a/Mailman/Handlers/AvoidDuplicates.py b/Mailman/Handlers/AvoidDuplicates.py deleted file mode 100644 index a652906a9..000000000 --- a/Mailman/Handlers/AvoidDuplicates.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright (C) 2002-2007 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""If the user wishes it, do not send duplicates of the same message. - -This module keeps an in-memory dictionary of Message-ID: and recipient pairs. -If a message with an identical Message-ID: is about to be sent to someone who -has already received a copy, we either drop the message, add a duplicate -warning header, or pass it through, depending on the user's preferences. -""" - -from email.Utils import getaddresses, formataddr -from Mailman.configuration import config - -COMMASPACE = ', ' - - - -def process(mlist, msg, msgdata): - recips = msgdata.get('recips') - # Short circuit - if not recips: - return - # Seed this set with addresses we don't care about dup avoiding. - listaddrs = set((mlist.posting_address, - mlist.bounces_address, - mlist.owner_address, - mlist.request_address)) - explicit_recips = listaddrs.copy() - # Figure out the set of explicit recipients - cc_addresses = {} - for header in ('to', 'cc', 'resent-to', 'resent-cc'): - addrs = getaddresses(msg.get_all(header, [])) - header_addresses = dict((addr, formataddr((name, addr))) - for name, addr in addrs - if addr) - if header == 'cc': - # Yes, it's possible that an address is mentioned in multiple CC - # headers using different names. In that case, the last real name - # will win, but that doesn't seem like such a big deal. Besides, - # how else would you chose? - cc_addresses.update(header_addresses) - # Ignore the list addresses for purposes of dup avoidance. - explicit_recips |= set(header_addresses) - # Now strip out the list addresses - explicit_recips -= listaddrs - if not explicit_recips: - # No one was explicitly addressed, so we can't do any dup collapsing - return - newrecips = set() - for r in recips: - # If this recipient is explicitly addressed... - if r in explicit_recips: - send_duplicate = True - # If the member wants to receive duplicates, or if the recipient - # is not a member at all, they will get a copy. - # header. - member = mlist.members.get_member(r) - if member and not member.receive_list_copy: - send_duplicate = False - # We'll send a duplicate unless the user doesn't wish it. If - # personalization is enabled, the add-dupe-header flag will add a - # X-Mailman-Duplicate: yes header for this user's message. - if send_duplicate: - msgdata.setdefault('add-dup-header', set()).add(r) - newrecips.add(r) - elif r in cc_addresses: - del cc_addresses[r] - else: - # Otherwise, this is the first time they've been in the recips - # list. Add them to the newrecips list and flag them as having - # received this message. - newrecips.add(r) - # Set the new list of recipients. XXX recips should always be a set. - msgdata['recips'] = list(newrecips) - # RFC 2822 specifies zero or one CC header - if cc_addresses: - del msg['cc'] - msg['CC'] = COMMASPACE.join(cc_addresses.values()) diff --git a/Mailman/Handlers/CalcRecips.py b/Mailman/Handlers/CalcRecips.py deleted file mode 100644 index ebaf941c9..000000000 --- a/Mailman/Handlers/CalcRecips.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright (C) 1998-2007 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""Calculate the regular (i.e. non-digest) recipients of the message. - -This module calculates the non-digest recipients for the message based on the -list's membership and configuration options. It places the list of recipients -on the `recips' attribute of the message. This attribute is used by the -SendmailDeliver and BulkDeliver modules. -""" - -from Mailman import Errors -from Mailman import Message -from Mailman import Utils -from Mailman.configuration import config -from Mailman.i18n import _ -from Mailman.interfaces import DeliveryStatus - - - -def process(mlist, msg, msgdata): - # Short circuit if we've already calculated the recipients list, - # regardless of whether the list is empty or not. - if 'recips' in msgdata: - return - # Should the original sender should be included in the recipients list? - include_sender = True - sender = msg.get_sender() - member = mlist.members.get_member(sender) - if member and not member.receive_own_postings: - include_sender = False - # Support for urgent messages, which bypasses digests and disabled - # delivery and forces an immediate delivery to all members Right Now. We - # are specifically /not/ allowing the site admins password to work here - # because we want to discourage the practice of sending the site admin - # password through email in the clear. (see also Approve.py) - missing = [] - password = msg.get('urgent', missing) - if password is not missing: - if mlist.Authenticate((config.AuthListModerator, - config.AuthListAdmin), - password): - recips = mlist.getMemberCPAddresses(mlist.getRegularMemberKeys() + - mlist.getDigestMemberKeys()) - msgdata['recips'] = recips - return - else: - # Bad Urgent: password, so reject it instead of passing it on. I - # think it's better that the sender know they screwed up than to - # deliver it normally. - realname = mlist.real_name - text = _("""\ -Your urgent message to the %(realname)s mailing list was not authorized for -delivery. The original message as received by Mailman is attached. -""") - raise Errors.RejectMessage, Utils.wrap(text) - # Calculate the regular recipients of the message - recips = set(member.address.address - for member in mlist.regular_members.members - if member.delivery_status == DeliveryStatus.enabled) - # Remove the sender if they don't want to receive their own posts - if not include_sender and member.address.address in recips: - recips.remove(member.address.address) - # Handle topic classifications - do_topic_filters(mlist, msg, msgdata, recips) - # Bookkeeping - msgdata['recips'] = recips - - - -def do_topic_filters(mlist, msg, msgdata, recips): - if not mlist.topics_enabled: - # MAS: if topics are currently disabled for the list, send to all - # regardless of ReceiveNonmatchingTopics - return - hits = msgdata.get('topichits') - zaprecips = [] - if hits: - # The message hit some topics, so only deliver this message to those - # who are interested in one of the hit topics. - for user in recips: - utopics = mlist.getMemberTopics(user) - if not utopics: - # This user is not interested in any topics, so they get all - # postings. - continue - # BAW: Slow, first-match, set intersection! - for topic in utopics: - if topic in hits: - # The user wants this message - break - else: - # The user was interested in topics, but not any of the ones - # this message matched, so zap him. - zaprecips.append(user) - else: - # The semantics for a message that did not hit any of the pre-canned - # topics is to troll through the membership list, looking for users - # who selected at least one topic of interest, but turned on - # ReceiveNonmatchingTopics. - for user in recips: - if not mlist.getMemberTopics(user): - # The user did not select any topics of interest, so he gets - # this message by default. - continue - if not mlist.getMemberOption(user, - config.ReceiveNonmatchingTopics): - # The user has interest in some topics, but elects not to - # receive message that match no topics, so zap him. - zaprecips.append(user) - # Otherwise, the user wants non-matching messages. - # Prune out the non-receiving users - for user in zaprecips: - recips.remove(user) diff --git a/Mailman/Handlers/Cleanse.py b/Mailman/Handlers/Cleanse.py deleted file mode 100644 index bf25a4591..000000000 --- a/Mailman/Handlers/Cleanse.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (C) 1998-2008 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""Cleanse certain headers from all messages.""" - -import logging - -from email.Utils import formataddr - -from Mailman.Handlers.CookHeaders import uheader - -log = logging.getLogger('mailman.post') - - - -def process(mlist, msg, msgdata): - # Always remove this header from any outgoing messages. Be sure to do - # this after the information on the header is actually used, but before a - # permanent record of the header is saved. - del msg['approved'] - # Remove this one too. - del msg['approve'] - # Also remove this header since it can contain a password - del msg['urgent'] - # We remove other headers from anonymous lists - if mlist.anonymous_list: - log.info('post to %s from %s anonymized', - mlist.fqdn_listname, msg.get('from')) - del msg['from'] - del msg['reply-to'] - del msg['sender'] - # Hotmail sets this one - del msg['x-originating-email'] - i18ndesc = str(uheader(mlist, mlist.description, 'From')) - msg['From'] = formataddr((i18ndesc, mlist.posting_address)) - msg['Reply-To'] = mlist.posting_address - # Some headers can be used to fish for membership - del msg['return-receipt-to'] - del msg['disposition-notification-to'] - del msg['x-confirm-reading-to'] - # Pegasus mail uses this one... sigh - del msg['x-pmrqc'] - # Don't let this header be spoofed. See RFC 5064. - del msg['archived-at'] diff --git a/Mailman/Handlers/CleanseDKIM.py b/Mailman/Handlers/CleanseDKIM.py deleted file mode 100644 index 9dee4fcb0..000000000 --- a/Mailman/Handlers/CleanseDKIM.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (C) 2006-2007 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""Remove any 'DomainKeys' (or similar) header lines. - -The values contained in these header lines are intended to be used by the -recipient to detect forgery or tampering in transit, and the modifications -made by Mailman to the headers and body of the message will cause these keys -to appear invalid. Removing them will at least avoid this misleading result, -and it will also give the MTA the opportunity to regenerate valid keys -originating at the Mailman server for the outgoing message. -""" - -from Mailman.configuration import config - - - -def process(mlist, msg, msgdata): - if config.REMOVE_DKIM_HEADERS: - del msg['domainkey-signature'] - del msg['dkim-signature'] - del msg['authentication-results'] diff --git a/Mailman/Handlers/CookHeaders.py b/Mailman/Handlers/CookHeaders.py deleted file mode 100644 index 4797de62b..000000000 --- a/Mailman/Handlers/CookHeaders.py +++ /dev/null @@ -1,338 +0,0 @@ -# Copyright (C) 1998-2008 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""Cook a message's headers.""" - -import re - -from email.Charset import Charset -from email.Errors import HeaderParseError -from email.Header import Header, decode_header, make_header -from email.Utils import parseaddr, formataddr, getaddresses - -from Mailman import Utils -from Mailman import Version -from Mailman.app.archiving import get_archiver -from Mailman.configuration import config -from Mailman.i18n import _ -from Mailman.interfaces import Personalization, ReplyToMunging - -CONTINUATION = ',\n\t' -COMMASPACE = ', ' -MAXLINELEN = 78 - -nonascii = re.compile('[^\s!-~]') - - - -def uheader(mlist, s, header_name=None, continuation_ws='\t', maxlinelen=None): - # Get the charset to encode the string in. Then search if there is any - # non-ascii character is in the string. If there is and the charset is - # us-ascii then we use iso-8859-1 instead. If the string is ascii only - # we use 'us-ascii' if another charset is specified. - charset = Utils.GetCharSet(mlist.preferred_language) - if nonascii.search(s): - # use list charset but ... - if charset == 'us-ascii': - charset = 'iso-8859-1' - else: - # there is no nonascii so ... - charset = 'us-ascii' - return Header(s, charset, maxlinelen, header_name, continuation_ws) - - - -def process(mlist, msg, msgdata): - # Set the "X-Ack: no" header if noack flag is set. - if msgdata.get('noack'): - del msg['x-ack'] - msg['X-Ack'] = 'no' - # Because we're going to modify various important headers in the email - # message, we want to save some of the information in the msgdata - # dictionary for later. Specifically, the sender header will get waxed, - # but we need it for the Acknowledge module later. - msgdata['original_sender'] = msg.get_sender() - # VirginRunner sets _fasttrack for internally crafted messages. - fasttrack = msgdata.get('_fasttrack') - if not msgdata.get('isdigest') and not fasttrack: - try: - prefix_subject(mlist, msg, msgdata) - except (UnicodeError, ValueError): - # TK: Sometimes subject header is not MIME encoded for 8bit - # simply abort prefixing. - pass - # Mark message so we know we've been here, but leave any existing - # X-BeenThere's intact. - msg['X-BeenThere'] = mlist.posting_address - # Add Precedence: and other useful headers. None of these are standard - # and finding information on some of them are fairly difficult. Some are - # just common practice, and we'll add more here as they become necessary. - # Good places to look are: - # - # http://www.dsv.su.se/~jpalme/ietf/jp-ietf-home.html - # http://www.faqs.org/rfcs/rfc2076.html - # - # None of these headers are added if they already exist. BAW: some - # consider the advertising of this a security breach. I.e. if there are - # known exploits in a particular version of Mailman and we know a site is - # using such an old version, they may be vulnerable. It's too easy to - # edit the code to add a configuration variable to handle this. - if 'x-mailman-version' not in msg: - msg['X-Mailman-Version'] = Version.VERSION - # We set "Precedence: list" because this is the recommendation from the - # sendmail docs, the most authoritative source of this header's semantics. - if 'precedence' not in msg: - msg['Precedence'] = 'list' - # Reply-To: munging. Do not do this if the message is "fast tracked", - # meaning it is internally crafted and delivered to a specific user. BAW: - # Yuck, I really hate this feature but I've caved under the sheer pressure - # of the (very vocal) folks want it. OTOH, RFC 2822 allows Reply-To: to - # be a list of addresses, so instead of replacing the original, simply - # augment it. RFC 2822 allows max one Reply-To: header so collapse them - # if we're adding a value, otherwise don't touch it. (Should we collapse - # in all cases?) - if not fasttrack: - # A convenience function, requires nested scopes. pair is (name, addr) - new = [] - d = {} - def add(pair): - lcaddr = pair[1].lower() - if lcaddr in d: - return - d[lcaddr] = pair - new.append(pair) - # List admin wants an explicit Reply-To: added - if mlist.reply_goes_to_list == ReplyToMunging.explicit_header: - add(parseaddr(mlist.reply_to_address)) - # If we're not first stripping existing Reply-To: then we need to add - # the original Reply-To:'s to the list we're building up. In both - # cases we'll zap the existing field because RFC 2822 says max one is - # allowed. - if not mlist.first_strip_reply_to: - orig = msg.get_all('reply-to', []) - for pair in getaddresses(orig): - add(pair) - # Set Reply-To: header to point back to this list. Add this last - # because some folks think that some MUAs make it easier to delete - # addresses from the right than from the left. - if mlist.reply_goes_to_list == ReplyToMunging.point_to_list: - i18ndesc = uheader(mlist, mlist.description, 'Reply-To') - add((str(i18ndesc), mlist.posting_address)) - del msg['reply-to'] - # Don't put Reply-To: back if there's nothing to add! - if new: - # Preserve order - msg['Reply-To'] = COMMASPACE.join( - [formataddr(pair) for pair in new]) - # The To field normally contains the list posting address. However - # when messages are fully personalized, that header will get - # overwritten with the address of the recipient. We need to get the - # posting address in one of the recipient headers or they won't be - # able to reply back to the list. It's possible the posting address - # was munged into the Reply-To header, but if not, we'll add it to a - # Cc header. BAW: should we force it into a Reply-To header in the - # above code? - # Also skip Cc if this is an anonymous list as list posting address - # is already in From and Reply-To in this case. - if (mlist.personalize == Personalization.full and - mlist.reply_goes_to_list <> ReplyToMunging.point_to_list and - not mlist.anonymous_list): - # Watch out for existing Cc headers, merge, and remove dups. Note - # that RFC 2822 says only zero or one Cc header is allowed. - new = [] - d = {} - for pair in getaddresses(msg.get_all('cc', [])): - add(pair) - i18ndesc = uheader(mlist, mlist.description, 'Cc') - add((str(i18ndesc), mlist.posting_address)) - del msg['Cc'] - msg['Cc'] = COMMASPACE.join([formataddr(pair) for pair in new]) - # Add list-specific headers as defined in RFC 2369 and RFC 2919, but only - # if the message is being crafted for a specific list (e.g. not for the - # password reminders). - # - # BAW: Some people really hate the List-* headers. It seems that the free - # version of Eudora (possibly on for some platforms) does not hide these - # headers by default, pissing off their users. Too bad. Fix the MUAs. - if msgdata.get('_nolist') or not mlist.include_rfc2369_headers: - return - # This will act like an email address for purposes of formataddr() - listid = '%s.%s' % (mlist.list_name, mlist.host_name) - cset = Utils.GetCharSet(mlist.preferred_language) - if mlist.description: - # Don't wrap the header since here we just want to get it properly RFC - # 2047 encoded. - i18ndesc = uheader(mlist, mlist.description, 'List-Id', maxlinelen=998) - listid_h = formataddr((str(i18ndesc), listid)) - else: - # without desc we need to ensure the MUST brackets - listid_h = '<%s>' % listid - # We always add a List-ID: header. - del msg['list-id'] - msg['List-Id'] = listid_h - # For internally crafted messages, we also add a (nonstandard), - # "X-List-Administrivia: yes" header. For all others (i.e. those coming - # from list posts), we add a bunch of other RFC 2369 headers. - requestaddr = mlist.request_address - subfieldfmt = '<%s>, <mailto:%s>' - listinfo = mlist.script_url('listinfo') - headers = {} - # XXX reduced_list_headers used to suppress List-Help, List-Subject, and - # List-Unsubscribe from UserNotification. That doesn't seem to make sense - # any more, so always add those three headers (others will still be - # suppressed). - headers.update({ - 'List-Help' : '<mailto:%s?subject=help>' % requestaddr, - 'List-Unsubscribe': subfieldfmt % (listinfo, mlist.leave_address), - 'List-Subscribe' : subfieldfmt % (listinfo, mlist.join_address), - }) - archiver = get_archiver() - if msgdata.get('reduced_list_headers'): - headers['X-List-Administrivia'] = 'yes' - else: - # List-Post: is controlled by a separate attribute - if mlist.include_list_post_header: - headers['List-Post'] = '<mailto:%s>' % mlist.posting_address - # Add this header if we're archiving - if mlist.archive: - archiveurl = archiver.get_list_url(mlist) - headers['List-Archive'] = '<%s>' % archiveurl - # XXX RFC 2369 also defines a List-Owner header which we are not currently - # supporting, but should. - # - # Draft RFC 5064 defines an Archived-At header which contains the pointer - # directly to the message in the archive. If the currently defined - # archiver can tell us the URL, go ahead and include this header. - archived_at = archiver.get_message_url(mlist, msg) - if archived_at is not None: - headers['Archived-At'] = archived_at - # First we delete any pre-existing headers because the RFC permits only - # one copy of each, and we want to be sure it's ours. - for h, v in headers.items(): - del msg[h] - # Wrap these lines if they are too long. 78 character width probably - # shouldn't be hardcoded, but is at least text-MUA friendly. The - # adding of 2 is for the colon-space separator. - if len(h) + 2 + len(v) > 78: - v = CONTINUATION.join(v.split(', ')) - msg[h] = v - - - -def prefix_subject(mlist, msg, msgdata): - # Add the subject prefix unless the message is a digest or is being fast - # tracked (e.g. internally crafted, delivered to a single user such as the - # list admin). - if not mlist.subject_prefix.strip(): - return - prefix = mlist.subject_prefix - subject = msg.get('subject', '') - # Try to figure out what the continuation_ws is for the header - if isinstance(subject, Header): - lines = str(subject).splitlines() - else: - lines = subject.splitlines() - ws = '\t' - if len(lines) > 1 and lines[1] and lines[1][0] in ' \t': - ws = lines[1][0] - msgdata['origsubj'] = subject - # The subject may be multilingual but we take the first charset as major - # one and try to decode. If it is decodable, returned subject is in one - # line and cset is properly set. If fail, subject is mime-encoded and - # cset is set as us-ascii. See detail for ch_oneline() (CookHeaders one - # line function). - subject, cset = ch_oneline(subject) - # TK: Python interpreter has evolved to be strict on ascii charset code - # range. It is safe to use unicode string when manupilating header - # contents with re module. It would be best to return unicode in - # ch_oneline() but here is temporary solution. - subject = unicode(subject, cset) - # If the subject_prefix contains '%d', it is replaced with the - # mailing list sequential number. Sequential number format allows - # '%d' or '%05d' like pattern. - prefix_pattern = re.escape(prefix) - # unescape '%' :-< - prefix_pattern = '%'.join(prefix_pattern.split(r'\%')) - p = re.compile('%\d*d') - if p.search(prefix, 1): - # prefix have number, so we should search prefix w/number in subject. - # Also, force new style. - prefix_pattern = p.sub(r'\s*\d+\s*', prefix_pattern) - subject = re.sub(prefix_pattern, '', subject) - rematch = re.match('((RE|AW|SV|VS)(\[\d+\])?:\s*)+', subject, re.I) - if rematch: - subject = subject[rematch.end():] - recolon = 'Re:' - else: - recolon = '' - # At this point, subject may become null if someone post mail with - # subject: [subject prefix] - if subject.strip() == '': - subject = _('(no subject)') - cset = Utils.GetCharSet(mlist.preferred_language) - # and substitute %d in prefix with post_id - try: - prefix = prefix % mlist.post_id - except TypeError: - pass - # Get the header as a Header instance, with proper unicode conversion - if not recolon: - h = uheader(mlist, prefix, 'Subject', continuation_ws=ws) - else: - h = uheader(mlist, prefix, 'Subject', continuation_ws=ws) - h.append(recolon) - # TK: Subject is concatenated and unicode string. - subject = subject.encode(cset, 'replace') - h.append(subject, cset) - del msg['subject'] - msg['Subject'] = h - ss = uheader(mlist, recolon, 'Subject', continuation_ws=ws) - ss.append(subject, cset) - msgdata['stripped_subject'] = ss - - - -def ch_oneline(headerstr): - # Decode header string in one line and convert into single charset - # copied and modified from ToDigest.py and Utils.py - # return (string, cset) tuple as check for failure - try: - d = decode_header(headerstr) - # At this point, we should rstrip() every string because some - # MUA deliberately add trailing spaces when composing return - # message. - d = [(s.rstrip(),c) for (s,c) in d] - # Find all charsets in the original header. We use 'utf-8' rather - # than using the first charset (in mailman 2.1.x) if multiple - # charsets are used. - csets = [] - for (s,c) in d: - if c and c not in csets: - csets.append(c) - if len(csets) == 0: - cset = 'us-ascii' - elif len(csets) == 1: - cset = csets[0] - else: - cset = 'utf-8' - h = make_header(d) - ustr = unicode(h) - oneline = u''.join(ustr.splitlines()) - return oneline.encode(cset, 'replace'), cset - except (LookupError, UnicodeError, ValueError, HeaderParseError): - # possibly charset problem. return with undecoded string in one line. - return ''.join(headerstr.splitlines()), 'us-ascii' diff --git a/Mailman/Handlers/Decorate.py b/Mailman/Handlers/Decorate.py deleted file mode 100644 index 6891fed8d..000000000 --- a/Mailman/Handlers/Decorate.py +++ /dev/null @@ -1,207 +0,0 @@ -# Copyright (C) 1998-2007 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""Decorate a message by sticking the header and footer around it.""" - -import re -import logging - -from email.MIMEText import MIMEText -from string import Template - -from Mailman import Errors -from Mailman import Utils -from Mailman.Message import Message -from Mailman.configuration import config -from Mailman.i18n import _ - -log = logging.getLogger('mailman.error') - - - -def process(mlist, msg, msgdata): - # Digests and Mailman-craft messages should not get additional headers - if msgdata.get('isdigest') or msgdata.get('nodecorate'): - return - d = {} - if msgdata.get('personalize'): - # Calculate the extra personalization dictionary. Note that the - # length of the recips list better be exactly 1. - recips = msgdata.get('recips') - assert isinstance(recips, list) and len(recips) == 1, ( - 'The number of intended recipients must be exactly 1') - member = recips[0].lower() - d['user_address'] = member - try: - d['user_delivered_to'] = mlist.getMemberCPAddress(member) - # BAW: Hmm, should we allow this? - d['user_password'] = mlist.getMemberPassword(member) - d['user_language'] = mlist.getMemberLanguage(member) - username = mlist.getMemberName(member) or None - d['user_name'] = username or d['user_delivered_to'] - d['user_optionsurl'] = mlist.GetOptionsURL(member) - except Errors.NotAMemberError: - pass - # These strings are descriptive for the log file and shouldn't be i18n'd - d.update(msgdata.get('decoration-data', {})) - header = decorate(mlist, mlist.msg_header, d) - footer = decorate(mlist, mlist.msg_footer, d) - # Escape hatch if both the footer and header are empty - if not header and not footer: - return - # Be MIME smart here. We only attach the header and footer by - # concatenation when the message is a non-multipart of type text/plain. - # Otherwise, if it is not a multipart, we make it a multipart, and then we - # add the header and footer as text/plain parts. - # - # BJG: In addition, only add the footer if the message's character set - # matches the charset of the list's preferred language. This is a - # suboptimal solution, and should be solved by allowing a list to have - # multiple headers/footers, for each language the list supports. - # - # Also, if the list's preferred charset is us-ascii, we can always - # safely add the header/footer to a plain text message since all - # charsets Mailman supports are strict supersets of us-ascii -- - # no, UTF-16 emails are not supported yet. - # - # TK: Message with 'charset=' cause trouble. So, instead of - # mgs.get_content_charset('us-ascii') ... - mcset = msg.get_content_charset() or 'us-ascii' - lcset = Utils.GetCharSet(mlist.preferred_language) - msgtype = msg.get_content_type() - # BAW: If the charsets don't match, should we add the header and footer by - # MIME multipart chroming the message? - wrap = True - if not msg.is_multipart() and msgtype == 'text/plain': - # Save the RFC-3676 format parameters. - format = msg.get_param('format') - delsp = msg.get_param('delsp') - # Save 'Content-Transfer-Encoding' header in case decoration fails. - cte = msg.get('content-transfer-encoding') - # header/footer is now in unicode (2.2) - try: - oldpayload = unicode(msg.get_payload(decode=True), mcset) - del msg['content-transfer-encoding'] - frontsep = endsep = u'' - if header and not header.endswith('\n'): - frontsep = u'\n' - if footer and not oldpayload.endswith('\n'): - endsep = u'\n' - payload = header + frontsep + oldpayload + endsep + footer - # When setting the payload for the message, try various charset - # encodings until one does not produce a UnicodeError. We'll try - # charsets in this order: the list's charset, the message's - # charset, then utf-8. It's okay if some of these are duplicates. - for cset in (lcset, mcset, 'utf-8'): - try: - msg.set_payload(payload.encode(cset), cset) - except UnicodeError: - pass - else: - if format: - msg.set_param('format', format) - if delsp: - msg.set_param('delsp', delsp) - wrap = False - break - except (LookupError, UnicodeError): - if cte: - # Restore the original c-t-e. - del msg['content-transfer-encoding'] - msg['Content-Transfer-Encoding'] = cte - elif msg.get_content_type() == 'multipart/mixed': - # The next easiest thing to do is just prepend the header and append - # the footer as additional subparts - payload = msg.get_payload() - if not isinstance(payload, list): - payload = [payload] - if footer: - mimeftr = MIMEText(footer.encode(lcset), 'plain', lcset) - mimeftr['Content-Disposition'] = 'inline' - payload.append(mimeftr) - if header: - mimehdr = MIMEText(header.encode(lcset), 'plain', lcset) - mimehdr['Content-Disposition'] = 'inline' - payload.insert(0, mimehdr) - msg.set_payload(payload) - wrap = False - # If we couldn't add the header or footer in a less intrusive way, we can - # at least do it by MIME encapsulation. We want to keep as much of the - # outer chrome as possible. - if not wrap: - return - # Because of the way Message objects are passed around to process(), we - # need to play tricks with the outer message -- i.e. the outer one must - # remain the same instance. So we're going to create a clone of the outer - # message, with all the header chrome intact, then copy the payload to it. - # This will give us a clone of the original message, and it will form the - # basis of the interior, wrapped Message. - inner = Message() - # Which headers to copy? Let's just do the Content-* headers - for h, v in msg.items(): - if h.lower().startswith('content-'): - inner[h] = v - inner.set_payload(msg.get_payload()) - # For completeness - inner.set_unixfrom(msg.get_unixfrom()) - inner.preamble = msg.preamble - inner.epilogue = msg.epilogue - # Don't copy get_charset, as this might be None, even if - # get_content_charset isn't. However, do make sure there is a default - # content-type, even if the original message was not MIME. - inner.set_default_type(msg.get_default_type()) - # BAW: HACK ALERT. - if hasattr(msg, '__version__'): - inner.__version__ = msg.__version__ - # Now, play games with the outer message to make it contain three - # subparts: the header (if any), the wrapped message, and the footer (if - # any). - payload = [inner] - if header: - mimehdr = MIMEText(header.encode(lcset), 'plain', lcset) - mimehdr['Content-Disposition'] = 'inline' - payload.insert(0, mimehdr) - if footer: - mimeftr = MIMEText(footer.encode(lcset), 'plain', lcset) - mimeftr['Content-Disposition'] = 'inline' - payload.append(mimeftr) - msg.set_payload(payload) - del msg['content-type'] - del msg['content-transfer-encoding'] - del msg['content-disposition'] - msg['Content-Type'] = 'multipart/mixed' - - - -def decorate(mlist, template, extradict=None): - # Create a dictionary which includes the default set of interpolation - # variables allowed in headers and footers. These will be augmented by - # any key/value pairs in the extradict. - d = dict(real_name = mlist.real_name, - list_name = mlist.list_name, - fqdn_listname = mlist.fqdn_listname, - host_name = mlist.host_name, - web_page_url = mlist.web_page_url, - description = mlist.description, - info = mlist.info, - cgiext = config.CGIEXT, - ) - if extradict is not None: - d.update(extradict) - text = Template(template).safe_substitute(d) - # Turn any \r\n line endings into just \n - return re.sub(r' *\r?\n', r'\n', text) diff --git a/Mailman/Handlers/FileRecips.py b/Mailman/Handlers/FileRecips.py deleted file mode 100644 index 8ce07b432..000000000 --- a/Mailman/Handlers/FileRecips.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (C) 2001-2007 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""Get the normal delivery recipients from a Sendmail style :include: file.""" - -from __future__ import with_statement - -import os -import errno - -from Mailman import Errors - - - -def process(mlist, msg, msgdata): - if 'recips' in msgdata: - return - filename = os.path.join(mlist.full_path, 'members.txt') - try: - with open(filename) as fp: - addrs = set(line.strip() for line in fp) - except IOError, e: - if e.errno <> errno.ENOENT: - raise - msgdata['recips'] = set() - return - # If the sender is a member of the list, remove them from the file recips. - sender = msg.get_sender() - member = mlist.members.get_member(sender) - if member is not None: - addrs.discard(member.address.address) - msgdata['recips'] = addrs diff --git a/Mailman/Handlers/MimeDel.py b/Mailman/Handlers/MimeDel.py deleted file mode 100644 index 93349cb23..000000000 --- a/Mailman/Handlers/MimeDel.py +++ /dev/null @@ -1,261 +0,0 @@ -# Copyright (C) 2002-2007 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""MIME-stripping filter for Mailman. - -This module scans a message for MIME content, removing those sections whose -MIME types match one of a list of matches. multipart/alternative sections are -replaced by the first non-empty component, and multipart/mixed sections -wrapping only single sections after other processing are replaced by their -contents. -""" - -import os -import errno -import logging -import tempfile - -from email.Iterators import typed_subpart_iterator -from os.path import splitext - -from Mailman import Errors -from Mailman.Message import UserNotification -from Mailman.Utils import oneline -from Mailman.Version import VERSION -from Mailman.configuration import config -from Mailman.i18n import _ -from Mailman.queue import Switchboard - -log = logging.getLogger('mailman.error') - - - -def process(mlist, msg, msgdata): - # Short-circuits - if not mlist.filter_content: - return - if msgdata.get('isdigest'): - return - # We also don't care about our own digests or plaintext - ctype = msg.get_content_type() - mtype = msg.get_content_maintype() - # Check to see if the outer type matches one of the filter types - filtertypes = mlist.filter_mime_types - passtypes = mlist.pass_mime_types - if ctype in filtertypes or mtype in filtertypes: - dispose(mlist, msg, msgdata, - _("The message's content type was explicitly disallowed")) - # Check to see if there is a pass types and the outer type doesn't match - # one of these types - if passtypes and not (ctype in passtypes or mtype in passtypes): - dispose(mlist, msg, msgdata, - _("The message's content type was not explicitly allowed")) - # Filter by file extensions - filterexts = mlist.filter_filename_extensions - passexts = mlist.pass_filename_extensions - fext = get_file_ext(msg) - if fext: - if fext in filterexts: - dispose(mlist, msg, msgdata, - _("The message's file extension was explicitly disallowed")) - if passexts and not (fext in passexts): - dispose(mlist, msg, msgdata, - _("The message's file extension was not explicitly allowed")) - numparts = len([subpart for subpart in msg.walk()]) - # If the message is a multipart, filter out matching subparts - if msg.is_multipart(): - # Recursively filter out any subparts that match the filter list - prelen = len(msg.get_payload()) - filter_parts(msg, filtertypes, passtypes, filterexts, passexts) - # If the outer message is now an empty multipart (and it wasn't - # before!) then, again it gets discarded. - postlen = len(msg.get_payload()) - if postlen == 0 and prelen > 0: - dispose(mlist, msg, msgdata, - _("After content filtering, the message was empty")) - # Now replace all multipart/alternatives with just the first non-empty - # alternative. BAW: We have to special case when the outer part is a - # multipart/alternative because we need to retain most of the outer part's - # headers. For now we'll move the subpart's payload into the outer part, - # and then copy over its Content-Type: and Content-Transfer-Encoding: - # headers (any others?). - if mlist.collapse_alternatives: - collapse_multipart_alternatives(msg) - if ctype == 'multipart/alternative': - firstalt = msg.get_payload(0) - reset_payload(msg, firstalt) - # If we removed some parts, make note of this - changedp = 0 - if numparts <> len([subpart for subpart in msg.walk()]): - changedp = 1 - # Now perhaps convert all text/html to text/plain - if mlist.convert_html_to_plaintext and config.HTML_TO_PLAIN_TEXT_COMMAND: - changedp += to_plaintext(msg) - # If we're left with only two parts, an empty body and one attachment, - # recast the message to one of just that part - if msg.is_multipart() and len(msg.get_payload()) == 2: - if msg.get_payload(0).get_payload() == '': - useful = msg.get_payload(1) - reset_payload(msg, useful) - changedp = 1 - if changedp: - msg['X-Content-Filtered-By'] = 'Mailman/MimeDel %s' % VERSION - - - -def reset_payload(msg, subpart): - # Reset payload of msg to contents of subpart, and fix up content headers - payload = subpart.get_payload() - msg.set_payload(payload) - del msg['content-type'] - del msg['content-transfer-encoding'] - del msg['content-disposition'] - del msg['content-description'] - msg['Content-Type'] = subpart.get('content-type', 'text/plain') - cte = subpart.get('content-transfer-encoding') - if cte: - msg['Content-Transfer-Encoding'] = cte - cdisp = subpart.get('content-disposition') - if cdisp: - msg['Content-Disposition'] = cdisp - cdesc = subpart.get('content-description') - if cdesc: - msg['Content-Description'] = cdesc - - - -def filter_parts(msg, filtertypes, passtypes, filterexts, passexts): - # Look at all the message's subparts, and recursively filter - if not msg.is_multipart(): - return 1 - payload = msg.get_payload() - prelen = len(payload) - newpayload = [] - for subpart in payload: - keep = filter_parts(subpart, filtertypes, passtypes, - filterexts, passexts) - if not keep: - continue - ctype = subpart.get_content_type() - mtype = subpart.get_content_maintype() - if ctype in filtertypes or mtype in filtertypes: - # Throw this subpart away - continue - if passtypes and not (ctype in passtypes or mtype in passtypes): - # Throw this subpart away - continue - # check file extension - fext = get_file_ext(subpart) - if fext: - if fext in filterexts: - continue - if passexts and not (fext in passexts): - continue - newpayload.append(subpart) - # Check to see if we discarded all the subparts - postlen = len(newpayload) - msg.set_payload(newpayload) - if postlen == 0 and prelen > 0: - # We threw away everything - return 0 - return 1 - - - -def collapse_multipart_alternatives(msg): - if not msg.is_multipart(): - return - newpayload = [] - for subpart in msg.get_payload(): - if subpart.get_content_type() == 'multipart/alternative': - try: - firstalt = subpart.get_payload(0) - newpayload.append(firstalt) - except IndexError: - pass - else: - newpayload.append(subpart) - msg.set_payload(newpayload) - - - -def to_plaintext(msg): - changedp = 0 - for subpart in typed_subpart_iterator(msg, 'text', 'html'): - filename = tempfile.mktemp('.html') - fp = open(filename, 'w') - try: - fp.write(subpart.get_payload(decode=1)) - fp.close() - cmd = os.popen(config.HTML_TO_PLAIN_TEXT_COMMAND % - {'filename': filename}) - plaintext = cmd.read() - rtn = cmd.close() - if rtn: - log.error('HTML->text/plain error: %s', rtn) - finally: - try: - os.unlink(filename) - except OSError, e: - if e.errno <> errno.ENOENT: raise - # Now replace the payload of the subpart and twiddle the Content-Type: - del subpart['content-transfer-encoding'] - subpart.set_payload(plaintext) - subpart.set_type('text/plain') - changedp = 1 - return changedp - - - -def dispose(mlist, msg, msgdata, why): - # filter_action == 0 just discards, see below - if mlist.filter_action == 1: - # Bounce the message to the original author - raise Errors.RejectMessage, why - if mlist.filter_action == 2: - # Forward it on to the list owner - listname = mlist.internal_name() - mlist.ForwardMessage( - msg, - text=_("""\ -The attached message matched the %(listname)s mailing list's content filtering -rules and was prevented from being forwarded on to the list membership. You -are receiving the only remaining copy of the discarded message. - -"""), - subject=_('Content filtered message notification')) - if mlist.filter_action == 3 and \ - config.OWNERS_CAN_PRESERVE_FILTERED_MESSAGES: - badq = Switchboard(config.BADQUEUE_DIR) - badq.enqueue(msg, msgdata) - # Most cases also discard the message - raise Errors.DiscardMessage - -def get_file_ext(m): - """ - Get filename extension. Caution: some virus don't put filename - in 'Content-Disposition' header. -""" - fext = '' - filename = m.get_filename('') or m.get_param('name', '') - if filename: - fext = splitext(oneline(filename,'utf-8'))[1] - if len(fext) > 1: - fext = fext[1:] - else: - fext = '' - return fext diff --git a/Mailman/Handlers/Moderate.py b/Mailman/Handlers/Moderate.py deleted file mode 100644 index 464215d25..000000000 --- a/Mailman/Handlers/Moderate.py +++ /dev/null @@ -1,168 +0,0 @@ -# Copyright (C) 2001-2007 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""Posting moderation filter.""" - -import re - -from email.MIMEMessage import MIMEMessage -from email.MIMEText import MIMEText - -from Mailman import Errors -from Mailman import Message -from Mailman import Utils -from Mailman.Handlers import Hold -from Mailman.configuration import config -from Mailman.i18n import _ - - - -class ModeratedMemberPost(Hold.ModeratedPost): - # BAW: I wanted to use the reason below to differentiate between this - # situation and normal ModeratedPost reasons. Greg Ward and Stonewall - # Ballard thought the language was too harsh and mentioned offense taken - # by some list members. I'd still like this class's reason to be - # different than the base class's reason, but we'll use this until someone - # can come up with something more clever but inoffensive. - # - # reason = _('Posts by member are currently quarantined for moderation') - pass - - - -def process(mlist, msg, msgdata): - if msgdata.get('approved') or msgdata.get('fromusenet'): - return - # First of all, is the poster a member or not? - for sender in msg.get_senders(): - if mlist.isMember(sender): - break - else: - sender = None - if sender: - # If the member's moderation flag is on, then perform the moderation - # action. - if mlist.getMemberOption(sender, config.Moderate): - # Note that for member_moderation_action, 0==Hold, 1=Reject, - # 2==Discard - if mlist.member_moderation_action == 0: - # Hold. BAW: WIBNI we could add the member_moderation_notice - # to the notice sent back to the sender? - msgdata['sender'] = sender - Hold.hold_for_approval(mlist, msg, msgdata, - ModeratedMemberPost) - elif mlist.member_moderation_action == 1: - # Reject - text = mlist.member_moderation_notice - if text: - text = Utils.wrap(text) - else: - # Use the default RejectMessage notice string - text = None - raise Errors.RejectMessage, text - elif mlist.member_moderation_action == 2: - # Discard. BAW: Again, it would be nice if we could send a - # discard notice to the sender - raise Errors.DiscardMessage - else: - assert 0, 'bad member_moderation_action' - # Should we do anything explict to mark this message as getting past - # this point? No, because further pipeline handlers will need to do - # their own thing. - return - else: - sender = msg.get_sender() - # From here on out, we're dealing with non-members. - if matches_p(sender, mlist.accept_these_nonmembers): - return - if matches_p(sender, mlist.hold_these_nonmembers): - Hold.hold_for_approval(mlist, msg, msgdata, Hold.NonMemberPost) - # No return - if matches_p(sender, mlist.reject_these_nonmembers): - do_reject(mlist) - # No return - if matches_p(sender, mlist.discard_these_nonmembers): - do_discard(mlist, msg) - # No return - # Okay, so the sender wasn't specified explicitly by any of the non-member - # moderation configuration variables. Handle by way of generic non-member - # action. - assert 0 <= mlist.generic_nonmember_action <= 4 - if mlist.generic_nonmember_action == 0: - # Accept - return - elif mlist.generic_nonmember_action == 1: - Hold.hold_for_approval(mlist, msg, msgdata, Hold.NonMemberPost) - elif mlist.generic_nonmember_action == 2: - do_reject(mlist) - elif mlist.generic_nonmember_action == 3: - do_discard(mlist, msg) - - - -def matches_p(sender, nonmembers): - # First strip out all the regular expressions - plainaddrs = [addr for addr in nonmembers if not addr.startswith('^')] - addrdict = Utils.List2Dict(plainaddrs, foldcase=1) - if addrdict.has_key(sender): - return 1 - # Now do the regular expression matches - for are in nonmembers: - if are.startswith('^'): - try: - cre = re.compile(are, re.IGNORECASE) - except re.error: - continue - if cre.search(sender): - return 1 - return 0 - - - -def do_reject(mlist): - listowner = mlist.GetOwnerEmail() - if mlist.nonmember_rejection_notice: - raise Errors.RejectMessage, \ - Utils.wrap(_(mlist.nonmember_rejection_notice)) - else: - raise Errors.RejectMessage, Utils.wrap(_("""\ -You are not allowed to post to this mailing list, and your message has been -automatically rejected. If you think that your messages are being rejected in -error, contact the mailing list owner at %(listowner)s.""")) - - - -def do_discard(mlist, msg): - sender = msg.get_sender() - # Do we forward auto-discards to the list owners? - if mlist.forward_auto_discards: - lang = mlist.preferred_language - varhelp = '%s/?VARHELP=privacy/sender/discard_these_nonmembers' % \ - mlist.GetScriptURL('admin', absolute=1) - nmsg = Message.UserNotification(mlist.GetOwnerEmail(), - mlist.GetBouncesEmail(), - _('Auto-discard notification'), - lang=lang) - nmsg.set_type('multipart/mixed') - text = MIMEText(Utils.wrap(_( - 'The attached message has been automatically discarded.')), - _charset=Utils.GetCharSet(lang)) - nmsg.attach(text) - nmsg.attach(MIMEMessage(msg)) - nmsg.send(mlist) - # Discard this sucker - raise Errors.DiscardMessage diff --git a/Mailman/Handlers/OwnerRecips.py b/Mailman/Handlers/OwnerRecips.py deleted file mode 100644 index c70e17777..000000000 --- a/Mailman/Handlers/OwnerRecips.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (C) 2001-2007 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""Calculate the list owner recipients (includes moderators).""" - - - -def process(mlist, msg, msgdata): - # The recipients are the owner and the moderator - msgdata['recips'] = mlist.owner + mlist.moderator - # Don't decorate these messages with the header/footers - msgdata['nodecorate'] = 1 - msgdata['personalize'] = 0 diff --git a/Mailman/Handlers/Replybot.py b/Mailman/Handlers/Replybot.py deleted file mode 100644 index 7017d9dd5..000000000 --- a/Mailman/Handlers/Replybot.py +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright (C) 1998-2007 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""Handler for auto-responses.""" - -import time -import logging -import datetime - -from string import Template - -from Mailman import Message -from Mailman import Utils -from Mailman.i18n import _ - -log = logging.getLogger('mailman.error') - -__i18n_templates__ = True -NODELTA = datetime.timedelta() - - - -def process(mlist, msg, msgdata): - # Normally, the replybot should get a shot at this message, but there are - # some important short-circuits, mostly to suppress 'bot storms, at least - # for well behaved email bots (there are other governors for misbehaving - # 'bots). First, if the original message has an "X-Ack: No" header, we - # skip the replybot. Then, if the message has a Precedence header with - # values bulk, junk, or list, and there's no explicit "X-Ack: yes" header, - # we short-circuit. Finally, if the message metadata has a true 'noack' - # key, then we skip the replybot too. - ack = msg.get('x-ack', '').lower() - if ack == 'no' or msgdata.get('noack'): - return - precedence = msg.get('precedence', '').lower() - if ack <> 'yes' and precedence in ('bulk', 'junk', 'list'): - return - # Check to see if the list is even configured to autorespond to this email - # message. Note: the mailowner script sets the `toadmin' or `toowner' key - # (which for replybot purposes are equivalent), and the mailcmd script - # sets the `torequest' key. - toadmin = msgdata.get('toowner') - torequest = msgdata.get('torequest') - if ((toadmin and not mlist.autorespond_admin) or - (torequest and not mlist.autorespond_requests) or \ - (not toadmin and not torequest and not mlist.autorespond_postings)): - return - # Now see if we're in the grace period for this sender. graceperiod <= 0 - # means always autorespond, as does an "X-Ack: yes" header (useful for - # debugging). - sender = msg.get_sender() - now = time.time() - graceperiod = mlist.autoresponse_graceperiod - if graceperiod > NODELTA and ack <> 'yes': - if toadmin: - quiet_until = mlist.admin_responses.get(sender, 0) - elif torequest: - quiet_until = mlist.request_responses.get(sender, 0) - else: - quiet_until = mlist.postings_responses.get(sender, 0) - if quiet_until > now: - return - # Okay, we know we're going to auto-respond to this sender, craft the - # message, send it, and update the database. - realname = mlist.real_name - subject = _( - 'Auto-response for your message to the "$realname" mailing list') - # Do string interpolation into the autoresponse text - d = dict(listname = realname, - listurl = mlist.script_url('listinfo'), - requestemail = mlist.request_address, - owneremail = mlist.owner_address, - ) - if toadmin: - rtext = mlist.autoresponse_admin_text - elif torequest: - rtext = mlist.autoresponse_request_text - else: - rtext = mlist.autoresponse_postings_text - # Interpolation and Wrap the response text. - text = Utils.wrap(Template(rtext).safe_substitute(d)) - outmsg = Message.UserNotification(sender, mlist.bounces_address, - subject, text, mlist.preferred_language) - outmsg['X-Mailer'] = _('The Mailman Replybot') - # prevent recursions and mail loops! - outmsg['X-Ack'] = 'No' - outmsg.send(mlist) - # update the grace period database - if graceperiod > NODELTA: - # graceperiod is in days, we need # of seconds - quiet_until = now + graceperiod * 24 * 60 * 60 - if toadmin: - mlist.admin_responses[sender] = quiet_until - elif torequest: - mlist.request_responses[sender] = quiet_until - else: - mlist.postings_responses[sender] = quiet_until diff --git a/Mailman/Handlers/SMTPDirect.py b/Mailman/Handlers/SMTPDirect.py deleted file mode 100644 index 0037d676a..000000000 --- a/Mailman/Handlers/SMTPDirect.py +++ /dev/null @@ -1,389 +0,0 @@ -# Copyright (C) 1998-2007 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""Local SMTP direct drop-off. - -This module delivers messages via SMTP to a locally specified daemon. This -should be compatible with any modern SMTP server. It is expected that the MTA -handles all final delivery. We have to play tricks so that the list object -isn't locked while delivery occurs synchronously. - -Note: This file only handles single threaded delivery. See SMTPThreaded.py -for a threaded implementation. -""" - -import copy -import time -import email -import socket -import logging -import smtplib - -from email.Charset import Charset -from email.Header import Header -from email.Utils import formataddr - -from Mailman import Errors -from Mailman import Utils -from Mailman.Handlers import Decorate -from Mailman.SafeDict import MsgSafeDict -from Mailman.configuration import config -from Mailman.interfaces import Personalization - -DOT = '.' - -log = logging.getLogger('mailman.smtp') -flog = logging.getLogger('mailman.smtp-failure') -every_log = logging.getLogger('mailman.' + config.SMTP_LOG_EVERY_MESSAGE[0]) -success_log = logging.getLogger('mailman.' + config.SMTP_LOG_SUCCESS[0]) -refused_log = logging.getLogger('mailman.' + config.SMTP_LOG_REFUSED[0]) -failure_log = logging.getLogger('mailman.' + config.SMTP_LOG_EACH_FAILURE[0]) - - - -# Manage a connection to the SMTP server -class Connection: - def __init__(self): - self.__conn = None - - def __connect(self): - self.__conn = smtplib.SMTP() - self.__conn.connect(config.SMTPHOST, config.SMTPPORT) - self.__numsessions = config.SMTP_MAX_SESSIONS_PER_CONNECTION - - def sendmail(self, envsender, recips, msgtext): - if self.__conn is None: - self.__connect() - try: - results = self.__conn.sendmail(envsender, recips, msgtext) - except smtplib.SMTPException: - # For safety, close this connection. The next send attempt will - # automatically re-open it. Pass the exception on up. - self.quit() - raise - # This session has been successfully completed. - self.__numsessions -= 1 - # By testing exactly for equality to 0, we automatically handle the - # case for SMTP_MAX_SESSIONS_PER_CONNECTION <= 0 meaning never close - # the connection. We won't worry about wraparound <wink>. - if self.__numsessions == 0: - self.quit() - return results - - def quit(self): - if self.__conn is None: - return - try: - self.__conn.quit() - except smtplib.SMTPException: - pass - self.__conn = None - - - -def process(mlist, msg, msgdata): - recips = msgdata.get('recips') - if not recips: - # Nobody to deliver to! - return - # Calculate the non-VERP envelope sender. - envsender = msgdata.get('envsender') - if envsender is None: - if mlist: - envsender = mlist.GetBouncesEmail() - else: - envsender = Utils.get_site_noreply() - # Time to split up the recipient list. If we're personalizing or VERPing - # then each chunk will have exactly one recipient. We'll then hand craft - # an envelope sender and stitch a message together in memory for each one - # separately. If we're not VERPing, then we'll chunkify based on - # SMTP_MAX_RCPTS. Note that most MTAs have a limit on the number of - # recipients they'll swallow in a single transaction. - deliveryfunc = None - if (not msgdata.has_key('personalize') or msgdata['personalize']) and ( - msgdata.get('verp') or mlist.personalize <> Personalization.none): - chunks = [[recip] for recip in recips] - msgdata['personalize'] = 1 - deliveryfunc = verpdeliver - elif config.SMTP_MAX_RCPTS <= 0: - chunks = [recips] - else: - chunks = chunkify(recips, config.SMTP_MAX_RCPTS) - # See if this is an unshunted message for which some were undelivered - if msgdata.has_key('undelivered'): - chunks = msgdata['undelivered'] - # If we're doing bulk delivery, then we can stitch up the message now. - if deliveryfunc is None: - # Be sure never to decorate the message more than once! - if not msgdata.get('decorated'): - Decorate.process(mlist, msg, msgdata) - msgdata['decorated'] = True - deliveryfunc = bulkdeliver - refused = {} - t0 = time.time() - # Open the initial connection - origrecips = msgdata['recips'] - # MAS: get the message sender now for logging. If we're using 'sender' - # and not 'from', bulkdeliver changes it for bounce processing. If we're - # VERPing, it doesn't matter because bulkdeliver is working on a copy, but - # otherwise msg gets changed. If the list is anonymous, the original - # sender is long gone, but Cleanse.py has logged it. - origsender = msgdata.get('original_sender', msg.get_sender()) - # `undelivered' is a copy of chunks that we pop from to do deliveries. - # This seems like a good tradeoff between robustness and resource - # utilization. If delivery really fails (i.e. qfiles/shunt type - # failures), then we'll pick up where we left off with `undelivered'. - # This means at worst, the last chunk for which delivery was attempted - # could get duplicates but not every one, and no recips should miss the - # message. - conn = Connection() - try: - msgdata['undelivered'] = chunks - while chunks: - chunk = chunks.pop() - msgdata['recips'] = chunk - try: - deliveryfunc(mlist, msg, msgdata, envsender, refused, conn) - except Exception: - # If /anything/ goes wrong, push the last chunk back on the - # undelivered list and re-raise the exception. We don't know - # how many of the last chunk might receive the message, so at - # worst, everyone in this chunk will get a duplicate. Sigh. - chunks.append(chunk) - raise - del msgdata['undelivered'] - finally: - conn.quit() - msgdata['recips'] = origrecips - # Log the successful post - t1 = time.time() - d = MsgSafeDict(msg, {'time' : t1-t0, - # BAW: Urg. This seems inefficient. - 'size' : len(msg.as_string()), - '#recips' : len(recips), - '#refused': len(refused), - 'listname': mlist.internal_name(), - 'sender' : origsender, - }) - # We have to use the copy() method because extended call syntax requires a - # concrete dictionary object; it does not allow a generic mapping (XXX is - # this still true in Python 2.3?). - if config.SMTP_LOG_EVERY_MESSAGE: - every_log.info('%s', config.SMTP_LOG_EVERY_MESSAGE[1] % d) - - if refused: - if config.SMTP_LOG_REFUSED: - refused_log.info('%s', config.SMTP_LOG_REFUSED[1] % d) - - elif msgdata.get('tolist'): - # Log the successful post, but only if it really was a post to the - # mailing list. Don't log sends to the -owner, or -admin addrs. - # -request addrs should never get here. BAW: it may be useful to log - # the other messages, but in that case, we should probably have a - # separate configuration variable to control that. - if config.SMTP_LOG_SUCCESS: - success_log.info('%s', config.SMTP_LOG_SUCCESS[1] % d) - - # Process any failed deliveries. - tempfailures = [] - permfailures = [] - for recip, (code, smtpmsg) in refused.items(): - # DRUMS is an internet draft, but it says: - # - # [RFC-821] incorrectly listed the error where an SMTP server - # exhausts its implementation limit on the number of RCPT commands - # ("too many recipients") as having reply code 552. The correct - # reply code for this condition is 452. Clients SHOULD treat a 552 - # code in this case as a temporary, rather than permanent failure - # so the logic below works. - # - if code >= 500 and code <> 552: - # A permanent failure - permfailures.append(recip) - else: - # Deal with persistent transient failures by queuing them up for - # future delivery. TBD: this could generate lots of log entries! - tempfailures.append(recip) - if config.SMTP_LOG_EACH_FAILURE: - d.update({'recipient': recip, - 'failcode' : code, - 'failmsg' : smtpmsg}) - failure_log.info('%s', config.SMTP_LOG_EACH_FAILURE[1] % d) - # Return the results - if tempfailures or permfailures: - raise Errors.SomeRecipientsFailed(tempfailures, permfailures) - - - -def chunkify(recips, chunksize): - # First do a simple sort on top level domain. It probably doesn't buy us - # much to try to sort on MX record -- that's the MTA's job. We're just - # trying to avoid getting a max recips error. Split the chunks along - # these lines (as suggested originally by Chuq Von Rospach and slightly - # elaborated by BAW). - chunkmap = {'com': 1, - 'net': 2, - 'org': 2, - 'edu': 3, - 'us' : 3, - 'ca' : 3, - } - buckets = {} - for r in recips: - tld = None - i = r.rfind('.') - if i >= 0: - tld = r[i+1:] - bin = chunkmap.get(tld, 0) - bucket = buckets.get(bin, []) - bucket.append(r) - buckets[bin] = bucket - # Now start filling the chunks - chunks = [] - currentchunk = [] - chunklen = 0 - for bin in buckets.values(): - for r in bin: - currentchunk.append(r) - chunklen = chunklen + 1 - if chunklen >= chunksize: - chunks.append(currentchunk) - currentchunk = [] - chunklen = 0 - if currentchunk: - chunks.append(currentchunk) - currentchunk = [] - chunklen = 0 - return chunks - - - -def verpdeliver(mlist, msg, msgdata, envsender, failures, conn): - for recip in msgdata['recips']: - # We now need to stitch together the message with its header and - # footer. If we're VERPIng, we have to calculate the envelope sender - # for each recipient. Note that the list of recipients must be of - # length 1. - # - # BAW: ezmlm includes the message number in the envelope, used when - # sending a notification to the user telling her how many messages - # they missed due to bouncing. Neat idea. - msgdata['recips'] = [recip] - # Make a copy of the message and decorate + delivery that - msgcopy = copy.deepcopy(msg) - Decorate.process(mlist, msgcopy, msgdata) - # Calculate the envelope sender, which we may be VERPing - if msgdata.get('verp'): - bmailbox, bdomain = Utils.ParseEmail(envsender) - rmailbox, rdomain = Utils.ParseEmail(recip) - if rdomain is None: - # The recipient address is not fully-qualified. We can't - # deliver it to this person, nor can we craft a valid verp - # header. I don't think there's much we can do except ignore - # this recipient. - log.info('Skipping VERP delivery to unqual recip: %s', recip) - continue - d = {'bounces': bmailbox, - 'mailbox': rmailbox, - 'host' : DOT.join(rdomain), - } - envsender = '%s@%s' % ((config.VERP_FORMAT % d), DOT.join(bdomain)) - if mlist.personalize == Personalization.full: - # When fully personalizing, we want the To address to point to the - # recipient, not to the mailing list - del msgcopy['to'] - name = None - if mlist.isMember(recip): - name = mlist.getMemberName(recip) - if name: - # Convert the name to an email-safe representation. If the - # name is a byte string, convert it first to Unicode, given - # the character set of the member's language, replacing bad - # characters for which we can do nothing about. Once we have - # the name as Unicode, we can create a Header instance for it - # so that it's properly encoded for email transport. - charset = Utils.GetCharSet(mlist.getMemberLanguage(recip)) - if charset == 'us-ascii': - # Since Header already tries both us-ascii and utf-8, - # let's add something a bit more useful. - charset = 'iso-8859-1' - charset = Charset(charset) - codec = charset.input_codec or 'ascii' - if not isinstance(name, unicode): - name = unicode(name, codec, 'replace') - name = Header(name, charset).encode() - msgcopy['To'] = formataddr((name, recip)) - else: - msgcopy['To'] = recip - # We can flag the mail as a duplicate for each member, if they've - # already received this message, as calculated by Message-ID. See - # AvoidDuplicates.py for details. - del msgcopy['x-mailman-copy'] - if msgdata.get('add-dup-header', {}).has_key(recip): - msgcopy['X-Mailman-Copy'] = 'yes' - # For the final delivery stage, we can just bulk deliver to a party of - # one. ;) - bulkdeliver(mlist, msgcopy, msgdata, envsender, failures, conn) - - - -def bulkdeliver(mlist, msg, msgdata, envsender, failures, conn): - # Do some final cleanup of the message header. Start by blowing away - # any the Sender: and Errors-To: headers so remote MTAs won't be - # tempted to delivery bounces there instead of our envelope sender - # - # BAW An interpretation of RFCs 2822 and 2076 could argue for not touching - # the Sender header at all. Brad Knowles points out that MTAs tend to - # wipe existing Return-Path headers, and old MTAs may still honor - # Errors-To while new ones will at worst ignore the header. - del msg['sender'] - del msg['errors-to'] - msg['Sender'] = envsender - msg['Errors-To'] = envsender - # Get the plain, flattened text of the message, sans unixfrom - msgtext = msg.as_string() - refused = {} - recips = msgdata['recips'] - msgid = msg['message-id'] - try: - # Send the message - refused = conn.sendmail(envsender, recips, msgtext) - except smtplib.SMTPRecipientsRefused, e: - flog.error('All recipients refused: %s, msgid: %s', e, msgid) - refused = e.recipients - except smtplib.SMTPResponseException, e: - flog.error('SMTP session failure: %s, %s, msgid: %s', - e.smtp_code, e.smtp_error, msgid) - # If this was a permanent failure, don't add the recipients to the - # refused, because we don't want them to be added to failures. - # Otherwise, if the MTA rejects the message because of the message - # content (e.g. it's spam, virii, or has syntactic problems), then - # this will end up registering a bounce score for every recipient. - # Definitely /not/ what we want. - if e.smtp_code < 500 or e.smtp_code == 552: - # It's a temporary failure - for r in recips: - refused[r] = (e.smtp_code, e.smtp_error) - except (socket.error, IOError, smtplib.SMTPException), e: - # MTA not responding, or other socket problems, or any other kind of - # SMTPException. In that case, nothing got delivered, so treat this - # as a temporary failure. - flog.error('Low level smtp error: %s, msgid: %s', e, msgid) - error = str(e) - for r in recips: - refused[r] = (-1, error) - failures.update(refused) diff --git a/Mailman/Handlers/Scrubber.py b/Mailman/Handlers/Scrubber.py deleted file mode 100644 index fb1b6e602..000000000 --- a/Mailman/Handlers/Scrubber.py +++ /dev/null @@ -1,500 +0,0 @@ -# Copyright (C) 2001-2008 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""Cleanse a message for archiving.""" - -from __future__ import with_statement - -import os -import re -import sha -import time -import errno -import logging -import binascii - -from email.charset import Charset -from email.generator import Generator -from email.utils import make_msgid, parsedate -from locknix.lockfile import Lock -from mimetypes import guess_all_extensions - -from Mailman import Utils -from Mailman.Errors import DiscardMessage -from Mailman.app.archiving import get_archiver -from Mailman.configuration import config -from Mailman.i18n import _ - -# Path characters for common platforms -pre = re.compile(r'[/\\:]') -# All other characters to strip out of Content-Disposition: filenames -# (essentially anything that isn't an alphanum, dot, dash, or underscore). -sre = re.compile(r'[^-\w.]') -# Regexp to strip out leading dots -dre = re.compile(r'^\.*') - -BR = '<br>\n' -SPACE = ' ' - -log = logging.getLogger('mailman.error') - - - -def guess_extension(ctype, ext): - # mimetypes maps multiple extensions to the same type, e.g. .doc, .dot, - # and .wiz are all mapped to application/msword. This sucks for finding - # the best reverse mapping. If the extension is one of the giving - # mappings, we'll trust that, otherwise we'll just guess. :/ - all = guess_all_extensions(ctype, strict=False) - if ext in all: - return ext - return all and all[0] - - - -# We're using a subclass of the standard Generator because we want to suppress -# headers in the subparts of multiparts. We use a hack -- the ctor argument -# skipheaders to accomplish this. It's set to true for the outer Message -# object, but false for all internal objects. We recognize that -# sub-Generators will get created passing only mangle_from_ and maxheaderlen -# to the ctors. -# -# This isn't perfect because we still get stuff like the multipart boundaries, -# but see below for how we corrupt that to our nefarious goals. -class ScrubberGenerator(Generator): - def __init__(self, outfp, mangle_from_=True, - maxheaderlen=78, skipheaders=True): - Generator.__init__(self, outfp, mangle_from_=False) - self.__skipheaders = skipheaders - - def _write_headers(self, msg): - if not self.__skipheaders: - Generator._write_headers(self, msg) - - -def safe_strftime(fmt, t): - try: - return time.strftime(fmt, t) - except (TypeError, ValueError, OverflowError): - return None - - -def calculate_attachments_dir(mlist, msg, msgdata): - # Calculate the directory that attachments for this message will go - # under. To avoid inode limitations, the scheme will be: - # archives/private/<listname>/attachments/YYYYMMDD/<msgid-hash>/<files> - # Start by calculating the date-based and msgid-hash components. - fmt = '%Y%m%d' - datestr = msg.get('Date') - if datestr: - now = parsedate(datestr) - else: - now = time.gmtime(msgdata.get('received_time', time.time())) - datedir = safe_strftime(fmt, now) - if not datedir: - datestr = msgdata.get('X-List-Received-Date') - if datestr: - datedir = safe_strftime(fmt, datestr) - if not datedir: - # What next? Unixfrom, I guess. - parts = msg.get_unixfrom().split() - try: - month = {'Jan':1, 'Feb':2, 'Mar':3, 'Apr':4, 'May':5, 'Jun':6, - 'Jul':7, 'Aug':8, 'Sep':9, 'Oct':10, 'Nov':11, 'Dec':12, - }.get(parts[3], 0) - day = int(parts[4]) - year = int(parts[6]) - except (IndexError, ValueError): - # Best we can do I think - month = day = year = 0 - datedir = '%04d%02d%02d' % (year, month, day) - assert datedir - # As for the msgid hash, we'll base this part on the Message-ID: so that - # all attachments for the same message end up in the same directory (we'll - # uniquify the filenames in that directory as needed). We use the first 2 - # and last 2 bytes of the SHA1 hash of the message id as the basis of the - # directory name. Clashes here don't really matter too much, and that - # still gives us a 32-bit space to work with. - msgid = msg['message-id'] - if msgid is None: - msgid = msg['Message-ID'] = make_msgid() - # We assume that the message id actually /is/ unique! - digest = sha.new(msgid).hexdigest() - return os.path.join('attachments', datedir, digest[:4] + digest[-4:]) - - -def replace_payload_by_text(msg, text, charset): - # TK: This is a common function in replacing the attachment and the main - # message by a text (scrubbing). - del msg['content-type'] - del msg['content-transfer-encoding'] - if isinstance(text, unicode): - text = text.encode(charset) - if not isinstance(charset, str): - charset = str(charset) - msg.set_payload(text, charset) - - - -def process(mlist, msg, msgdata=None): - sanitize = config.ARCHIVE_HTML_SANITIZER - outer = True - if msgdata is None: - msgdata = {} - if msgdata: - # msgdata is available if it is in GLOBAL_PIPELINE - # ie. not in digest or archiver - # check if the list owner want to scrub regular delivery - if not mlist.scrub_nondigest: - return - dir = calculate_attachments_dir(mlist, msg, msgdata) - charset = format = delsp = None - lcset = Utils.GetCharSet(mlist.preferred_language) - lcset_out = Charset(lcset).output_charset or lcset - # Now walk over all subparts of this message and scrub out various types - for part in msg.walk(): - ctype = part.get_content_type() - # If the part is text/plain, we leave it alone - if ctype == 'text/plain': - # We need to choose a charset for the scrubbed message, so we'll - # arbitrarily pick the charset of the first text/plain part in the - # message. - # - # Also get the RFC 3676 stuff from this part. This seems to - # work okay for scrub_nondigest. It will also work as far as - # scrubbing messages for the archive is concerned, but Pipermail - # doesn't pay any attention to the RFC 3676 parameters. The plain - # format digest is going to be a disaster in any case as some of - # messages will be format="flowed" and some not. ToDigest creates - # its own Content-Type: header for the plain digest which won't - # have RFC 3676 parameters. If the message Content-Type: headers - # are retained for display in the digest, the parameters will be - # there for information, but not for the MUA. This is the best we - # can do without having get_payload() process the parameters. - if charset is None: - charset = part.get_content_charset(lcset) - format = part.get_param('format') - delsp = part.get_param('delsp') - # TK: if part is attached then check charset and scrub if none - if part.get('content-disposition') and \ - not part.get_content_charset(): - url = save_attachment(mlist, part, dir) - filename = part.get_filename(_('not available')) - filename = Utils.oneline(filename, lcset) - replace_payload_by_text(part, _("""\ -An embedded and charset-unspecified text was scrubbed... -Name: %(filename)s -URL: %(url)s -"""), lcset) - elif ctype == 'text/html' and isinstance(sanitize, int): - if sanitize == 0: - if outer: - raise DiscardMessage - replace_payload_by_text(part, - _('HTML attachment scrubbed and removed'), - # Adding charset arg and removing content-type - # sets content-type to text/plain - lcset) - elif sanitize == 2: - # By leaving it alone, Pipermail will automatically escape it - pass - elif sanitize == 3: - # Pull it out as an attachment but leave it unescaped. This - # is dangerous, but perhaps useful for heavily moderated - # lists. - url = save_attachment(mlist, part, dir, filter_html=False) - replace_payload_by_text(part, _("""\ -An HTML attachment was scrubbed... -URL: %(url)s -"""), lcset) - else: - # HTML-escape it and store it as an attachment, but make it - # look a /little/ bit prettier. :( - payload = Utils.websafe(part.get_payload(decode=True)) - # For whitespace in the margin, change spaces into - # non-breaking spaces, and tabs into 8 of those. Then use a - # mono-space font. Still looks hideous to me, but then I'd - # just as soon discard them. - def doreplace(s): - return s.replace(' ', ' ').replace('\t', ' '*8) - lines = [doreplace(s) for s in payload.split('\n')] - payload = '<tt>\n' + BR.join(lines) + '\n</tt>\n' - part.set_payload(payload) - # We're replacing the payload with the decoded payload so this - # will just get in the way. - del part['content-transfer-encoding'] - url = save_attachment(mlist, part, dir, filter_html=False) - replace_payload_by_text(part, _("""\ -An HTML attachment was scrubbed... -URL: %(url)s -"""), lcset) - elif ctype == 'message/rfc822': - # This part contains a submessage, so it too needs scrubbing - submsg = part.get_payload(0) - url = save_attachment(mlist, part, dir) - subject = submsg.get('subject', _('no subject')) - date = submsg.get('date', _('no date')) - who = submsg.get('from', _('unknown sender')) - size = len(str(submsg)) - replace_payload_by_text(part, _("""\ -An embedded message was scrubbed... -From: %(who)s -Subject: %(subject)s -Date: %(date)s -Size: %(size)s -URL: %(url)s -"""), lcset) - # If the message isn't a multipart, then we'll strip it out as an - # attachment that would have to be separately downloaded. Pipermail - # will transform the url into a hyperlink. - elif part._payload and not part.is_multipart(): - payload = part.get_payload(decode=True) - ctype = part.get_content_type() - # XXX Under email 2.5, it is possible that payload will be None. - # This can happen when you have a Content-Type: multipart/* with - # only one part and that part has two blank lines between the - # first boundary and the end boundary. In email 3.0 you end up - # with a string in the payload. I think in this case it's safe to - # ignore the part. - if payload is None: - continue - size = len(payload) - url = save_attachment(mlist, part, dir) - desc = part.get('content-description', _('not available')) - desc = Utils.oneline(desc, lcset) - filename = part.get_filename(_('not available')) - filename = Utils.oneline(filename, lcset) - replace_payload_by_text(part, _("""\ -A non-text attachment was scrubbed... -Name: %(filename)s -Type: %(ctype)s -Size: %(size)d bytes -Desc: %(desc)s -URL: %(url)s -"""), lcset) - outer = False - # We still have to sanitize multipart messages to flat text because - # Pipermail can't handle messages with list payloads. This is a kludge; - # def (n) clever hack ;). - if msg.is_multipart() and sanitize <> 2: - # By default we take the charset of the first text/plain part in the - # message, but if there was none, we'll use the list's preferred - # language's charset. - if not charset or charset == 'us-ascii': - charset = lcset_out - else: - # normalize to the output charset if input/output are different - charset = Charset(charset).output_charset or charset - # We now want to concatenate all the parts which have been scrubbed to - # text/plain, into a single text/plain payload. We need to make sure - # all the characters in the concatenated string are in the same - # encoding, so we'll use the 'replace' key in the coercion call. - # BAW: Martin's original patch suggested we might want to try - # generalizing to utf-8, and that's probably a good idea (eventually). - text = [] - charsets = [] - for part in msg.walk(): - # TK: bug-id 1099138 and multipart - # MAS test payload - if part may fail if there are no headers. - if not part._payload or part.is_multipart(): - continue - # All parts should be scrubbed to text/plain by now. - partctype = part.get_content_type() - if partctype <> 'text/plain': - text.append(_('Skipped content of type %(partctype)s\n')) - continue - try: - t = part.get_payload(decode=True) or '' - # MAS: TypeError exception can occur if payload is None. This - # was observed with a message that contained an attached - # message/delivery-status part. Because of the special parsing - # of this type, this resulted in a text/plain sub-part with a - # null body. See bug 1430236. - except (binascii.Error, TypeError): - t = part.get_payload() or '' - # Email problem was solved by Mark Sapiro. (TK) - partcharset = part.get_content_charset('us-ascii') - try: - t = unicode(t, partcharset, 'replace') - except (UnicodeError, LookupError, ValueError, TypeError, - AssertionError): - # We can get here if partcharset is bogus in come way. - # Replace funny characters. We use errors='replace'. - t = unicode(t, 'ascii', 'replace') - # Separation is useful - if isinstance(t, basestring): - if not t.endswith('\n'): - t += '\n' - text.append(t) - if partcharset not in charsets: - charsets.append(partcharset) - # Now join the text and set the payload - sep = _('-------------- next part --------------\n') - assert isinstance(sep, unicode), ( - 'Expected a unicode separator, got %s' % type(sep)) - rept = sep.join(text) - # Replace entire message with text and scrubbed notice. - # Try with message charsets and utf-8 - if 'utf-8' not in charsets: - charsets.append('utf-8') - for charset in charsets: - try: - replace_payload_by_text(msg, rept, charset) - break - # Bogus charset can throw several exceptions - except (UnicodeError, LookupError, ValueError, TypeError, - AssertionError): - pass - if format: - msg.set_param('format', format) - if delsp: - msg.set_param('delsp', delsp) - return msg - - - -def makedirs(dir): - # Create all the directories to store this attachment in and try to make - # sure that the permissions of the directories are set correctly. - try: - os.makedirs(dir, 02775) - except OSError, e: - if e.errno == errno.EEXIST: - return - # Some systems such as FreeBSD ignore mkdir's mode, so walk the just - # created directories and try to set the mode, ignoring any OSErrors that - # occur here. - for dirpath, dirnames, filenames in os.walk(dir): - try: - os.chmod(dirpath, 02775) - except OSError: - pass - - - -def save_attachment(mlist, msg, dir, filter_html=True): - fsdir = os.path.join(config.PRIVATE_ARCHIVE_FILE_DIR, - mlist.fqdn_listname, dir) - makedirs(fsdir) - # Figure out the attachment type and get the decoded data - decodedpayload = msg.get_payload(decode=True) - # BAW: mimetypes ought to handle non-standard, but commonly found types, - # e.g. image/jpg (should be image/jpeg). For now we just store such - # things as application/octet-streams since that seems the safest. - ctype = msg.get_content_type() - # i18n file name is encoded - lcset = Utils.GetCharSet(mlist.preferred_language) - filename = Utils.oneline(msg.get_filename(''), lcset) - filename, fnext = os.path.splitext(filename) - # For safety, we should confirm this is valid ext for content-type - # but we can use fnext if we introduce fnext filtering - if config.SCRUBBER_USE_ATTACHMENT_FILENAME_EXTENSION: - # HTML message doesn't have filename :-( - ext = fnext or guess_extension(ctype, fnext) - else: - ext = guess_extension(ctype, fnext) - if not ext: - # We don't know what it is, so assume it's just a shapeless - # application/octet-stream, unless the Content-Type: is - # message/rfc822, in which case we know we'll coerce the type to - # text/plain below. - if ctype == 'message/rfc822': - ext = '.txt' - else: - ext = '.bin' - # Allow only alphanumerics, dash, underscore, and dot - ext = sre.sub('', ext) - path = None - # We need a lock to calculate the next attachment number - with Lock(os.path.join(fsdir, 'attachments.lock')): - # Now base the filename on what's in the attachment, uniquifying it if - # necessary. - if not filename or config.SCRUBBER_DONT_USE_ATTACHMENT_FILENAME: - filebase = 'attachment' - else: - # Sanitize the filename given in the message headers - parts = pre.split(filename) - filename = parts[-1] - # Strip off leading dots - filename = dre.sub('', filename) - # Allow only alphanumerics, dash, underscore, and dot - filename = sre.sub('', filename) - # If the filename's extension doesn't match the type we guessed, - # which one should we go with? For now, let's go with the one we - # guessed so attachments can't lie about their type. Also, if the - # filename /has/ no extension, then tack on the one we guessed. - # The extension was removed from the name above. - filebase = filename - # Now we're looking for a unique name for this file on the file - # system. If msgdir/filebase.ext isn't unique, we'll add a counter - # after filebase, e.g. msgdir/filebase-cnt.ext - counter = 0 - extra = '' - while True: - path = os.path.join(fsdir, filebase + extra + ext) - # Generally it is not a good idea to test for file existance - # before just trying to create it, but the alternatives aren't - # wonderful (i.e. os.open(..., O_CREAT | O_EXCL) isn't - # NFS-safe). Besides, we have an exclusive lock now, so we're - # guaranteed that no other process will be racing with us. - if os.path.exists(path): - counter += 1 - extra = '-%04d' % counter - else: - break - # `path' now contains the unique filename for the attachment. There's - # just one more step we need to do. If the part is text/html and - # ARCHIVE_HTML_SANITIZER is a string (which it must be or we wouldn't be - # here), then send the attachment through the filter program for - # sanitization - if filter_html and ctype == 'text/html': - base, ext = os.path.splitext(path) - tmppath = base + '-tmp' + ext - fp = open(tmppath, 'w') - try: - fp.write(decodedpayload) - fp.close() - cmd = config.ARCHIVE_HTML_SANITIZER % {'filename' : tmppath} - progfp = os.popen(cmd, 'r') - decodedpayload = progfp.read() - status = progfp.close() - if status: - log.error('HTML sanitizer exited with non-zero status: %s', - status) - finally: - os.unlink(tmppath) - # BAW: Since we've now sanitized the document, it should be plain - # text. Blarg, we really want the sanitizer to tell us what the type - # if the return data is. :( - ext = '.txt' - path = base + '.txt' - # Is it a message/rfc822 attachment? - elif ctype == 'message/rfc822': - submsg = msg.get_payload() - # BAW: I'm sure we can eventually do better than this. :( - decodedpayload = Utils.websafe(str(submsg)) - fp = open(path, 'w') - fp.write(decodedpayload) - fp.close() - # Now calculate the url to the list's archive. - baseurl = get_archiver().get_list_url(mlist) - if not baseurl.endswith('/'): - baseurl += '/' - # Trailing space will definitely be a problem with format=flowed. - # Bracket the URL instead. - url = '<' + baseurl + '%s/%s%s%s>' % (dir, filebase, extra, ext) - return url diff --git a/Mailman/Handlers/Tagger.py b/Mailman/Handlers/Tagger.py deleted file mode 100644 index 023148fd7..000000000 --- a/Mailman/Handlers/Tagger.py +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright (C) 2001-2007 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""Extract topics from the original mail message.""" - -import re -import email -import email.Errors -import email.Iterators -import email.Parser - -OR = '|' -CRNL = '\r\n' -EMPTYSTRING = '' -NLTAB = '\n\t' - - - -def process(mlist, msg, msgdata): - if not mlist.topics_enabled: - return - # Extract the Subject:, Keywords:, and possibly body text - matchlines = [] - matchlines.append(msg.get('subject', None)) - matchlines.append(msg.get('keywords', None)) - if mlist.topics_bodylines_limit == 0: - # Don't scan any body lines - pass - elif mlist.topics_bodylines_limit < 0: - # Scan all body lines - matchlines.extend(scanbody(msg)) - else: - # Scan just some of the body lines - matchlines.extend(scanbody(msg, mlist.topics_bodylines_limit)) - matchlines = filter(None, matchlines) - # For each regular expression in the topics list, see if any of the lines - # of interest from the message match the regexp. If so, the message gets - # added to the specific topics bucket. - hits = {} - for name, pattern, desc, emptyflag in mlist.topics: - pattern = OR.join(pattern.splitlines()) - cre = re.compile(pattern, re.IGNORECASE) - for line in matchlines: - if cre.search(line): - hits[name] = 1 - break - if hits: - msgdata['topichits'] = hits.keys() - msg['X-Topics'] = NLTAB.join(hits.keys()) - - - -def scanbody(msg, numlines=None): - # We only scan the body of the message if it is of MIME type text/plain, - # or if the outer type is multipart/alternative and there is a text/plain - # part. Anything else, and the body is ignored for header-scan purposes. - found = None - if msg.get_content_type() == 'text/plain': - found = msg - elif msg.is_multipart()\ - and msg.get_content_type() == 'multipart/alternative': - for found in msg.get_payload(): - if found.get_content_type() == 'text/plain': - break - else: - found = None - if not found: - return [] - # Now that we have a Message object that meets our criteria, let's extract - # the first numlines of body text. - lines = [] - lineno = 0 - reader = list(email.Iterators.body_line_iterator(msg)) - while numlines is None or lineno < numlines: - try: - line = reader.pop(0) - except IndexError: - break - # Blank lines don't count - if not line.strip(): - continue - lineno += 1 - lines.append(line) - # Concatenate those body text lines with newlines, and then create a new - # message object from those lines. - p = _ForgivingParser() - msg = p.parsestr(EMPTYSTRING.join(lines)) - return msg.get_all('subject', []) + msg.get_all('keywords', []) - - - -class _ForgivingParser(email.Parser.HeaderParser): - # Be a little more forgiving about non-header/continuation lines, since - # we'll just read as much as we can from "header-like" lines in the body. - # - # BAW: WIBNI we didn't have to cut-n-paste this whole thing just to - # specialize the way it returns? - def _parseheaders(self, container, fp): - # Parse the headers, returning a list of header/value pairs. None as - # the header means the Unix-From header. - lastheader = '' - lastvalue = [] - lineno = 0 - while 1: - # Don't strip the line before we test for the end condition, - # because whitespace-only header lines are RFC compliant - # continuation lines. - line = fp.readline() - if not line: - break - line = line.splitlines()[0] - if not line: - break - # Ignore the trailing newline - lineno += 1 - # Check for initial Unix From_ line - if line.startswith('From '): - if lineno == 1: - container.set_unixfrom(line) - continue - else: - break - # Header continuation line - if line[0] in ' \t': - if not lastheader: - break - lastvalue.append(line) - continue - # Normal, non-continuation header. BAW: this should check to make - # sure it's a legal header, e.g. doesn't contain spaces. Also, we - # should expose the header matching algorithm in the API, and - # allow for a non-strict parsing mode (that ignores the line - # instead of raising the exception). - i = line.find(':') - if i < 0: - break - if lastheader: - container[lastheader] = NLTAB.join(lastvalue) - lastheader = line[:i] - lastvalue = [line[i+1:].lstrip()] - # Make sure we retain the last header - if lastheader: - container[lastheader] = NLTAB.join(lastvalue) diff --git a/Mailman/Handlers/ToArchive.py b/Mailman/Handlers/ToArchive.py deleted file mode 100644 index c65e86f60..000000000 --- a/Mailman/Handlers/ToArchive.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (C) 1998-2007 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""Add the message to the archives.""" - -from Mailman.configuration import config -from Mailman.queue import Switchboard - - - -def process(mlist, msg, msgdata): - # short circuits - if msgdata.get('isdigest') or not mlist.archive: - return - # Common practice seems to favor "X-No-Archive: yes". No other value for - # this header seems to make sense, so we'll just test for it's presence. - # I'm keeping "X-Archive: no" for backwards compatibility. - if 'x-no-archive' in msg or msg.get('x-archive', '').lower() == 'no': - return - # Send the message to the archiver queue - archq = Switchboard(config.ARCHQUEUE_DIR) - # Send the message to the queue - archq.enqueue(msg, msgdata) diff --git a/Mailman/Handlers/ToDigest.py b/Mailman/Handlers/ToDigest.py deleted file mode 100644 index 6f06df43b..000000000 --- a/Mailman/Handlers/ToDigest.py +++ /dev/null @@ -1,420 +0,0 @@ -# Copyright (C) 1998-2007 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""Add the message to the list's current digest and possibly send it.""" - -# Messages are accumulated to a Unix mailbox compatible file containing all -# the messages destined for the digest. This file must be parsable by the -# mailbox.UnixMailbox class (i.e. it must be ^From_ quoted). -# -# When the file reaches the size threshold, it is moved to the qfiles/digest -# directory and the DigestRunner will craft the MIME, rfc1153, and -# (eventually) URL-subject linked digests from the mbox. - -from __future__ import with_statement - -import os -import re -import copy -import time -import logging - -from StringIO import StringIO # cStringIO can't handle unicode. -from email.charset import Charset -from email.generator import Generator -from email.header import decode_header, make_header, Header -from email.mime.base import MIMEBase -from email.mime.message import MIMEMessage -from email.mime.text import MIMEText -from email.parser import Parser -from email.utils import formatdate, getaddresses, make_msgid - -from Mailman import Errors -from Mailman import Message -from Mailman import Utils -from Mailman import i18n -from Mailman.Handlers.Decorate import decorate -from Mailman.Handlers.Scrubber import process as scrubber -from Mailman.Mailbox import Mailbox -from Mailman.Mailbox import Mailbox -from Mailman.configuration import config -from Mailman.interfaces import DeliveryMode, DeliveryStatus -from Mailman.queue import Switchboard - -_ = i18n._ -__i18n_templates__ = True - -UEMPTYSTRING = u'' -EMPTYSTRING = '' - -log = logging.getLogger('mailman.error') - - - -def process(mlist, msg, msgdata): - # Short circuit non-digestable lists. - if not mlist.digestable or msgdata.get('isdigest'): - return - mboxfile = os.path.join(mlist.full_path, 'digest.mbox') - mboxfp = open(mboxfile, 'a+') - mbox = Mailbox(mboxfp) - mbox.AppendMessage(msg) - # Calculate the current size of the accumulation file. This will not tell - # us exactly how big the MIME, rfc1153, or any other generated digest - # message will be, but it's the most easily available metric to decide - # whether the size threshold has been reached. - mboxfp.flush() - size = os.path.getsize(mboxfile) - if size / 1024.0 >= mlist.digest_size_threshold: - # This is a bit of a kludge to get the mbox file moved to the digest - # queue directory. - try: - # Enclose in try/except here because a error in send_digest() can - # silently stop regular delivery. Unsuccessful digest delivery - # should be tried again by cron and the site administrator will be - # notified of any error explicitly by the cron error message. - mboxfp.seek(0) - send_digests(mlist, mboxfp) - os.unlink(mboxfile) - except Exception, errmsg: - # Bare except is generally prohibited in Mailman, but we can't - # forecast what exceptions can occur here. - log.exception('send_digests() failed: %s', errmsg) - mboxfp.close() - - - -def send_digests(mlist, mboxfp): - # Set the digest volume and time - if mlist.digest_last_sent_at: - bump = False - # See if we should bump the digest volume number - timetup = time.localtime(mlist.digest_last_sent_at) - now = time.localtime(time.time()) - freq = mlist.digest_volume_frequency - if freq == 0 and timetup[0] < now[0]: - # Yearly - bump = True - elif freq == 1 and timetup[1] <> now[1]: - # Monthly, but we take a cheap way to calculate this. We assume - # that the clock isn't going to be reset backwards. - bump = True - elif freq == 2 and (timetup[1] % 4 <> now[1] % 4): - # Quarterly, same caveat - bump = True - elif freq == 3: - # Once again, take a cheap way of calculating this - weeknum_last = int(time.strftime('%W', timetup)) - weeknum_now = int(time.strftime('%W', now)) - if weeknum_now > weeknum_last or timetup[0] > now[0]: - bump = True - elif freq == 4 and timetup[7] <> now[7]: - # Daily - bump = True - if bump: - mlist.bump_digest_volume() - mlist.digest_last_sent_at = time.time() - # Wrapper around actually digest crafter to set up the language context - # properly. All digests are translated to the list's preferred language. - with i18n.using_language(mlist.preferred_language): - send_i18n_digests(mlist, mboxfp) - - - -def send_i18n_digests(mlist, mboxfp): - mbox = Mailbox(mboxfp) - # Prepare common information (first lang/charset) - lang = mlist.preferred_language - lcset = Utils.GetCharSet(lang) - lcset_out = Charset(lcset).output_charset or lcset - # Common Information (contd) - realname = mlist.real_name - volume = mlist.volume - issue = mlist.next_digest_number - digestid = _('$realname Digest, Vol $volume, Issue $issue') - digestsubj = Header(digestid, lcset, header_name='Subject') - # Set things up for the MIME digest. Only headers not added by - # CookHeaders need be added here. - # Date/Message-ID should be added here also. - mimemsg = Message.Message() - mimemsg['Content-Type'] = 'multipart/mixed' - mimemsg['MIME-Version'] = '1.0' - mimemsg['From'] = mlist.request_address - mimemsg['Subject'] = digestsubj - mimemsg['To'] = mlist.posting_address - mimemsg['Reply-To'] = mlist.posting_address - mimemsg['Date'] = formatdate(localtime=1) - mimemsg['Message-ID'] = make_msgid() - # Set things up for the rfc1153 digest - plainmsg = StringIO() - rfc1153msg = Message.Message() - rfc1153msg['From'] = mlist.request_address - rfc1153msg['Subject'] = digestsubj - rfc1153msg['To'] = mlist.posting_address - rfc1153msg['Reply-To'] = mlist.posting_address - rfc1153msg['Date'] = formatdate(localtime=1) - rfc1153msg['Message-ID'] = make_msgid() - separator70 = '-' * 70 - separator30 = '-' * 30 - # In the rfc1153 digest, the masthead contains the digest boilerplate plus - # any digest header. In the MIME digests, the masthead and digest header - # are separate MIME subobjects. In either case, it's the first thing in - # the digest, and we can calculate it now, so go ahead and add it now. - mastheadtxt = Utils.maketext( - 'masthead.txt', - {'real_name' : mlist.real_name, - 'got_list_email': mlist.posting_address, - 'got_listinfo_url': mlist.script_url('listinfo'), - 'got_request_email': mlist.request_address, - 'got_owner_email': mlist.owner_address, - }, mlist=mlist) - # MIME - masthead = MIMEText(mastheadtxt.encode(lcset), _charset=lcset) - masthead['Content-Description'] = digestid - mimemsg.attach(masthead) - # RFC 1153 - print >> plainmsg, mastheadtxt - print >> plainmsg - # Now add the optional digest header - if mlist.digest_header: - headertxt = decorate(mlist, mlist.digest_header, _('digest header')) - # MIME - header = MIMEText(headertxt.encode(lcset), _charset=lcset) - header['Content-Description'] = _('Digest Header') - mimemsg.attach(header) - # RFC 1153 - print >> plainmsg, headertxt - print >> plainmsg - # Now we have to cruise through all the messages accumulated in the - # mailbox file. We can't add these messages to the plainmsg and mimemsg - # yet, because we first have to calculate the table of contents - # (i.e. grok out all the Subjects). Store the messages in a list until - # we're ready for them. - # - # Meanwhile prepare things for the table of contents - toc = StringIO() - print >> toc, _("Today's Topics:\n") - # Now cruise through all the messages in the mailbox of digest messages, - # building the MIME payload and core of the RFC 1153 digest. We'll also - # accumulate Subject: headers and authors for the table-of-contents. - messages = [] - msgcount = 0 - msg = mbox.next() - while msg is not None: - if msg == '': - # It was an unparseable message - msg = mbox.next() - continue - msgcount += 1 - messages.append(msg) - # Get the Subject header - msgsubj = msg.get('subject', _('(no subject)')) - subject = Utils.oneline(msgsubj, in_unicode=True) - # Don't include the redundant subject prefix in the toc - mo = re.match('(re:? *)?(%s)' % re.escape(mlist.subject_prefix), - subject, re.IGNORECASE) - if mo: - subject = subject[:mo.start(2)] + subject[mo.end(2):] - username = '' - addresses = getaddresses([Utils.oneline(msg.get('from', ''), - in_unicode=True)]) - # Take only the first author we find - if isinstance(addresses, list) and addresses: - username = addresses[0][0] - if not username: - username = addresses[0][1] - if username: - username = ' (%s)' % username - # Put count and Wrap the toc subject line - wrapped = Utils.wrap('%2d. %s' % (msgcount, subject), 65) - slines = wrapped.split('\n') - # See if the user's name can fit on the last line - if len(slines[-1]) + len(username) > 70: - slines.append(username) - else: - slines[-1] += username - # Add this subject to the accumulating topics - first = True - for line in slines: - if first: - print >> toc, ' ', line - first = False - else: - print >> toc, ' ', line.lstrip() - # We do not want all the headers of the original message to leak - # through in the digest messages. For this phase, we'll leave the - # same set of headers in both digests, i.e. those required in RFC 1153 - # plus a couple of other useful ones. We also need to reorder the - # headers according to RFC 1153. Later, we'll strip out headers for - # for the specific MIME or plain digests. - keeper = {} - all_keepers = {} - for header in (config.MIME_DIGEST_KEEP_HEADERS + - config.PLAIN_DIGEST_KEEP_HEADERS): - all_keepers[header] = True - all_keepers = all_keepers.keys() - for keep in all_keepers: - keeper[keep] = msg.get_all(keep, []) - # Now remove all unkempt headers :) - for header in msg.keys(): - del msg[header] - # And add back the kept header in the RFC 1153 designated order - for keep in all_keepers: - for field in keeper[keep]: - msg[keep] = field - # And a bit of extra stuff - msg['Message'] = `msgcount` - # Get the next message in the digest mailbox - msg = mbox.next() - # Now we're finished with all the messages in the digest. First do some - # sanity checking and then on to adding the toc. - if msgcount == 0: - # Why did we even get here? - return - toctext = toc.getvalue() - # MIME - try: - tocpart = MIMEText(toctext.encode(lcset), _charset=lcset) - except UnicodeError: - tocpart = MIMEText(toctext.encode('utf-8'), _charset='utf-8') - tocpart['Content-Description']= _("Today's Topics ($msgcount messages)") - mimemsg.attach(tocpart) - # RFC 1153 - print >> plainmsg, toctext - print >> plainmsg - # For RFC 1153 digests, we now need the standard separator - print >> plainmsg, separator70 - print >> plainmsg - # Now go through and add each message - mimedigest = MIMEBase('multipart', 'digest') - mimemsg.attach(mimedigest) - first = True - for msg in messages: - # MIME. Make a copy of the message object since the rfc1153 - # processing scrubs out attachments. - mimedigest.attach(MIMEMessage(copy.deepcopy(msg))) - # rfc1153 - if first: - first = False - else: - print >> plainmsg, separator30 - print >> plainmsg - # Use Mailman.Handlers.Scrubber.process() to get plain text - try: - msg = scrubber(mlist, msg) - except Errors.DiscardMessage: - print >> plainmsg, _('[Message discarded by content filter]') - continue - # Honor the default setting - for h in config.PLAIN_DIGEST_KEEP_HEADERS: - if msg[h]: - uh = Utils.wrap('%s: %s' % (h, Utils.oneline(msg[h], - in_unicode=True))) - uh = '\n\t'.join(uh.split('\n')) - print >> plainmsg, uh - print >> plainmsg - # If decoded payload is empty, this may be multipart message. - # -- just stringfy it. - payload = msg.get_payload(decode=True) \ - or msg.as_string().split('\n\n',1)[1] - mcset = msg.get_content_charset('us-ascii') - try: - payload = unicode(payload, mcset, 'replace') - except (LookupError, TypeError): - # unknown or empty charset - payload = unicode(payload, 'us-ascii', 'replace') - print >> plainmsg, payload - if not payload.endswith('\n'): - print >> plainmsg - # Now add the footer - if mlist.digest_footer: - footertxt = decorate(mlist, mlist.digest_footer, _('digest footer')) - # MIME - footer = MIMEText(footertxt.encode(lcset), _charset=lcset) - footer['Content-Description'] = _('Digest Footer') - mimemsg.attach(footer) - # RFC 1153 - # BAW: This is not strictly conformant RFC 1153. The trailer is only - # supposed to contain two lines, i.e. the "End of ... Digest" line and - # the row of asterisks. If this screws up MUAs, the solution is to - # add the footer as the last message in the RFC 1153 digest. I just - # hate the way that VM does that and I think it's confusing to users, - # so don't do it unless there's a clamor. - print >> plainmsg, separator30 - print >> plainmsg - print >> plainmsg, footertxt - print >> plainmsg - # Do the last bit of stuff for each digest type - signoff = _('End of ') + digestid - # MIME - # BAW: This stuff is outside the normal MIME goo, and it's what the old - # MIME digester did. No one seemed to complain, probably because you - # won't see it in an MUA that can't display the raw message. We've never - # got complaints before, but if we do, just wax this. It's primarily - # included for (marginally useful) backwards compatibility. - mimemsg.postamble = signoff - # rfc1153 - print >> plainmsg, signoff - print >> plainmsg, '*' * len(signoff) - # Do our final bit of housekeeping, and then send each message to the - # outgoing queue for delivery. - mlist.next_digest_number += 1 - virginq = Switchboard(config.VIRGINQUEUE_DIR) - # Calculate the recipients lists - plainrecips = set() - mimerecips = set() - # When someone turns off digest delivery, they will get one last digest to - # ensure that there will be no gaps in the messages they receive. - # Currently, this dictionary contains the email addresses of those folks - # who should get one last digest. We need to find the corresponding - # IMember records. - digest_members = set(mlist.digest_members.members) - for address in mlist.one_last_digest: - member = mlist.digest_members.get_member(address) - if member: - digest_members.add(member) - for member in digest_members: - if member.delivery_status <> DeliveryStatus.enabled: - continue - # Send the digest to the case-preserved address of the digest members. - email_address = member.address.original_address - if member.delivery_mode == DeliveryMode.plaintext_digests: - plainrecips.add(email_address) - elif member.delivery_mode == DeliveryMode.mime_digests: - mimerecips.add(email_address) - else: - raise AssertionError( - 'Digest member "%s" unexpected delivery mode: %s' % - (email_address, member.delivery_mode)) - # Zap this since we're now delivering the last digest to these folks. - mlist.one_last_digest.clear() - # MIME - virginq.enqueue(mimemsg, - recips=mimerecips, - listname=mlist.fqdn_listname, - isdigest=True) - # RFC 1153 - # If the entire digest message can't be encoded by list charset, fall - # back to 'utf-8'. - try: - rfc1153msg.set_payload(plainmsg.getvalue().encode(lcset), lcset) - except UnicodeError: - rfc1153msg.set_payload(plainmsg.getvalue().encode('utf-8'), 'utf-8') - virginq.enqueue(rfc1153msg, - recips=plainrecips, - listname=mlist.fqdn_listname, - isdigest=True) diff --git a/Mailman/Handlers/ToOutgoing.py b/Mailman/Handlers/ToOutgoing.py deleted file mode 100644 index 9c8650a98..000000000 --- a/Mailman/Handlers/ToOutgoing.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (C) 1998-2007 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""Re-queue the message to the outgoing queue. - -This module is only for use by the IncomingRunner for delivering messages -posted to the list membership. Anything else that needs to go out to some -recipient should just be placed in the out queue directly. -""" - -from Mailman.configuration import config -from Mailman.interfaces import Personalization -from Mailman.queue import Switchboard - - - -def process(mlist, msg, msgdata): - interval = config.VERP_DELIVERY_INTERVAL - # Should we VERP this message? If personalization is enabled for this - # list and VERP_PERSONALIZED_DELIVERIES is true, then yes we VERP it. - # Also, if personalization is /not/ enabled, but VERP_DELIVERY_INTERVAL is - # set (and we've hit this interval), then again, this message should be - # VERPed. Otherwise, no. - # - # Note that the verp flag may already be set, e.g. by mailpasswds using - # VERP_PASSWORD_REMINDERS. Preserve any existing verp flag. - if 'verp' in msgdata: - pass - elif mlist.personalize <> Personalization.none: - if config.VERP_PERSONALIZED_DELIVERIES: - msgdata['verp'] = True - elif interval == 0: - # Never VERP - pass - elif interval == 1: - # VERP every time - msgdata['verp'] = True - else: - # VERP every `interval' number of times - msgdata['verp'] = not (int(mlist.post_id) % interval) - # And now drop the message in qfiles/out - outq = Switchboard(config.OUTQUEUE_DIR) - outq.enqueue(msg, msgdata, listname=mlist.fqdn_listname) diff --git a/Mailman/Handlers/ToUsenet.py b/Mailman/Handlers/ToUsenet.py deleted file mode 100644 index 09bb28bdd..000000000 --- a/Mailman/Handlers/ToUsenet.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (C) 1998-2007 by the Free Software Foundation, Inc. -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, -# USA. - -"""Move the message to the mail->news queue.""" - -import logging - -from Mailman.configuration import config -from Mailman.queue import Switchboard - -COMMASPACE = ', ' - -log = logging.getLogger('mailman.error') - - - -def process(mlist, msg, msgdata): - # short circuits - if not mlist.gateway_to_news or \ - msgdata.get('isdigest') or \ - msgdata.get('fromusenet'): - return - # sanity checks - error = [] - if not mlist.linked_newsgroup: - error.append('no newsgroup') - if not mlist.nntp_host: - error.append('no NNTP host') - if error: - log.error('NNTP gateway improperly configured: %s', - COMMASPACE.join(error)) - return - # Put the message in the news runner's queue - newsq = Switchboard(config.NEWSQUEUE_DIR) - newsq.enqueue(msg, msgdata, listname=mlist.fqdn_listname) diff --git a/Mailman/Handlers/__init__.py b/Mailman/Handlers/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/Mailman/Handlers/__init__.py +++ /dev/null |